Move source code into "Source" folder

This commit is contained in:
Robert McRackan 2022-05-09 10:31:45 -04:00
parent 1ee73fa1a7
commit 389fbb2371
287 changed files with 26 additions and 8 deletions

View file

@ -0,0 +1,157 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace FileManager
{
/// <summary>
/// Tracks actual locations of files.
/// </summary>
public class BackgroundFileSystem
{
public string RootDirectory { get; private set; }
public string SearchPattern { get; private set; }
public SearchOption SearchOption { get; private set; }
private FileSystemWatcher fileSystemWatcher { get; set; }
private BlockingCollection<FileSystemEventArgs> directoryChangesEvents { get; set; }
private Task backgroundScanner { get; set; }
private object fsCacheLocker { get; } = new();
private List<string> fsCache { get; } = new();
public BackgroundFileSystem(string rootDirectory, string searchPattern, SearchOption searchOptions)
{
RootDirectory = rootDirectory;
SearchPattern = searchPattern;
SearchOption = searchOptions;
Init();
}
public string FindFile(System.Text.RegularExpressions.Regex regex)
{
lock (fsCacheLocker)
return fsCache.FirstOrDefault(s => regex.IsMatch(s));
}
public void RefreshFiles()
{
lock (fsCacheLocker)
{
fsCache.Clear();
fsCache.AddRange(FileUtility.SaferEnumerateFiles(RootDirectory, SearchPattern, SearchOption));
}
}
private void Init()
{
Stop();
lock (fsCacheLocker)
fsCache.AddRange(FileUtility.SaferEnumerateFiles(RootDirectory, SearchPattern, SearchOption));
directoryChangesEvents = new BlockingCollection<FileSystemEventArgs>();
fileSystemWatcher = new FileSystemWatcher(RootDirectory)
{
IncludeSubdirectories = true,
EnableRaisingEvents = true
};
fileSystemWatcher.Created += FileSystemWatcher_Changed;
fileSystemWatcher.Deleted += FileSystemWatcher_Changed;
fileSystemWatcher.Renamed += FileSystemWatcher_Changed;
fileSystemWatcher.Error += FileSystemWatcher_Error;
backgroundScanner = new Task(BackgroundScanner);
backgroundScanner.Start();
}
private void Stop()
{
//Stop raising events
fileSystemWatcher?.Dispose();
//Calling CompleteAdding() will cause background scanner to terminate.
directoryChangesEvents?.CompleteAdding();
//Wait for background scanner to terminate before reinitializing.
backgroundScanner?.Wait();
//Dispose of directoryChangesEvents after backgroundScanner exists.
directoryChangesEvents?.Dispose();
lock (fsCacheLocker)
fsCache.Clear();
}
private void FileSystemWatcher_Error(object sender, ErrorEventArgs e)
{
Stop();
Init();
}
private void FileSystemWatcher_Changed(object sender, FileSystemEventArgs e)
{
directoryChangesEvents.Add(e);
}
#region Background Thread
private void BackgroundScanner()
{
while (directoryChangesEvents.TryTake(out FileSystemEventArgs change, -1))
{
lock (fsCacheLocker)
UpdateLocalCache(change);
}
}
private void UpdateLocalCache(FileSystemEventArgs change)
{
if (change.ChangeType == WatcherChangeTypes.Deleted)
{
RemovePath(change.FullPath);
}
else if (change.ChangeType == WatcherChangeTypes.Created)
{
AddPath(change.FullPath);
}
else if (change.ChangeType == WatcherChangeTypes.Renamed && change is RenamedEventArgs renameChange)
{
RemovePath(renameChange.OldFullPath);
AddPath(renameChange.FullPath);
}
}
private void RemovePath(string path)
{
var pathsToRemove = fsCache.Where(p => p.StartsWith(path)).ToArray();
foreach (var p in pathsToRemove)
fsCache.Remove(p);
}
private void AddPath(string path)
{
if (File.GetAttributes(path).HasFlag(FileAttributes.Directory))
AddUniqueFiles(FileUtility.SaferEnumerateFiles(path, SearchPattern, SearchOption));
else
AddUniqueFile(path);
}
private void AddUniqueFiles(IEnumerable<string> newFiles)
{
foreach (var file in newFiles)
AddUniqueFile(file);
}
private void AddUniqueFile(string newFile)
{
if (!fsCache.Contains(newFile))
fsCache.Add(newFile);
}
#endregion
~BackgroundFileSystem() => Stop();
}
}

View file

@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dinah.Core" Version="4.0.6.1" />
<PackageReference Include="Polly" Version="7.2.3" />
</ItemGroup>
</Project>

View file

@ -0,0 +1,64 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Dinah.Core;
namespace FileManager
{
/// <summary>Get valid filename. Advanced features incl. parameterized template</summary>
public class FileNamingTemplate
{
/// <summary>Proposed full file path. May contain optional html-styled template tags. Eg: &lt;name&gt;</summary>
public string Template { get; }
/// <param name="template">Proposed file name with optional html-styled template tags.</param>
public FileNamingTemplate(string template) => Template = ArgumentValidator.EnsureNotNullOrWhiteSpace(template, nameof(template));
/// <summary>Optional step 1: Replace html-styled template tags with parameters. Eg {"name", "Bill Gates"} => /&lt;name&gt;/ => /Bill Gates/</summary>
public Dictionary<string, object> ParameterReplacements { get; } = new Dictionary<string, object>();
/// <summary>Convenience method</summary>
public void AddParameterReplacement(string key, object value)
// using .Add() instead of "[key] = value" will make unintended overwriting throw exception
=> ParameterReplacements.Add(key, value);
/// <summary>If set, truncate each parameter replacement to this many characters. Default 50</summary>
public int? ParameterMaxSize { get; set; } = 50;
/// <summary>Optional step 2: Replace all illegal characters with this. Default=<see cref="string.Empty"/></summary>
public string IllegalCharacterReplacements { get; set; }
/// <summary>Generate a valid path for this file or directory</summary>
public string GetFilePath(bool returnFirstExisting = false)
{
var filename = Template;
foreach (var r in ParameterReplacements)
filename = filename.Replace($"<{formatKey(r.Key)}>", formatValue(r.Value));
return FileUtility.GetValidFilename(filename, IllegalCharacterReplacements, returnFirstExisting);
}
private static string formatKey(string key)
=> key
.Replace("<", "")
.Replace(">", "");
private string formatValue(object value)
{
if (value is null)
return "";
// Other illegal characters will be taken care of later. Must take care of slashes now so params can't introduce new folders.
// Esp important for file templates.
var val = value
.ToString()
.Replace($"{System.IO.Path.DirectorySeparatorChar}", IllegalCharacterReplacements)
.Replace($"{System.IO.Path.AltDirectorySeparatorChar}", IllegalCharacterReplacements);
return
ParameterMaxSize.HasValue && ParameterMaxSize.Value > 0
? val.Truncate(ParameterMaxSize.Value)
: val;
}
}
}

View file

@ -0,0 +1,272 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Dinah.Core;
using Polly;
using Polly.Retry;
namespace FileManager
{
public static class FileUtility
{
/// <summary>
/// "txt" => ".txt"
/// <br />".txt" => ".txt"
/// <br />null or whitespace => ""
/// </summary>
public static string GetStandardizedExtension(string extension)
=> string.IsNullOrWhiteSpace(extension)
? (extension ?? "")?.Trim()
: '.' + extension.Trim().Trim('.');
/// <summary>
/// Return position with correct number of leading zeros.
/// <br />- 2 of 9 => "2"
/// <br />- 2 of 90 => "02"
/// <br />- 2 of 900 => "002"
/// </summary>
/// <param name="position">position in sequence. The 'x' in 'x of y'</param>
/// <param name="total">total qty in sequence. The 'y' in 'x of y'</param>
public static string GetSequenceFormatted(int position, int total)
{
ArgumentValidator.EnsureGreaterThan(position, nameof(position), 0);
ArgumentValidator.EnsureGreaterThan(total, nameof(total), 0);
if (position > total)
throw new ArgumentException($"{position} may not be greater than {total}");
return position.ToString().PadLeft(total.ToString().Length, '0');
}
private const int MAX_FILENAME_LENGTH = 255;
private const int MAX_DIRECTORY_LENGTH = 247;
/// <summary>
/// Ensure valid file name path:
/// <br/>- remove invalid chars
/// <br/>- ensure uniqueness
/// <br/>- enforce max file length
/// </summary>
public static string GetValidFilename(string path, string illegalCharacterReplacements = "", bool returnFirstExisting = false)
{
ArgumentValidator.EnsureNotNull(path, nameof(path));
// remove invalid chars
path = GetSafePath(path, illegalCharacterReplacements);
// ensure uniqueness and check lengths
var dir = Path.GetDirectoryName(path);
dir = dir.Truncate(MAX_DIRECTORY_LENGTH);
var filename = Path.GetFileNameWithoutExtension(path);
var fileStem = Path.Combine(dir, filename);
var extension = Path.GetExtension(path);
var fullfilename = fileStem.Truncate(MAX_FILENAME_LENGTH - extension.Length) + extension;
fullfilename = removeInvalidWhitespace(fullfilename);
var i = 0;
while (File.Exists(fullfilename) && !returnFirstExisting)
{
var increm = $" ({++i})";
fullfilename = fileStem.Truncate(MAX_FILENAME_LENGTH - increm.Length - extension.Length) + increm + extension;
}
return fullfilename;
}
// GetInvalidFileNameChars contains everything in GetInvalidPathChars plus ':', '*', '?', '\\', '/'
/// <summary>Use with file name, not full path. Valid path charaters which are invalid file name characters will be replaced: ':', '\\', '/'</summary>
public static string GetSafeFileName(string str, string illegalCharacterReplacements = "")
=> string.Join(illegalCharacterReplacements ?? "", str.Split(Path.GetInvalidFileNameChars()));
/// <summary>Use with full path, not file name. Valid path charaters which are invalid file name characters will be retained: '\\', '/'</summary>
public static string GetSafePath(string path, string illegalCharacterReplacements = "")
{
ArgumentValidator.EnsureNotNull(path, nameof(path));
path = replaceInvalidChars(path, illegalCharacterReplacements);
path = standardizeSlashes(path);
path = replaceColons(path, illegalCharacterReplacements);
path = removeDoubleSlashes(path);
return path;
}
private static char[] invalidChars { get; } = Path.GetInvalidPathChars().Union(new[] {
'*', '?',
// these are weird. If you run Path.GetInvalidPathChars() in Visual Studio's "C# Interactive", then these characters are included.
// In live code, Path.GetInvalidPathChars() does not include them
'"', '<', '>'
}).ToArray();
private static string replaceInvalidChars(string path, string illegalCharacterReplacements)
=> string.Join(illegalCharacterReplacements ?? "", path.Split(invalidChars));
private static string standardizeSlashes(string path)
=> path.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
private static string replaceColons(string path, string illegalCharacterReplacements)
{
// replace all colons except within the first 2 chars
var builder = new System.Text.StringBuilder();
for (var i = 0; i < path.Length; i++)
{
var c = path[i];
if (i >= 2 && c == ':')
builder.Append(illegalCharacterReplacements);
else
builder.Append(c);
}
return builder.ToString();
}
private static string removeDoubleSlashes(string path)
{
if (path.Length < 2)
return path;
// exception: don't try to condense the initial dbl bk slashes in a path. eg: \\192.168.0.1
var remainder = path[1..];
var dblSeparator = $"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}";
while (remainder.Contains(dblSeparator))
remainder = remainder.Replace(dblSeparator, $"{Path.DirectorySeparatorChar}");
return path[0] + remainder;
}
private static string removeInvalidWhitespace_pattern { get; } = $@"[\s\.]*\{Path.DirectorySeparatorChar}\s*";
private static Regex removeInvalidWhitespace_regex { get; } = new(removeInvalidWhitespace_pattern, RegexOptions.Compiled | RegexOptions.IgnorePatternWhitespace);
/// <summary>no part of the path may begin or end in whitespace</summary>
private static string removeInvalidWhitespace(string fullfilename)
{
// no whitespace at beginning or end
// replace whitespace around path slashes
// regex (with space added for clarity)
// \s* \\ \s* => \
// no ending dots. beginning dots are valid
// regex is easier by ending with separator
fullfilename += Path.DirectorySeparatorChar;
fullfilename = removeInvalidWhitespace_regex.Replace(fullfilename, Path.DirectorySeparatorChar.ToString());
// take seperator back off
fullfilename = RemoveLastCharacter(fullfilename);
fullfilename = removeDoubleSlashes(fullfilename);
return fullfilename;
}
public static string RemoveLastCharacter(this string str) => string.IsNullOrEmpty(str) ? str : str[..^1];
/// <summary>
/// Move file.
/// <br/>- Ensure valid file name path: remove invalid chars, ensure uniqueness, enforce max file length
/// <br/>- Perform <see cref="SaferMove"/>
/// <br/>- Return valid path
/// </summary>
public static string SaferMoveToValidPath(string source, string destination)
{
destination = GetValidFilename(destination);
SaferMove(source, destination);
return destination;
}
private static int maxRetryAttempts { get; } = 3;
private static TimeSpan pauseBetweenFailures { get; } = TimeSpan.FromMilliseconds(100);
private static RetryPolicy retryPolicy { get; } =
Policy
.Handle<Exception>()
.WaitAndRetry(maxRetryAttempts, i => pauseBetweenFailures);
/// <summary>Delete file. No error when source does not exist. Retry up to 3 times before throwing exception.</summary>
public static void SaferDelete(string source)
=> retryPolicy.Execute(() =>
{
try
{
if (!File.Exists(source))
{
Serilog.Log.Logger.Debug("No file to delete: {@DebugText}", new { source });
return;
}
Serilog.Log.Logger.Debug("Attempt to delete file: {@DebugText}", new { source });
File.Delete(source);
Serilog.Log.Logger.Information("File successfully deleted: {@DebugText}", new { source });
}
catch (Exception e)
{
Serilog.Log.Logger.Error(e, "Failed to delete file: {@DebugText}", new { source });
throw;
}
});
/// <summary>Move file. No error when source does not exist. Retry up to 3 times before throwing exception.</summary>
public static void SaferMove(string source, string destination)
=> retryPolicy.Execute(() =>
{
try
{
if (!File.Exists(source))
{
Serilog.Log.Logger.Debug("No file to move: {@DebugText}", new { source });
return;
}
SaferDelete(destination);
var dir = Path.GetDirectoryName(destination);
Serilog.Log.Logger.Debug("Attempt to create directory: {@DebugText}", new { dir });
Directory.CreateDirectory(dir);
Serilog.Log.Logger.Debug("Attempt to move file: {@DebugText}", new { source, destination });
File.Move(source, destination);
Serilog.Log.Logger.Information("File successfully moved: {@DebugText}", new { source, destination });
}
catch (Exception e)
{
Serilog.Log.Logger.Error(e, "Failed to move file: {@DebugText}", new { source, destination });
throw;
}
});
/// <summary>
/// A safer way to get all the files in a directory and sub directory without crashing on UnauthorizedException or PathTooLongException
/// </summary>
/// <param name="rootPath">Starting directory</param>
/// <param name="patternMatch">Filename pattern match</param>
/// <param name="searchOption">Search subdirectories or only top level directory for files</param>
/// <returns>List of files</returns>
public static IEnumerable<string> SaferEnumerateFiles(string path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly)
{
var foundFiles = Enumerable.Empty<string>();
if (searchOption == SearchOption.AllDirectories)
{
try
{
IEnumerable<string> subDirs = Directory.EnumerateDirectories(path);
// Add files in subdirectories recursively to the list
foreach (string dir in subDirs)
foundFiles = foundFiles.Concat(SaferEnumerateFiles(dir, searchPattern, searchOption));
}
catch (UnauthorizedAccessException) { }
catch (PathTooLongException) { }
}
try
{
// Add files from the current directory
foundFiles = foundFiles.Concat(Directory.EnumerateFiles(path, searchPattern));
}
catch (UnauthorizedAccessException) { }
return foundFiles;
}
}
}

View file

@ -0,0 +1,242 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace FileManager
{
public class PersistentDictionary
{
public string Filepath { get; }
public bool IsReadOnly { get; }
// optimize for strings. expectation is most settings will be strings and a rare exception will be something else
private Dictionary<string, string> stringCache { get; } = new Dictionary<string, string>();
private Dictionary<string, object> objectCache { get; } = new Dictionary<string, object>();
public PersistentDictionary(string filepath, bool isReadOnly = false)
{
Filepath = filepath;
IsReadOnly = isReadOnly;
if (File.Exists(Filepath))
return;
// will create any missing directories, incl subdirectories. if all already exist: no action
Directory.CreateDirectory(Path.GetDirectoryName(filepath));
if (IsReadOnly)
return;
createNewFile();
}
public string GetString(string propertyName)
{
if (!stringCache.ContainsKey(propertyName))
{
var jObject = readFile();
if (!jObject.ContainsKey(propertyName))
return null;
stringCache[propertyName] = jObject[propertyName].Value<string>();
}
return stringCache[propertyName];
}
public T GetNonString<T>(string propertyName)
{
var obj = GetObject(propertyName);
if (obj is null) return default;
if (obj is JValue jValue) return jValue.Value<T>();
if (obj is JObject jObject) return jObject.ToObject<T>();
return (T)obj;
}
public object GetObject(string propertyName)
{
if (!objectCache.ContainsKey(propertyName))
{
var jObject = readFile();
if (!jObject.ContainsKey(propertyName))
return null;
objectCache[propertyName] = jObject[propertyName].Value<object>();
}
return objectCache[propertyName];
}
public string GetStringFromJsonPath(string jsonPath, string propertyName) => GetStringFromJsonPath($"{jsonPath}.{propertyName}");
public string GetStringFromJsonPath(string jsonPath)
{
if (!stringCache.ContainsKey(jsonPath))
{
try
{
var jObject = readFile();
var token = jObject.SelectToken(jsonPath);
if (token is null)
return null;
stringCache[jsonPath] = (string)token;
}
catch
{
return null;
}
}
return stringCache[jsonPath];
}
public bool Exists(string propertyName) => readFile().ContainsKey(propertyName);
private object locker { get; } = new object();
public void SetString(string propertyName, string newValue)
{
// only do this check in string cache, NOT object cache
if (stringCache.ContainsKey(propertyName) && stringCache[propertyName] == newValue)
return;
// set cache
stringCache[propertyName] = newValue;
writeFile(propertyName, newValue);
}
public void SetNonString(string propertyName, object newValue)
{
// set cache
objectCache[propertyName] = newValue;
var parsedNewValue = JToken.Parse(JsonConvert.SerializeObject(newValue));
writeFile(propertyName, parsedNewValue);
}
private void writeFile(string propertyName, JToken newValue)
{
if (IsReadOnly)
return;
// write new setting to file
lock (locker)
{
var jObject = readFile();
var startContents = JsonConvert.SerializeObject(jObject, Formatting.Indented);
jObject[propertyName] = newValue;
var endContents = JsonConvert.SerializeObject(jObject, Formatting.Indented);
if (startContents == endContents)
return;
File.WriteAllText(Filepath, endContents);
}
try
{
var str = formatValueForLog(newValue?.ToString());
Serilog.Log.Logger.Information("Config changed. {@DebugInfo}", new { propertyName, newValue = str });
}
catch { }
}
/// <summary>WILL ONLY set if already present. WILL NOT create new</summary>
/// <returns>Value was changed</returns>
public bool SetWithJsonPath(string jsonPath, string propertyName, string newValue, bool suppressLogging = false)
{
if (IsReadOnly)
return false;
var path = $"{jsonPath}.{propertyName}";
{
// only do this check in string cache, NOT object cache
if (stringCache.ContainsKey(path) && stringCache[path] == newValue)
return false;
// set cache
stringCache[path] = newValue;
}
try
{
lock (locker)
{
var jObject = readFile();
var token = jObject.SelectToken(jsonPath);
if (token is null || token[propertyName] is null)
return false;
var oldValue = token.Value<string>(propertyName);
if (oldValue == newValue)
return false;
token[propertyName] = newValue;
File.WriteAllText(Filepath, JsonConvert.SerializeObject(jObject, Formatting.Indented));
}
}
catch (Exception exDebug)
{
return false;
}
if (!suppressLogging)
{
try
{
var str = formatValueForLog(newValue?.ToString());
Serilog.Log.Logger.Information("Config changed. {@DebugInfo}", new { jsonPath, propertyName, newValue = str });
}
catch { }
}
return true;
}
private static string formatValueForLog(string value)
=> value is null ? "[null]"
: string.IsNullOrEmpty(value) ? "[empty]"
: string.IsNullOrWhiteSpace(value) ? $"[whitespace. Length={value.Length}]"
: value.Length > 100 ? $"[Length={value.Length}] {value[0..50]}...{value[^50..^0]}"
: value;
private JObject readFile()
{
if (!File.Exists(Filepath))
{
var msg = "Unrecoverable error. Settings file cannot be found";
var ex = new FileNotFoundException(msg, Filepath);
Serilog.Log.Logger.Error(ex, msg);
throw ex;
}
var settingsJsonContents = File.ReadAllText(Filepath);
if (string.IsNullOrWhiteSpace(settingsJsonContents))
{
createNewFile();
settingsJsonContents = File.ReadAllText(Filepath);
}
var jObject = JsonConvert.DeserializeObject<JObject>(settingsJsonContents);
if (jObject is null)
{
var msg = "Unrecoverable error. Unable to read settings from Settings file";
var ex = new NullReferenceException(msg);
Serilog.Log.Logger.Error(ex, msg);
throw ex;
}
return jObject;
}
private void createNewFile()
{
File.WriteAllText(Filepath, "{}");
System.Threading.Thread.Sleep(100);
}
}
}

View file

@ -0,0 +1 @@
[assembly: System.Runtime.CompilerServices.InternalsVisibleTo(nameof(FileManager) + ".Tests")]