From 6f397d9b1204ffc579b2d232cc267f4530ce0e53 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Tue, 13 Feb 2024 00:56:27 +0100 Subject: [PATCH] rough impl of FSW, goodbye periodic filescan --- ...PeriodicFileScanner.cs => CacheMonitor.cs} | 381 ++++++++++++++---- MareSynchronos/FileCache/FileCacheManager.cs | 222 +++++----- MareSynchronos/Interop/IpcManager.cs | 14 +- MareSynchronos/MarePlugin.cs | 1 - MareSynchronos/Plugin.cs | 7 +- .../Services/CommandManagerService.cs | 8 +- MareSynchronos/Services/Mediator/Messages.cs | 1 + MareSynchronos/UI/IntroUI.cs | 10 +- MareSynchronos/UI/SettingsUi.cs | 62 ++- MareSynchronos/UI/UISharedService.cs | 70 ++-- MareSynchronos/Utils/Crypto.cs | 3 - 11 files changed, 525 insertions(+), 254 deletions(-) rename MareSynchronos/FileCache/{PeriodicFileScanner.cs => CacheMonitor.cs} (53%) diff --git a/MareSynchronos/FileCache/PeriodicFileScanner.cs b/MareSynchronos/FileCache/CacheMonitor.cs similarity index 53% rename from MareSynchronos/FileCache/PeriodicFileScanner.cs rename to MareSynchronos/FileCache/CacheMonitor.cs index 36aee0a..37c97cc 100644 --- a/MareSynchronos/FileCache/PeriodicFileScanner.cs +++ b/MareSynchronos/FileCache/CacheMonitor.cs @@ -2,12 +2,13 @@ using MareSynchronos.MareConfiguration; using MareSynchronos.Services; using MareSynchronos.Services.Mediator; +using MareSynchronos.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; namespace MareSynchronos.FileCache; -public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase +public sealed class CacheMonitor : DisposableMediatorSubscriberBase { private readonly MareConfigService _configService; private readonly DalamudUtilService _dalamudUtil; @@ -16,11 +17,10 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase private readonly IpcManager _ipcManager; private readonly PerformanceCollectorService _performanceCollector; private long _currentFileProgress = 0; - private bool _fileScanWasRunning = false; private CancellationTokenSource _scanCancellationTokenSource = new(); - private TimeSpan _timeUntilNextScan = TimeSpan.Zero; + private readonly string[] _allowedExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp", ".shpk"]; - public PeriodicFileScanner(ILogger logger, IpcManager ipcManager, MareConfigService configService, + public CacheMonitor(ILogger logger, IpcManager ipcManager, MareConfigService configService, FileCacheManager fileDbManager, MareMediator mediator, PerformanceCollectorService performanceCollector, DalamudUtilService dalamudUtil, FileCompactor fileCompactor) : base(logger, mediator) { @@ -30,38 +30,287 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase _performanceCollector = performanceCollector; _dalamudUtil = dalamudUtil; _fileCompactor = fileCompactor; - Mediator.Subscribe(this, (_) => StartScan()); + Mediator.Subscribe(this, (_) => + { + StartPenumbraWatcher(_ipcManager.PenumbraModDirectory); + StartMareWatcher(configService.Current.CacheFolder); + InvokeScan(); + }); Mediator.Subscribe(this, (msg) => HaltScan(msg.Source)); Mediator.Subscribe(this, (msg) => ResumeScan(msg.Source)); - Mediator.Subscribe(this, (_) => StartScan()); - Mediator.Subscribe(this, (_) => StartScan()); + Mediator.Subscribe(this, (_) => + { + StartMareWatcher(configService.Current.CacheFolder); + StartPenumbraWatcher(_ipcManager.PenumbraModDirectory); + InvokeScan(); + }); + Mediator.Subscribe(this, (msg) => StartPenumbraWatcher(msg.ModDirectory)); + if (_ipcManager.CheckPenumbraApi() && !string.IsNullOrEmpty(_ipcManager.PenumbraModDirectory)) + StartPenumbraWatcher(_ipcManager.PenumbraModDirectory); + if (configService.Current.HasValidSetup()) + { + StartMareWatcher(configService.Current.CacheFolder); + } } public long CurrentFileProgress => _currentFileProgress; public long FileCacheSize { get; set; } public ConcurrentDictionary HaltScanLocks { get; set; } = new(StringComparer.Ordinal); public bool IsScanRunning => CurrentFileProgress > 0 || TotalFiles > 0; - public string TimeUntilNextScan => _timeUntilNextScan.ToString(@"mm\:ss"); public long TotalFiles { get; private set; } public long TotalFilesStorage { get; private set; } - private int TimeBetweenScans => _configService.Current.TimeSpanBetweenScansInSeconds; public void HaltScan(string source) { if (!HaltScanLocks.ContainsKey(source)) HaltScanLocks[source] = 0; HaltScanLocks[source]++; + } - if (IsScanRunning && HaltScanLocks.Any(f => f.Value > 0)) + record WatcherChange(WatcherChangeTypes ChangeType, string? OldPath = null); + private readonly Dictionary _watcherChanges = new Dictionary(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _mareChanges = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public void StopMonitoring() + { + Logger.LogInformation("Stopping monitoring of Penumbra and Mare storage folders"); + MareWatcher?.Dispose(); + PenumbraWatcher?.Dispose(); + MareWatcher = null; + PenumbraWatcher = null; + } + + public void StartMareWatcher(string? marePath) + { + MareWatcher?.Dispose(); + if (string.IsNullOrEmpty(marePath)) { - _scanCancellationTokenSource?.Cancel(); - _fileScanWasRunning = true; + MareWatcher = null; + Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare."); + return; + } + + RecalculateFileCacheSize(); + + Logger.LogDebug("Initializing Mare FSW on {path}", marePath); + MareWatcher = new() + { + Path = marePath, + InternalBufferSize = 8388608, + NotifyFilter = NotifyFilters.CreationTime + | NotifyFilters.LastWrite + | NotifyFilters.FileName + | NotifyFilters.DirectoryName + | NotifyFilters.Size, + Filter = "*.*", + IncludeSubdirectories = false + }; + + MareWatcher.Deleted += MareWatcher_FileChanged; + MareWatcher.Created += MareWatcher_FileChanged; + } + + private void MareWatcher_FileChanged(object sender, FileSystemEventArgs e) + { + if (!_allowedExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return; + + lock (_watcherChanges) + { + _mareChanges[e.FullPath] = new(e.ChangeType); + } + + _ = MareWatcherExecution(); + } + + public void StartPenumbraWatcher(string? penumbraPath) + { + PenumbraWatcher?.Dispose(); + if (string.IsNullOrEmpty(penumbraPath)) + { + PenumbraWatcher = null; + Logger.LogWarning("Penumbra is not connected or the path is not set, cannot start FSW for Penumbra."); + return; + } + + Logger.LogDebug("Initializing Penumbra FSW on {path}", penumbraPath); + PenumbraWatcher = new() + { + Path = penumbraPath, + InternalBufferSize = 8388608, + NotifyFilter = NotifyFilters.CreationTime + | NotifyFilters.LastWrite + | NotifyFilters.FileName + | NotifyFilters.DirectoryName + | NotifyFilters.Size, + Filter = "*.*", + IncludeSubdirectories = true + }; + + PenumbraWatcher.Deleted += Fs_Changed; + PenumbraWatcher.Created += Fs_Changed; + PenumbraWatcher.Changed += Fs_Changed; + PenumbraWatcher.Renamed += Fs_Renamed; + PenumbraWatcher.EnableRaisingEvents = true; + } + + private void Fs_Changed(object sender, FileSystemEventArgs e) + { + if (Directory.Exists(e.FullPath)) return; + if (!_allowedExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return; + + if (e.ChangeType is not (WatcherChangeTypes.Changed or WatcherChangeTypes.Deleted or WatcherChangeTypes.Created)) + return; + + lock (_watcherChanges) + { + _watcherChanges[e.FullPath] = new(e.ChangeType); + } + + Logger.LogTrace("FSW {event}: {path}", e.ChangeType, e.FullPath); + + _ = PenumbraWatcherExecution(); + } + + private void Fs_Renamed(object sender, RenamedEventArgs e) + { + if (Directory.Exists(e.FullPath)) + { + var directoryFiles = Directory.GetFiles(e.FullPath, "*.*", SearchOption.AllDirectories); + lock (_watcherChanges) + { + foreach (var file in directoryFiles) + { + if (!_allowedExtensions.Any(ext => file.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) continue; + var oldPath = file.Replace(e.FullPath, e.OldFullPath, StringComparison.OrdinalIgnoreCase); + + _watcherChanges.Remove(oldPath); + _watcherChanges[file] = new(WatcherChangeTypes.Renamed, oldPath); + Logger.LogTrace("FSW Renamed: {path} -> {new}", oldPath, file); + + } + } + } + else + { + if (!_allowedExtensions.Any(ext => e.FullPath.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) return; + + lock (_watcherChanges) + { + _watcherChanges.Remove(e.OldFullPath); + _watcherChanges[e.FullPath] = new(WatcherChangeTypes.Renamed, e.OldFullPath); + } + + Logger.LogTrace("FSW Renamed: {path} -> {new}", e.OldFullPath, e.FullPath); + } + + _ = PenumbraWatcherExecution(); + } + + private CancellationTokenSource _penumbraFswCts = new(); + private CancellationTokenSource _mareFswCts = new(); + public FileSystemWatcher? PenumbraWatcher { get; private set; } + public FileSystemWatcher? MareWatcher { get; private set; } + + private async Task MareWatcherExecution() + { + _mareFswCts = _mareFswCts.CancelRecreate(); + var token = _mareFswCts.Token; + var delay = TimeSpan.FromSeconds(5); + Dictionary changes; + lock (_mareChanges) + changes = _mareChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal); + try + { + do + { + await Task.Delay(delay, token).ConfigureAwait(false); + } while (HaltScanLocks.Any(f => f.Value > 0)); + } + catch (TaskCanceledException) + { + return; + } + + lock (_mareChanges) + { + foreach (var key in changes.Keys) + { + _mareChanges.Remove(key); + } + } + + _ = RecalculateFileCacheSize(); + + if (changes.Any(c => c.Value.ChangeType == WatcherChangeTypes.Deleted)) + { + var threadCount = Math.Clamp((int)(Environment.ProcessorCount / 2.0f), 2, 8); + + Parallel.ForEach(changes, new ParallelOptions() + { + MaxDegreeOfParallelism = threadCount, + }, + (change) => + { + Logger.LogDebug("FSW Change: {change} = {val}", change.Key, change.Value); + _ = _fileDbManager.GetFileCacheByPath(change.Key); + }); + + _fileDbManager.WriteOutFullCsv(); } } - public void InvokeScan(bool forced = false) + private async Task PenumbraWatcherExecution() + { + _penumbraFswCts = _penumbraFswCts.CancelRecreate(); + var token = _penumbraFswCts.Token; + Dictionary changes; + lock (_watcherChanges) + changes = _watcherChanges.ToDictionary(t => t.Key, t => t.Value, StringComparer.Ordinal); + var delay = TimeSpan.FromSeconds(10); + try + { + do + { + await Task.Delay(delay, token).ConfigureAwait(false); + } while (HaltScanLocks.Any(f => f.Value > 0)); + } + catch (TaskCanceledException) + { + return; + } + + lock (_watcherChanges) + { + foreach (var key in changes.Keys) + { + _watcherChanges.Remove(key); + } + } + + var threadCount = Math.Clamp((int)(Environment.ProcessorCount / 2.0f), 2, 8); + + Parallel.ForEach(changes, new ParallelOptions() + { + MaxDegreeOfParallelism = threadCount, + }, + (change) => + { + Logger.LogDebug("FSW Change: {change} = {val}", change.Key, change.Value); + if (change.Value.ChangeType == WatcherChangeTypes.Deleted) + { + _fileDbManager.GetFileCacheByPath(change.Key); + } + else + { + if (change.Value.OldPath != null) _fileDbManager.GetFileCacheByPath(change.Value.OldPath); + _fileDbManager.CreateFileEntry(change.Key); + } + }); + + _fileDbManager.WriteOutFullCsv(); + } + + public void InvokeScan() { - bool isForced = forced; - bool isForcedFromExternal = forced; TotalFiles = 0; _currentFileProgress = 0; _scanCancellationTokenSource?.Cancel(); @@ -69,56 +318,36 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase var token = _scanCancellationTokenSource.Token; _ = Task.Run(async () => { - while (!token.IsCancellationRequested) + TotalFiles = 0; + _currentFileProgress = 0; + while (_dalamudUtil.IsOnFrameworkThread) { - while (HaltScanLocks.Any(f => f.Value > 0) || !_ipcManager.CheckPenumbraApi() || _dalamudUtil.IsOnFrameworkThread) - { - await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); - } - - isForced |= RecalculateFileCacheSize(); - if (!_configService.Current.FileScanPaused || isForced) - { - isForced = false; - TotalFiles = 0; - _currentFileProgress = 0; - while (_dalamudUtil.IsOnFrameworkThread) - { - Logger.LogWarning("Scanner is on framework, waiting for leaving thread before continuing"); - await Task.Delay(250, token).ConfigureAwait(false); - } - - Thread scanThread = new(() => - { - try - { - _performanceCollector.LogPerformance(this, "PeriodicFileScan", () => PeriodicFileScan(isForcedFromExternal, token)); - } - catch (Exception ex) - { - Logger.LogError(ex, "Error during Periodic File Scan"); - } - }) - { - Priority = ThreadPriority.Lowest, - IsBackground = true - }; - scanThread.Start(); - while (scanThread.IsAlive) - { - await Task.Delay(250).ConfigureAwait(false); - } - if (isForcedFromExternal) isForcedFromExternal = false; - TotalFiles = 0; - _currentFileProgress = 0; - } - _timeUntilNextScan = TimeSpan.FromSeconds(TimeBetweenScans); - while (_timeUntilNextScan.TotalSeconds >= 0 || _dalamudUtil.IsOnFrameworkThread) - { - await Task.Delay(TimeSpan.FromSeconds(1), token).ConfigureAwait(false); - _timeUntilNextScan -= TimeSpan.FromSeconds(1); - } + Logger.LogWarning("Scanner is on framework, waiting for leaving thread before continuing"); + await Task.Delay(250, token).ConfigureAwait(false); } + + Thread scanThread = new(() => + { + try + { + _performanceCollector.LogPerformance(this, "FullFileScan", () => FullFileScan(token)); + } + catch (Exception ex) + { + Logger.LogError(ex, "Error during Full File Scan"); + } + }) + { + Priority = ThreadPriority.Lowest, + IsBackground = true + }; + scanThread.Start(); + while (scanThread.IsAlive) + { + await Task.Delay(250).ConfigureAwait(false); + } + TotalFiles = 0; + _currentFileProgress = 0; }, token); } @@ -165,28 +394,19 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase HaltScanLocks[source]--; if (HaltScanLocks[source] < 0) HaltScanLocks[source] = 0; - - if (_fileScanWasRunning && HaltScanLocks.All(f => f.Value == 0)) - { - _fileScanWasRunning = false; - InvokeScan(forced: true); - } - } - - public void StartScan() - { - if (!_ipcManager.Initialized || !_configService.Current.HasValidSetup()) return; - Logger.LogTrace("Penumbra is active, configuration is valid, scan"); - InvokeScan(forced: true); } protected override void Dispose(bool disposing) { base.Dispose(disposing); _scanCancellationTokenSource?.Cancel(); + PenumbraWatcher?.Dispose(); + MareWatcher?.Dispose(); + _penumbraFswCts?.CancelDispose(); + _mareFswCts?.CancelDispose(); } - private void PeriodicFileScan(bool noWaiting, CancellationToken ct) + private void FullFileScan(CancellationToken ct) { TotalFiles = 1; var penumbraDir = _ipcManager.PenumbraModDirectory; @@ -210,7 +430,6 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase var previousThreadPriority = Thread.CurrentThread.Priority; Thread.CurrentThread.Priority = ThreadPriority.Lowest; Logger.LogDebug("Getting files from {penumbra} and {storage}", penumbraDir, _configService.Current.CacheFolder); - string[] ext = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp", ".shpk"]; Dictionary penumbraFiles = new(StringComparer.Ordinal); foreach (var folder in Directory.EnumerateDirectories(penumbraDir!)) @@ -221,7 +440,7 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase [ .. Directory.GetFiles(folder, "*.*", SearchOption.AllDirectories) .AsParallel() - .Where(f => ext.Any(e => f.EndsWith(e, StringComparison.OrdinalIgnoreCase)) + .Where(f => _allowedExtensions.Any(e => f.EndsWith(e, StringComparison.OrdinalIgnoreCase)) && !f.Contains(@"\bg\", StringComparison.OrdinalIgnoreCase) && !f.Contains(@"\bgcommon\", StringComparison.OrdinalIgnoreCase) && !f.Contains(@"\ui\", StringComparison.OrdinalIgnoreCase)), @@ -309,7 +528,6 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase Logger.LogWarning(ex, "Failed validating {path}", workload.ResolvedFilepath); } Interlocked.Increment(ref _currentFileProgress); - if (!noWaiting) Thread.Sleep(5); } Logger.LogTrace("Ending Worker Thread {i}", threadNr); @@ -390,7 +608,6 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase } Interlocked.Increment(ref _currentFileProgress); - if (!noWaiting) Thread.Sleep(5); }); Logger.LogTrace("Scanner added {notScanned} new files to db", allScannedFiles.Count(c => !c.Value)); @@ -406,6 +623,8 @@ public sealed class PeriodicFileScanner : DisposableMediatorSubscriberBase { _configService.Current.InitialScanComplete = true; _configService.Save(); + StartMareWatcher(_configService.Current.CacheFolder); + StartPenumbraWatcher(penumbraDir); } } } \ No newline at end of file diff --git a/MareSynchronos/FileCache/FileCacheManager.cs b/MareSynchronos/FileCache/FileCacheManager.cs index 9a2fef3..4c02eb4 100644 --- a/MareSynchronos/FileCache/FileCacheManager.cs +++ b/MareSynchronos/FileCache/FileCacheManager.cs @@ -3,6 +3,7 @@ using MareSynchronos.Interop; using MareSynchronos.MareConfiguration; using MareSynchronos.Services.Mediator; using MareSynchronos.Utils; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Globalization; @@ -10,7 +11,7 @@ using System.Text; namespace MareSynchronos.FileCache; -public sealed class FileCacheManager : IDisposable +public sealed class FileCacheManager : IHostedService { public const string CachePrefix = "{cache}"; public const string CsvSplit = "|"; @@ -30,101 +31,6 @@ public sealed class FileCacheManager : IDisposable _configService = configService; _mareMediator = mareMediator; _csvPath = Path.Combine(configService.ConfigurationDirectory, "FileCache.csv"); - - lock (_fileWriteLock) - { - try - { - if (File.Exists(CsvBakPath)) - { - File.Move(CsvBakPath, _csvPath, overwrite: true); - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to move BAK to ORG, deleting BAK"); - try - { - if (File.Exists(CsvBakPath)) - File.Delete(CsvBakPath); - } - catch (Exception ex1) - { - _logger.LogWarning(ex1, "Could not delete bak file"); - } - } - } - - if (File.Exists(_csvPath)) - { - bool success = false; - string[] entries = []; - int attempts = 0; - while (!success && attempts < 10) - { - try - { - entries = File.ReadAllLines(_csvPath); - success = true; - } - catch (Exception ex) - { - attempts++; - _logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath); - Thread.Sleep(100); - } - } - - if (!entries.Any()) - { - _logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath); - } - - Dictionary processedFiles = new(StringComparer.OrdinalIgnoreCase); - foreach (var entry in entries) - { - var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None); - try - { - var hash = splittedEntry[0]; - if (hash.Length != 40) throw new InvalidOperationException("Expected Hash length of 40, received " + hash.Length); - var path = splittedEntry[1]; - var time = splittedEntry[2]; - - if (processedFiles.ContainsKey(path)) - { - _logger.LogWarning("Already processed {file}, ignoring", path); - continue; - } - - processedFiles.Add(path, value: true); - - long size = -1; - long compressed = -1; - if (splittedEntry.Length > 3) - { - if (long.TryParse(splittedEntry[3], CultureInfo.InvariantCulture, out long result)) - { - size = result; - } - if (long.TryParse(splittedEntry[4], CultureInfo.InvariantCulture, out long resultCompressed)) - { - compressed = resultCompressed; - } - } - AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed))); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to initialize entry {entry}, ignoring", entry); - } - } - - if (processedFiles.Count != entries.Length) - { - WriteOutFullCsv(); - } - } } private string CsvBakPath => _csvPath + ".bak"; @@ -151,13 +57,6 @@ public sealed class FileCacheManager : IDisposable return CreateFileCacheEntity(fi, prefixedPath); } - public void Dispose() - { - _logger.LogTrace("Disposing {type}", GetType()); - WriteOutFullCsv(); - GC.SuppressFinalize(this); - } - public List GetAllFileCaches() => _fileCaches.Values.SelectMany(v => v).ToList(); public List GetAllFileCachesByHash(string hash) @@ -331,13 +230,14 @@ public sealed class FileCacheManager : IDisposable public void WriteOutFullCsv() { - StringBuilder sb = new(); - foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase)) - { - sb.AppendLine(entry.CsvEntry); - } lock (_fileWriteLock) { + StringBuilder sb = new(); + foreach (var entry in _fileCaches.SelectMany(k => k.Value).OrderBy(f => f.PrefixedFilePath, StringComparer.OrdinalIgnoreCase)) + { + sb.AppendLine(entry.CsvEntry); + } + if (File.Exists(_csvPath)) { File.Copy(_csvPath, CsvBakPath, overwrite: true); @@ -443,4 +343,110 @@ public sealed class FileCacheManager : IDisposable return fileCache; } + + public Task StartAsync(CancellationToken cancellationToken) + { + lock (_fileWriteLock) + { + try + { + if (File.Exists(CsvBakPath)) + { + File.Move(CsvBakPath, _csvPath, overwrite: true); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to move BAK to ORG, deleting BAK"); + try + { + if (File.Exists(CsvBakPath)) + File.Delete(CsvBakPath); + } + catch (Exception ex1) + { + _logger.LogWarning(ex1, "Could not delete bak file"); + } + } + } + + if (File.Exists(_csvPath)) + { + bool success = false; + string[] entries = []; + int attempts = 0; + while (!success && attempts < 10) + { + try + { + entries = File.ReadAllLines(_csvPath); + success = true; + } + catch (Exception ex) + { + attempts++; + _logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath); + Thread.Sleep(100); + } + } + + if (!entries.Any()) + { + _logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath); + } + + Dictionary processedFiles = new(StringComparer.OrdinalIgnoreCase); + foreach (var entry in entries) + { + var splittedEntry = entry.Split(CsvSplit, StringSplitOptions.None); + try + { + var hash = splittedEntry[0]; + if (hash.Length != 40) throw new InvalidOperationException("Expected Hash length of 40, received " + hash.Length); + var path = splittedEntry[1]; + var time = splittedEntry[2]; + + if (processedFiles.ContainsKey(path)) + { + _logger.LogWarning("Already processed {file}, ignoring", path); + continue; + } + + processedFiles.Add(path, value: true); + + long size = -1; + long compressed = -1; + if (splittedEntry.Length > 3) + { + if (long.TryParse(splittedEntry[3], CultureInfo.InvariantCulture, out long result)) + { + size = result; + } + if (long.TryParse(splittedEntry[4], CultureInfo.InvariantCulture, out long resultCompressed)) + { + compressed = resultCompressed; + } + } + AddHashedFile(ReplacePathPrefixes(new FileCacheEntity(hash, path, time, size, compressed))); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to initialize entry {entry}, ignoring", entry); + } + } + + if (processedFiles.Count != entries.Length) + { + WriteOutFullCsv(); + } + } + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + WriteOutFullCsv(); + return Task.CompletedTask; + } } \ No newline at end of file diff --git a/MareSynchronos/Interop/IpcManager.cs b/MareSynchronos/Interop/IpcManager.cs index bf1c6cd..8853304 100644 --- a/MareSynchronos/Interop/IpcManager.cs +++ b/MareSynchronos/Interop/IpcManager.cs @@ -156,7 +156,19 @@ public sealed class IpcManager : DisposableMediatorSubscriberBase } public bool Initialized => CheckPenumbraApiInternal() && CheckGlamourerApiInternal(); - public string? PenumbraModDirectory { get; private set; } + private string? _penumbraModDirectory; + public string? PenumbraModDirectory + { + get => _penumbraModDirectory; + private set + { + if (!string.Equals(_penumbraModDirectory, value, StringComparison.Ordinal)) + { + _penumbraModDirectory = value; + Mediator.Publish(new PenumbraDirectoryChangedMessage(_penumbraModDirectory)); + } + } + } public bool CheckCustomizePlusApi() => _customizePlusAvailable; diff --git a/MareSynchronos/MarePlugin.cs b/MareSynchronos/MarePlugin.cs index fe464d5..7ebf3a8 100644 --- a/MareSynchronos/MarePlugin.cs +++ b/MareSynchronos/MarePlugin.cs @@ -144,7 +144,6 @@ public class MarePlugin : MediatorSubscriberBase, IHostedService Mediator.Publish(new SwitchToIntroUiMessage()); return; } - _runtimeServiceScope.ServiceProvider.GetRequiredService().StartScan(); _runtimeServiceScope.ServiceProvider.GetRequiredService(); _runtimeServiceScope.ServiceProvider.GetRequiredService(); _runtimeServiceScope.ServiceProvider.GetRequiredService(); diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index dd09772..a5b8527 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -100,7 +100,7 @@ public sealed class Plugin : IDalamudPlugin collection.AddSingleton(); // add scoped services - collection.AddScoped(); + collection.AddScoped(); collection.AddScoped(); collection.AddScoped(); collection.AddScoped(); @@ -129,12 +129,12 @@ public sealed class Plugin : IDalamudPlugin s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService())); collection.AddScoped((s) => new CommandManagerService(commandManager, s.GetRequiredService(), - s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), + s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService())); collection.AddScoped((s) => new NotificationService(s.GetRequiredService>(), s.GetRequiredService(), pluginInterface.UiBuilder, chatGui, s.GetRequiredService())); collection.AddScoped((s) => new UiSharedService(s.GetRequiredService>(), s.GetRequiredService(), s.GetRequiredService(), - s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), + s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), pluginInterface, s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService())); collection.AddHostedService(p => p.GetRequiredService()); @@ -145,6 +145,7 @@ public sealed class Plugin : IDalamudPlugin collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); + collection.AddHostedService(p => p.GetRequiredService()); }) .Build() .RunAsync(_pluginCts.Token); diff --git a/MareSynchronos/Services/CommandManagerService.cs b/MareSynchronos/Services/CommandManagerService.cs index 6e4034f..0c6ae18 100644 --- a/MareSynchronos/Services/CommandManagerService.cs +++ b/MareSynchronos/Services/CommandManagerService.cs @@ -19,17 +19,17 @@ public sealed class CommandManagerService : IDisposable private readonly MareMediator _mediator; private readonly MareConfigService _mareConfigService; private readonly PerformanceCollectorService _performanceCollectorService; - private readonly PeriodicFileScanner _periodicFileScanner; + private readonly CacheMonitor _cacheMonitor; private readonly ServerConfigurationManager _serverConfigurationManager; public CommandManagerService(ICommandManager commandManager, PerformanceCollectorService performanceCollectorService, - ServerConfigurationManager serverConfigurationManager, PeriodicFileScanner periodicFileScanner, + ServerConfigurationManager serverConfigurationManager, CacheMonitor periodicFileScanner, ApiController apiController, MareMediator mediator, MareConfigService mareConfigService) { _commandManager = commandManager; _performanceCollectorService = performanceCollectorService; _serverConfigurationManager = serverConfigurationManager; - _periodicFileScanner = periodicFileScanner; + _cacheMonitor = periodicFileScanner; _apiController = apiController; _mediator = mediator; _mareConfigService = mareConfigService; @@ -87,7 +87,7 @@ public sealed class CommandManagerService : IDisposable } else if (string.Equals(splitArgs[0], "rescan", StringComparison.OrdinalIgnoreCase)) { - _periodicFileScanner.InvokeScan(forced: true); + _cacheMonitor.InvokeScan(); } else if (string.Equals(splitArgs[0], "perf", StringComparison.OrdinalIgnoreCase)) { diff --git a/MareSynchronos/Services/Mediator/Messages.cs b/MareSynchronos/Services/Mediator/Messages.cs index e22bffd..7643747 100644 --- a/MareSynchronos/Services/Mediator/Messages.cs +++ b/MareSynchronos/Services/Mediator/Messages.cs @@ -82,6 +82,7 @@ public record TargetPairMessage(Pair Pair) : MessageBase; public record CombatOrPerformanceStartMessage : MessageBase; public record CombatOrPerformanceEndMessage : MessageBase; public record EventMessage(Event Event) : MessageBase; +public record PenumbraDirectoryChangedMessage(string? ModDirectory) : MessageBase; #pragma warning restore S2094 #pragma warning restore MA0048 // File name must match type name \ No newline at end of file diff --git a/MareSynchronos/UI/IntroUI.cs b/MareSynchronos/UI/IntroUI.cs index 3fceced..fe45c07 100644 --- a/MareSynchronos/UI/IntroUI.cs +++ b/MareSynchronos/UI/IntroUI.cs @@ -16,7 +16,7 @@ namespace MareSynchronos.UI; public class IntroUi : WindowMediatorSubscriberBase { private readonly MareConfigService _configService; - private readonly PeriodicFileScanner _fileCacheManager; + private readonly CacheMonitor _cacheMonitor; private readonly Dictionary _languages = new(StringComparer.Ordinal) { { "English", "en" }, { "Deutsch", "de" }, { "Français", "fr" } }; private readonly ServerConfigurationManager _serverConfigurationManager; private readonly UiSharedService _uiShared; @@ -29,11 +29,11 @@ public class IntroUi : WindowMediatorSubscriberBase private string[]? _tosParagraphs; public IntroUi(ILogger logger, UiSharedService uiShared, MareConfigService configService, - PeriodicFileScanner fileCacheManager, ServerConfigurationManager serverConfigurationManager, MareMediator mareMediator) : base(logger, mareMediator, "Mare Synchronos Setup") + CacheMonitor fileCacheManager, ServerConfigurationManager serverConfigurationManager, MareMediator mareMediator) : base(logger, mareMediator, "Mare Synchronos Setup") { _uiShared = uiShared; _configService = configService; - _fileCacheManager = fileCacheManager; + _cacheMonitor = fileCacheManager; _serverConfigurationManager = serverConfigurationManager; IsOpen = false; @@ -163,11 +163,11 @@ public class IntroUi : WindowMediatorSubscriberBase _uiShared.DrawCacheDirectorySetting(); } - if (!_fileCacheManager.IsScanRunning && !string.IsNullOrEmpty(_configService.Current.CacheFolder) && _uiShared.HasValidPenumbraModPath && Directory.Exists(_configService.Current.CacheFolder)) + if (!_cacheMonitor.IsScanRunning && !string.IsNullOrEmpty(_configService.Current.CacheFolder) && _uiShared.HasValidPenumbraModPath && Directory.Exists(_configService.Current.CacheFolder)) { if (ImGui.Button("Start Scan##startScan")) { - _fileCacheManager.InvokeScan(forced: true); + _cacheMonitor.InvokeScan(); } } else diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index 37d18de..e35ba1c 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -7,6 +7,7 @@ using ImGuiNET; using MareSynchronos.API.Data; using MareSynchronos.API.Data.Comparer; using MareSynchronos.FileCache; +using MareSynchronos.Interop; using MareSynchronos.MareConfiguration; using MareSynchronos.MareConfiguration.Models; using MareSynchronos.PlayerData.Export; @@ -30,6 +31,8 @@ namespace MareSynchronos.UI; public class SettingsUi : WindowMediatorSubscriberBase { private readonly ApiController _apiController; + private readonly IpcManager _ipcManager; + private readonly CacheMonitor _cacheMonitor; private readonly MareConfigService _configService; private readonly ConcurrentDictionary> _currentDownloads = new(); private readonly FileCompactor _fileCompactor; @@ -64,7 +67,8 @@ public class SettingsUi : WindowMediatorSubscriberBase FileUploadManager fileTransferManager, FileTransferOrchestrator fileTransferOrchestrator, FileCacheManager fileCacheManager, - FileCompactor fileCompactor, ApiController apiController) : base(logger, mediator, "Mare Synchronos Settings") + FileCompactor fileCompactor, ApiController apiController, + IpcManager ipcManager, CacheMonitor cacheMonitor) : base(logger, mediator, "Mare Synchronos Settings") { _configService = configService; _mareCharaFileManager = mareCharaFileManager; @@ -75,6 +79,8 @@ public class SettingsUi : WindowMediatorSubscriberBase _fileTransferOrchestrator = fileTransferOrchestrator; _fileCacheManager = fileCacheManager; _apiController = apiController; + _ipcManager = ipcManager; + _cacheMonitor = cacheMonitor; _fileCompactor = fileCompactor; _uiShared = uiShared; AllowClickthrough = false; @@ -479,7 +485,57 @@ public class SettingsUi : WindowMediatorSubscriberBase "The storage governs itself by clearing data beyond the set storage size. Please set the storage size accordingly. It is not necessary to manually clear the storage."); _uiShared.DrawFileScanState(); - _uiShared.DrawTimeSpanBetweenScansSetting(); + ImGui.AlignTextToFramePadding(); + ImGui.TextUnformatted("Monitoring Penumbra Folder: " + (_cacheMonitor.PenumbraWatcher?.Path ?? "Not monitoring")); + if (string.IsNullOrEmpty(_cacheMonitor.PenumbraWatcher?.Path)) + { + ImGui.SameLine(); + using var id = ImRaii.PushId("penumbraMonitor"); + if (UiSharedService.NormalizedIconTextButton(FontAwesomeIcon.ArrowsToCircle, "Try to reinitialize Monitor")) + { + _cacheMonitor.StartPenumbraWatcher(_ipcManager.PenumbraModDirectory); + } + } + + ImGui.AlignTextToFramePadding(); + ImGui.TextUnformatted("Monitoring Mare Storage Folder: " + (_cacheMonitor.MareWatcher?.Path ?? "Not monitoring")); + if (string.IsNullOrEmpty(_cacheMonitor.MareWatcher?.Path)) + { + ImGui.SameLine(); + using var id = ImRaii.PushId("mareMonitor"); + if (UiSharedService.NormalizedIconTextButton(FontAwesomeIcon.ArrowsToCircle, "Try to reinitialize Monitor")) + { + _cacheMonitor.StartMareWatcher(_configService.Current.CacheFolder); + } + } + if (_cacheMonitor.MareWatcher == null || _cacheMonitor.PenumbraWatcher == null) + { + if (UiSharedService.NormalizedIconTextButton(FontAwesomeIcon.Play, "Resume Monitoring")) + { + _cacheMonitor.StartMareWatcher(_configService.Current.CacheFolder); + _cacheMonitor.StartPenumbraWatcher(_ipcManager.PenumbraModDirectory); + _cacheMonitor.InvokeScan(); + } + UiSharedService.AttachToolTip("Attempts to resume monitoring for both Penumbra and Mare Storage. " + + "Resuming the monitoring will also force a full scan to run." + Environment.NewLine + + "If the button remains present after clicking it, consult /xllog for errors"); + } + else + { + using (ImRaii.Disabled(!UiSharedService.CtrlPressed())) + { + if (UiSharedService.NormalizedIconTextButton(FontAwesomeIcon.Stop, "Stop Monitoring")) + { + _cacheMonitor.StopMonitoring(); + } + } + UiSharedService.AttachToolTip("Stops the monitoring for both Penumbra and Mare Storage. " + + "Do not stop the monitoring, unless you plan to move the Penumbra and Mare Storage folders, to ensure correct functionality of Mare." + Environment.NewLine + + "If you stop the monitoring to move folders around, resume it after you are finished moving the files." + + UiSharedService.TooltipSeparator + "Hold CTRL to enable this button"); + } + + _uiShared.DrawCacheDirectorySetting(); ImGui.TextUnformatted($"Currently utilized local storage: {UiSharedService.ByteToString(_uiShared.FileCacheSize)}"); bool isLinux = Util.IsWine(); @@ -579,8 +635,6 @@ public class SettingsUi : WindowMediatorSubscriberBase { File.Delete(file); } - - _uiShared.RecalculateFileCacheSize(); }); } UiSharedService.AttachToolTip("You normally do not need to do this. THIS IS NOT SOMETHING YOU SHOULD BE DOING TO TRY TO FIX SYNC ISSUES." + Environment.NewLine diff --git a/MareSynchronos/UI/UISharedService.cs b/MareSynchronos/UI/UISharedService.cs index f186f00..d64e940 100644 --- a/MareSynchronos/UI/UISharedService.cs +++ b/MareSynchronos/UI/UISharedService.cs @@ -40,7 +40,7 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase private readonly ApiController _apiController; - private readonly PeriodicFileScanner _cacheScanner; + private readonly CacheMonitor _cacheMonitor; private readonly MareConfigService _configService; @@ -74,13 +74,13 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase private int _serverSelectionIndex = -1; public UiSharedService(ILogger logger, IpcManager ipcManager, ApiController apiController, - PeriodicFileScanner cacheScanner, FileDialogManager fileDialogManager, + CacheMonitor cacheMonitor, FileDialogManager fileDialogManager, MareConfigService configService, DalamudUtilService dalamudUtil, DalamudPluginInterface pluginInterface, Dalamud.Localization localization, ServerConfigurationManager serverManager, MareMediator mediator) : base(logger, mediator) { _ipcManager = ipcManager; _apiController = apiController; - _cacheScanner = cacheScanner; + _cacheMonitor = cacheMonitor; FileDialogManager = fileDialogManager; _configService = configService; _dalamudUtil = dalamudUtil; @@ -109,7 +109,7 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase public bool EditTrackerPosition { get; set; } - public long FileCacheSize => _cacheScanner.FileCacheSize; + public long FileCacheSize => _cacheMonitor.FileCacheSize; public bool HasValidPenumbraModPath => !(_ipcManager.PenumbraModDirectory ?? string.Empty).IsNullOrEmpty() && Directory.Exists(_ipcManager.PenumbraModDirectory); @@ -583,7 +583,8 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase { _configService.Current.CacheFolder = path; _configService.Save(); - _cacheScanner.StartScan(); + _cacheMonitor.StartMareWatcher(path); + _cacheMonitor.InvokeScan(); } }); } @@ -657,41 +658,45 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase public void DrawFileScanState() { + ImGui.AlignTextToFramePadding(); ImGui.TextUnformatted("File Scanner Status"); ImGui.SameLine(); - if (_cacheScanner.IsScanRunning) + if (_cacheMonitor.IsScanRunning) { + ImGui.AlignTextToFramePadding(); + ImGui.TextUnformatted("Scan is running"); ImGui.TextUnformatted("Current Progress:"); ImGui.SameLine(); - ImGui.TextUnformatted(_cacheScanner.TotalFiles == 1 + ImGui.TextUnformatted(_cacheMonitor.TotalFiles == 1 ? "Collecting files" - : $"Processing {_cacheScanner.CurrentFileProgress}/{_cacheScanner.TotalFilesStorage} from storage ({_cacheScanner.TotalFiles} scanned in)"); + : $"Processing {_cacheMonitor.CurrentFileProgress}/{_cacheMonitor.TotalFilesStorage} from storage ({_cacheMonitor.TotalFiles} scanned in)"); AttachToolTip("Note: it is possible to have more files in storage than scanned in, " + "this is due to the scanner normally ignoring those files but the game loading them in and using them on your character, so they get " + "added to the local storage."); } - else if (_configService.Current.FileScanPaused) + else if (_cacheMonitor.HaltScanLocks.Any(f => f.Value > 0)) { - ImGui.TextUnformatted("File scanner is paused"); - ImGui.SameLine(); - if (ImGui.Button("Force Rescan##forcedrescan")) - { - _cacheScanner.InvokeScan(forced: true); - } - } - else if (_cacheScanner.HaltScanLocks.Any(f => f.Value > 0)) - { - ImGui.TextUnformatted("Halted (" + string.Join(", ", _cacheScanner.HaltScanLocks.Where(f => f.Value > 0).Select(locker => locker.Key + ": " + locker.Value + " halt requests")) + ")"); + ImGui.AlignTextToFramePadding(); + + ImGui.TextUnformatted("Halted (" + string.Join(", ", _cacheMonitor.HaltScanLocks.Where(f => f.Value > 0).Select(locker => locker.Key + ": " + locker.Value + " halt requests")) + ")"); ImGui.SameLine(); if (ImGui.Button("Reset halt requests##clearlocks")) { - _cacheScanner.ResetLocks(); + _cacheMonitor.ResetLocks(); } } else { - ImGui.TextUnformatted("Next scan in " + _cacheScanner.TimeUntilNextScan); + ImGui.TextUnformatted("Idle"); + if (_configService.Current.InitialScanComplete) + { + ImGui.SameLine(); + if (NormalizedIconTextButton(FontAwesomeIcon.Play, "Force rescan")) + { + _cacheMonitor.InvokeScan(); + } + } } } @@ -833,35 +838,12 @@ public partial class UiSharedService : DisposableMediatorSubscriberBase return _serverSelectionIndex; } - public void DrawTimeSpanBetweenScansSetting() - { - var timeSpan = _configService.Current.TimeSpanBetweenScansInSeconds; - if (ImGui.SliderInt("Seconds between scans##timespan", ref timeSpan, 20, 60)) - { - _configService.Current.TimeSpanBetweenScansInSeconds = timeSpan; - _configService.Save(); - } - DrawHelpText("This is the time in seconds between file scans. Increase it to reduce system load. A too high setting can cause issues when manually fumbling about in the cache or Penumbra mods folders."); - var isPaused = _configService.Current.FileScanPaused; - if (ImGui.Checkbox("Pause periodic file scan##filescanpause", ref isPaused)) - { - _configService.Current.FileScanPaused = isPaused; - _configService.Save(); - } - DrawHelpText("This allows you to stop the periodic scans of your Penumbra and Mare cache directories. Use this to move the Mare cache and Penumbra mod folders around. If you enable this permanently, run a Force rescan after adding mods to Penumbra."); - } - public void LoadLocalization(string languageCode) { _localization.SetupWithLangCode(languageCode); Strings.ToS = new Strings.ToSStrings(); } - public void RecalculateFileCacheSize() - { - _cacheScanner.InvokeScan(forced: true); - } - [LibraryImport("user32")] internal static partial short GetKeyState(int nVirtKey); diff --git a/MareSynchronos/Utils/Crypto.cs b/MareSynchronos/Utils/Crypto.cs index 2aa7bd4..62883ca 100644 --- a/MareSynchronos/Utils/Crypto.cs +++ b/MareSynchronos/Utils/Crypto.cs @@ -47,8 +47,5 @@ public static class Crypto return _hashListSHA1[stringToCompute] = BitConverter.ToString(_sha1CryptoProvider.ComputeHash(Encoding.UTF8.GetBytes(stringToCompute))).Replace("-", "", StringComparison.Ordinal); } - - - #pragma warning restore SYSLIB0021 // Type or member is obsolete } \ No newline at end of file