From 549a93654a6bf443a984b77a118f86b8537bb318 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Thu, 15 Feb 2024 02:38:41 +0100 Subject: [PATCH] make storage size calculation asynchronous and running in parallel --- MareSynchronos/FileCache/CacheMonitor.cs | 79 ++++++++++++++----- MareSynchronos/FileCache/FileCacheManager.cs | 12 +++ MareSynchronos/FileCache/FileCompactor.cs | 14 +++- MareSynchronos/Interop/IpcProvider.cs | 3 +- MareSynchronos/Plugin.cs | 6 +- MareSynchronos/Services/DalamudUtilService.cs | 2 + .../Services/Events/EventAggregator.cs | 2 + .../Services/Mediator/MareMediator.cs | 4 +- .../Services/PerformanceCollectorService.cs | 2 + MareSynchronos/UI/DtrEntry.cs | 6 +- MareSynchronos/UI/SettingsUi.cs | 17 ++-- 11 files changed, 110 insertions(+), 37 deletions(-) diff --git a/MareSynchronos/FileCache/CacheMonitor.cs b/MareSynchronos/FileCache/CacheMonitor.cs index a73c3c5..878e836 100644 --- a/MareSynchronos/FileCache/CacheMonitor.cs +++ b/MareSynchronos/FileCache/CacheMonitor.cs @@ -18,6 +18,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase private readonly PerformanceCollectorService _performanceCollector; private long _currentFileProgress = 0; private CancellationTokenSource _scanCancellationTokenSource = new(); + private readonly CancellationTokenSource _periodicCalculationTokenSource = new(); private readonly string[] _allowedExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp", ".shpk"]; public CacheMonitor(ILogger logger, IpcManager ipcManager, MareConfigService configService, @@ -57,6 +58,25 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase { StartMareWatcher(configService.Current.CacheFolder); } + + var token = _periodicCalculationTokenSource.Token; + _ = Task.Run(async () => + { + Logger.LogInformation("Starting Periodic Storage Directory Calculation Task"); + var token = _periodicCalculationTokenSource.Token; + while (!token.IsCancellationRequested) + { + try + { + RecalculateFileCacheSize(token); + } + catch + { + // ignore + } + await Task.Delay(TimeSpan.FromMinutes(1), token).ConfigureAwait(false); + } + }, token); } public long CurrentFileProgress => _currentFileProgress; @@ -86,17 +106,21 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase PenumbraWatcher = null; } + public bool StorageisNTFS { get; private set; } = false; + public void StartMareWatcher(string? marePath) { MareWatcher?.Dispose(); - if (string.IsNullOrEmpty(marePath)) + if (string.IsNullOrEmpty(marePath) || !Directory.Exists(marePath)) { MareWatcher = null; Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare."); return; } - RecalculateFileCacheSize(); + DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); + StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase); + Logger.LogInformation("Mare Storage is on NTFS drive: {isNtfs}", StorageisNTFS); Logger.LogDebug("Initializing Mare FSW on {path}", marePath); MareWatcher = new() @@ -248,9 +272,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase } } - _ = RecalculateFileCacheSize(); - - if (changes.Any(c => c.Value.ChangeType == WatcherChangeTypes.Deleted)) { lock (_fileDbManager) @@ -356,26 +377,43 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase }, token); } - public bool RecalculateFileCacheSize() + public void RecalculateFileCacheSize(CancellationToken token) { - FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder).Sum(f => + if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder)) { - try - { - return _fileCompactor.GetFileSizeOnDisk(f); - } - catch - { - return 0; - } - }); + FileCacheSize = 0; + return; + } - DriveInfo di = new DriveInfo(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); - FileCacheDriveFree = di.AvailableFreeSpace; + FileCacheSize = -1; + DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); + try + { + FileCacheDriveFree = di.AvailableFreeSpace; + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder); + } + + FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder) + .AsParallel().Sum(f => + { + token.ThrowIfCancellationRequested(); + + try + { + return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS); + } + catch + { + return 0; + } + }); var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d); - if (FileCacheSize < maxCacheInBytes) return false; + if (FileCacheSize < maxCacheInBytes) return; var allFiles = Directory.EnumerateFiles(_configService.Current.CacheFolder) .Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList(); @@ -387,8 +425,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase File.Delete(oldestFile.FullName); allFiles.Remove(oldestFile); } - - return true; } public void ResetLocks() @@ -412,6 +448,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase MareWatcher?.Dispose(); _penumbraFswCts?.CancelDispose(); _mareFswCts?.CancelDispose(); + _periodicCalculationTokenSource?.CancelDispose(); } private void FullFileScan(CancellationToken ct) diff --git a/MareSynchronos/FileCache/FileCacheManager.cs b/MareSynchronos/FileCache/FileCacheManager.cs index 7ec8d29..eddce49 100644 --- a/MareSynchronos/FileCache/FileCacheManager.cs +++ b/MareSynchronos/FileCache/FileCacheManager.cs @@ -352,12 +352,17 @@ public sealed class FileCacheManager : IHostedService public Task StartAsync(CancellationToken cancellationToken) { + _logger.LogInformation("Starting FileCacheManager"); lock (_fileWriteLock) { try { + _logger.LogInformation("Checking for {bakPath}", CsvBakPath); + if (File.Exists(CsvBakPath)) { + _logger.LogInformation("{bakPath} found, moving to {csvPath}", CsvBakPath, _csvPath); + File.Move(CsvBakPath, _csvPath, overwrite: true); } } @@ -378,6 +383,8 @@ public sealed class FileCacheManager : IHostedService if (File.Exists(_csvPath)) { + _logger.LogInformation("{csvPath} found, parsing", _csvPath); + bool success = false; string[] entries = []; int attempts = 0; @@ -385,6 +392,7 @@ public sealed class FileCacheManager : IHostedService { try { + _logger.LogInformation("Attempting to read {csvPath}", _csvPath); entries = File.ReadAllLines(_csvPath); success = true; } @@ -401,6 +409,8 @@ public sealed class FileCacheManager : IHostedService _logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath); } + _logger.LogInformation("Found {amount} files in {path}", entries.Length, _csvPath); + Dictionary processedFiles = new(StringComparer.OrdinalIgnoreCase); foreach (var entry in entries) { @@ -447,6 +457,8 @@ public sealed class FileCacheManager : IHostedService } } + _logger.LogInformation("Started FileCacheManager"); + return Task.CompletedTask; } diff --git a/MareSynchronos/FileCache/FileCompactor.cs b/MareSynchronos/FileCache/FileCompactor.cs index 0c0efdd..a5f3bea 100644 --- a/MareSynchronos/FileCache/FileCompactor.cs +++ b/MareSynchronos/FileCache/FileCompactor.cs @@ -62,9 +62,11 @@ public sealed class FileCompactor MassCompactRunning = false; } - public long GetFileSizeOnDisk(string filePath) + public long GetFileSizeOnDisk(string filePath, bool? isNTFS = null) { - if (Dalamud.Utility.Util.IsWine()) return new FileInfo(filePath).Length; + bool ntfs = isNTFS ?? string.Equals(new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); + + if (Dalamud.Utility.Util.IsWine() || !ntfs) return new FileInfo(filePath).Length; var clusterSize = GetClusterSize(filePath); if (clusterSize == -1) return new FileInfo(filePath).Length; @@ -105,6 +107,14 @@ public sealed class FileCompactor private void CompactFile(string filePath) { + var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName); + bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); + if (!isNTFS) + { + _logger.LogWarning("Drive for file {file} is not NTFS", filePath); + return; + } + var oldSize = new FileInfo(filePath).Length; var clusterSize = GetClusterSize(filePath); diff --git a/MareSynchronos/Interop/IpcProvider.cs b/MareSynchronos/Interop/IpcProvider.cs index ba1c821..2b76731 100644 --- a/MareSynchronos/Interop/IpcProvider.cs +++ b/MareSynchronos/Interop/IpcProvider.cs @@ -28,11 +28,12 @@ public class IpcProvider : IHostedService public Task StartAsync(CancellationToken cancellationToken) { - _logger.LogDebug("Starting IpcProvider Service"); + _logger.LogInformation("Starting IpcProviderService"); _loadFileProvider = _pi.GetIpcProvider("MareSynchronos.LoadMcdf"); _loadFileProvider.RegisterFunc(LoadMcdf); _loadFileAsyncProvider = _pi.GetIpcProvider>("MareSynchronos.LoadMcdfAsync"); _loadFileAsyncProvider.RegisterFunc(LoadMcdfAsync); + _logger.LogInformation("Started IpcProviderService"); return Task.CompletedTask; } diff --git a/MareSynchronos/Plugin.cs b/MareSynchronos/Plugin.cs index a5b8527..59e6174 100644 --- a/MareSynchronos/Plugin.cs +++ b/MareSynchronos/Plugin.cs @@ -137,15 +137,15 @@ public sealed class Plugin : IDalamudPlugin s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService(), pluginInterface, s.GetRequiredService(), s.GetRequiredService(), s.GetRequiredService())); - collection.AddHostedService(p => p.GetRequiredService()); + collection.AddHostedService(p => p.GetRequiredService()); + collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); - collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); collection.AddHostedService(p => p.GetRequiredService()); - collection.AddHostedService(p => p.GetRequiredService()); + collection.AddHostedService(p => p.GetRequiredService()); }) .Build() .RunAsync(_pluginCts.Token); diff --git a/MareSynchronos/Services/DalamudUtilService.cs b/MareSynchronos/Services/DalamudUtilService.cs index cff6e9f..8a14334 100644 --- a/MareSynchronos/Services/DalamudUtilService.cs +++ b/MareSynchronos/Services/DalamudUtilService.cs @@ -284,12 +284,14 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber public Task StartAsync(CancellationToken cancellationToken) { + _logger.LogInformation("Starting DalamudUtilService"); _framework.Update += FrameworkOnUpdate; if (IsLoggedIn) { _classJobId = _clientState.LocalPlayer!.ClassJob.Id; } + _logger.LogInformation("Started DalamudUtilService"); return Task.CompletedTask; } diff --git a/MareSynchronos/Services/Events/EventAggregator.cs b/MareSynchronos/Services/Events/EventAggregator.cs index cf057b4..640da9a 100644 --- a/MareSynchronos/Services/Events/EventAggregator.cs +++ b/MareSynchronos/Services/Events/EventAggregator.cs @@ -99,6 +99,8 @@ public class EventAggregator : MediatorSubscriberBase, IHostedService public Task StartAsync(CancellationToken cancellationToken) { + Logger.LogInformation("Starting EventAggregatorService"); + Logger.LogInformation("Started EventAggregatorService"); return Task.CompletedTask; } diff --git a/MareSynchronos/Services/Mediator/MareMediator.cs b/MareSynchronos/Services/Mediator/MareMediator.cs index b5a654b..e8f24c4 100644 --- a/MareSynchronos/Services/Mediator/MareMediator.cs +++ b/MareSynchronos/Services/Mediator/MareMediator.cs @@ -54,7 +54,7 @@ public sealed class MareMediator : IHostedService public Task StartAsync(CancellationToken cancellationToken) { - _logger.LogTrace("Starting MareMediator"); + _logger.LogInformation("Starting MareMediator"); _ = Task.Run(async () => { @@ -73,7 +73,7 @@ public sealed class MareMediator : IHostedService } }); - _logger.LogTrace("Started MareMediator"); + _logger.LogInformation("Started MareMediator"); return Task.CompletedTask; } diff --git a/MareSynchronos/Services/PerformanceCollectorService.cs b/MareSynchronos/Services/PerformanceCollectorService.cs index aab74a4..617c5b8 100644 --- a/MareSynchronos/Services/PerformanceCollectorService.cs +++ b/MareSynchronos/Services/PerformanceCollectorService.cs @@ -78,7 +78,9 @@ public sealed class PerformanceCollectorService : IHostedService public Task StartAsync(CancellationToken cancellationToken) { + _logger.LogInformation("Starting PerformanceCollectorService"); _ = Task.Run(PeriodicLogPrune, _periodicLogPruneTask.Token); + _logger.LogInformation("Started PerformanceCollectorService"); return Task.CompletedTask; } diff --git a/MareSynchronos/UI/DtrEntry.cs b/MareSynchronos/UI/DtrEntry.cs index 18ca0ba..a3c9def 100644 --- a/MareSynchronos/UI/DtrEntry.cs +++ b/MareSynchronos/UI/DtrEntry.cs @@ -46,7 +46,9 @@ public sealed class DtrEntry : IDisposable, IHostedService public Task StartAsync(CancellationToken cancellationToken) { + _logger.LogInformation("Starting DtrEntry"); _runTask = Task.Run(RunAsync, _cancellationTokenSource.Token); + _logger.LogInformation("Started DtrEntry"); return Task.CompletedTask; } @@ -127,7 +129,7 @@ public sealed class DtrEntry : IDisposable, IHostedService { visiblePairs = _pairManager.GetOnlineUserPairs() .Where(x => x.IsVisible) - .Select(x => string.Format("{0} ({1})", _configService.Current.PreferNoteInDtrTooltip ? x.GetNote() ?? x.PlayerName : x.PlayerName, x.UserData.AliasOrUID )); + .Select(x => string.Format("{0} ({1})", _configService.Current.PreferNoteInDtrTooltip ? x.GetNote() ?? x.PlayerName : x.PlayerName, x.UserData.AliasOrUID)); } else { @@ -135,7 +137,7 @@ public sealed class DtrEntry : IDisposable, IHostedService .Where(x => x.IsVisible) .Select(x => string.Format("{0}", _configService.Current.PreferNoteInDtrTooltip ? x.GetNote() ?? x.PlayerName : x.PlayerName)); } - + tooltip = $"Mare Synchronos: Connected{Environment.NewLine}----------{Environment.NewLine}{string.Join(Environment.NewLine, visiblePairs)}"; } else diff --git a/MareSynchronos/UI/SettingsUi.cs b/MareSynchronos/UI/SettingsUi.cs index 3b1afab..12c451b 100644 --- a/MareSynchronos/UI/SettingsUi.cs +++ b/MareSynchronos/UI/SettingsUi.cs @@ -537,7 +537,10 @@ public class SettingsUi : WindowMediatorSubscriberBase _uiShared.DrawCacheDirectorySetting(); ImGui.AlignTextToFramePadding(); - ImGui.TextUnformatted($"Currently utilized local storage: {UiSharedService.ByteToString(_cacheMonitor.FileCacheSize)}"); + if (_cacheMonitor.FileCacheSize >= 0) + ImGui.TextUnformatted($"Currently utilized local storage: {UiSharedService.ByteToString(_cacheMonitor.FileCacheSize)}"); + else + ImGui.TextUnformatted($"Currently utilized local storage: Calculating..."); ImGui.TextUnformatted($"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}"); bool useFileCompactor = _configService.Current.UseCompactor; bool isLinux = Util.IsWine(); @@ -545,7 +548,7 @@ public class SettingsUi : WindowMediatorSubscriberBase { UiSharedService.ColorTextWrapped("Hint: To free up space when using Mare consider enabling the File Compactor", ImGuiColors.DalamudYellow); } - if (isLinux) ImGui.BeginDisabled(); + if (isLinux || !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); if (ImGui.Checkbox("Use file compactor", ref useFileCompactor)) { _configService.Current.UseCompactor = useFileCompactor; @@ -561,7 +564,8 @@ public class SettingsUi : WindowMediatorSubscriberBase _ = Task.Run(() => { _fileCompactor.CompactStorage(compress: true); - _ = _cacheMonitor.RecalculateFileCacheSize(); + CancellationTokenSource cts = new(); + _cacheMonitor.RecalculateFileCacheSize(cts.Token); }); } UiSharedService.AttachToolTip("This will run compression on all files in your current Mare Storage." + Environment.NewLine @@ -572,7 +576,8 @@ public class SettingsUi : WindowMediatorSubscriberBase _ = Task.Run(() => { _fileCompactor.CompactStorage(compress: false); - _ = _cacheMonitor.RecalculateFileCacheSize(); + CancellationTokenSource cts = new(); + _cacheMonitor.RecalculateFileCacheSize(cts.Token); }); } UiSharedService.AttachToolTip("This will run decompression on all files in your current Mare Storage."); @@ -581,10 +586,10 @@ public class SettingsUi : WindowMediatorSubscriberBase { UiSharedService.ColorText($"File compactor currently running ({_fileCompactor.Progress})", ImGuiColors.DalamudYellow); } - if (isLinux) + if (isLinux || !_cacheMonitor.StorageisNTFS) { ImGui.EndDisabled(); - ImGui.TextUnformatted("The file compactor is only available on Windows."); + ImGui.TextUnformatted("The file compactor is only available on Windows and NTFS drives."); } ImGuiHelpers.ScaledDummy(new Vector2(10, 10));