make storage size calculation asynchronous and running in parallel

MareSynchronos/FileCache/CacheMonitor.cs

@@ -18,6 +18,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
     private readonly PerformanceCollectorService _performanceCollector;
     private long _currentFileProgress = 0;
     private CancellationTokenSource _scanCancellationTokenSource = new();
+    private readonly CancellationTokenSource _periodicCalculationTokenSource = new();
     private readonly string[] _allowedExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp", ".shpk"];
 
     public CacheMonitor(ILogger<CacheMonitor> logger, IpcManager ipcManager, MareConfigService configService,
@@ -57,6 +58,25 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         {
             StartMareWatcher(configService.Current.CacheFolder);
         }
+
+        var token = _periodicCalculationTokenSource.Token;
+        _ = Task.Run(async () =>
+        {
+            Logger.LogInformation("Starting Periodic Storage Directory Calculation Task");
+            var token = _periodicCalculationTokenSource.Token;
+            while (!token.IsCancellationRequested)
+            {
+                try
+                {
+                    RecalculateFileCacheSize(token);
+                }
+                catch
+                {
+                    // ignore
+                }
+                await Task.Delay(TimeSpan.FromMinutes(1), token).ConfigureAwait(false);
+            }
+        }, token);
     }
 
     public long CurrentFileProgress => _currentFileProgress;
@@ -86,17 +106,21 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         PenumbraWatcher = null;
     }
 
+    public bool StorageisNTFS { get; private set; } = false;
+
     public void StartMareWatcher(string? marePath)
     {
         MareWatcher?.Dispose();
-        if (string.IsNullOrEmpty(marePath))
+        if (string.IsNullOrEmpty(marePath) || !Directory.Exists(marePath))
         {
             MareWatcher = null;
             Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare.");
             return;
         }
 
-        RecalculateFileCacheSize();
+        DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
+        StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase);
+        Logger.LogInformation("Mare Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
 
         Logger.LogDebug("Initializing Mare FSW on {path}", marePath);
         MareWatcher = new()
@@ -248,9 +272,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
             }
         }
 
-        _ = RecalculateFileCacheSize();
-
-
         if (changes.Any(c => c.Value.ChangeType == WatcherChangeTypes.Deleted))
         {
             lock (_fileDbManager)
@@ -356,26 +377,43 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         }, token);
     }
 
-    public bool RecalculateFileCacheSize()
+    public void RecalculateFileCacheSize(CancellationToken token)
     {
-        FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder).Sum(f =>
+        if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder))
         {
-            try
+            FileCacheSize = 0;
-            {
+            return;
-                return _fileCompactor.GetFileSizeOnDisk(f);
+        }
-            }
-            catch
-            {
-                return 0;
-            }
-        });
 
-        DriveInfo di = new DriveInfo(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
+        FileCacheSize = -1;
-        FileCacheDriveFree = di.AvailableFreeSpace;
+        DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
+        try
+        {
+            FileCacheDriveFree = di.AvailableFreeSpace;
+        }
+        catch (Exception ex)
+        {
+            Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder);
+        }
+
+        FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder)
+            .AsParallel().Sum(f =>
+            {
+                token.ThrowIfCancellationRequested();
+
+                try
+                {
+                    return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS);
+                }
+                catch
+                {
+                    return 0;
+                }
+            });
 
         var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
 
-        if (FileCacheSize < maxCacheInBytes) return false;
+        if (FileCacheSize < maxCacheInBytes) return;
 
         var allFiles = Directory.EnumerateFiles(_configService.Current.CacheFolder)
             .Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
@@ -387,8 +425,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
             File.Delete(oldestFile.FullName);
             allFiles.Remove(oldestFile);
         }
-
-        return true;
     }
 
     public void ResetLocks()
@@ -412,6 +448,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
         MareWatcher?.Dispose();
         _penumbraFswCts?.CancelDispose();
         _mareFswCts?.CancelDispose();
+        _periodicCalculationTokenSource?.CancelDispose();
     }
 
     private void FullFileScan(CancellationToken ct)

MareSynchronos/FileCache/FileCacheManager.cs

@@ -352,12 +352,17 @@ public sealed class FileCacheManager : IHostedService
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
+        _logger.LogInformation("Starting FileCacheManager");
         lock (_fileWriteLock)
         {
             try
             {
+                _logger.LogInformation("Checking for {bakPath}", CsvBakPath);
+
                 if (File.Exists(CsvBakPath))
                 {
+                    _logger.LogInformation("{bakPath} found, moving to {csvPath}", CsvBakPath, _csvPath);
+
                     File.Move(CsvBakPath, _csvPath, overwrite: true);
                 }
             }
@@ -378,6 +383,8 @@ public sealed class FileCacheManager : IHostedService
 
         if (File.Exists(_csvPath))
         {
+            _logger.LogInformation("{csvPath} found, parsing", _csvPath);
+
             bool success = false;
             string[] entries = [];
             int attempts = 0;
@@ -385,6 +392,7 @@ public sealed class FileCacheManager : IHostedService
             {
                 try
                 {
+                    _logger.LogInformation("Attempting to read {csvPath}", _csvPath);
                     entries = File.ReadAllLines(_csvPath);
                     success = true;
                 }
@@ -401,6 +409,8 @@ public sealed class FileCacheManager : IHostedService
                 _logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath);
             }
 
+            _logger.LogInformation("Found {amount} files in {path}", entries.Length, _csvPath);
+
             Dictionary<string, bool> processedFiles = new(StringComparer.OrdinalIgnoreCase);
             foreach (var entry in entries)
             {
@@ -447,6 +457,8 @@ public sealed class FileCacheManager : IHostedService
             }
         }
 
+        _logger.LogInformation("Started FileCacheManager");
+
         return Task.CompletedTask;
     }
 

MareSynchronos/FileCache/FileCompactor.cs

@@ -62,9 +62,11 @@ public sealed class FileCompactor
         MassCompactRunning = false;
     }
 
-    public long GetFileSizeOnDisk(string filePath)
+    public long GetFileSizeOnDisk(string filePath, bool? isNTFS = null)
     {
-        if (Dalamud.Utility.Util.IsWine()) return new FileInfo(filePath).Length;
+        bool ntfs = isNTFS ?? string.Equals(new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
+
+        if (Dalamud.Utility.Util.IsWine() || !ntfs) return new FileInfo(filePath).Length;
 
         var clusterSize = GetClusterSize(filePath);
         if (clusterSize == -1) return new FileInfo(filePath).Length;
@@ -105,6 +107,14 @@ public sealed class FileCompactor
 
     private void CompactFile(string filePath)
     {
+        var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName);
+        bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
+        if (!isNTFS)
+        {
+            _logger.LogWarning("Drive for file {file} is not NTFS", filePath);
+            return;
+        }
+
         var oldSize = new FileInfo(filePath).Length;
         var clusterSize = GetClusterSize(filePath);
 

MareSynchronos/Interop/IpcProvider.cs

@@ -28,11 +28,12 @@ public class IpcProvider : IHostedService
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
-        _logger.LogDebug("Starting IpcProvider Service");
+        _logger.LogInformation("Starting IpcProviderService");
         _loadFileProvider = _pi.GetIpcProvider<string, GameObject, bool>("MareSynchronos.LoadMcdf");
         _loadFileProvider.RegisterFunc(LoadMcdf);
         _loadFileAsyncProvider = _pi.GetIpcProvider<string, GameObject, Task<bool>>("MareSynchronos.LoadMcdfAsync");
         _loadFileAsyncProvider.RegisterFunc(LoadMcdfAsync);
+        _logger.LogInformation("Started IpcProviderService");
         return Task.CompletedTask;
     }
 

MareSynchronos/Plugin.cs

@@ -137,15 +137,15 @@ public sealed class Plugin : IDalamudPlugin
                 s.GetRequiredService<CacheMonitor>(), s.GetRequiredService<FileDialogManager>(), s.GetRequiredService<MareConfigService>(), s.GetRequiredService<DalamudUtilService>(),
                 pluginInterface, s.GetRequiredService<Dalamud.Localization>(), s.GetRequiredService<ServerConfigurationManager>(), s.GetRequiredService<MareMediator>()));
 
-            collection.AddHostedService(p => p.GetRequiredService<MareMediator>());
+            collection.AddHostedService(p => p.GetRequiredService<MarePlugin>());
+            collection.AddHostedService(p => p.GetRequiredService<FileCacheManager>());
             collection.AddHostedService(p => p.GetRequiredService<ConfigurationMigrator>());
             collection.AddHostedService(p => p.GetRequiredService<DalamudUtilService>());
             collection.AddHostedService(p => p.GetRequiredService<PerformanceCollectorService>());
             collection.AddHostedService(p => p.GetRequiredService<DtrEntry>());
-            collection.AddHostedService(p => p.GetRequiredService<MarePlugin>());
             collection.AddHostedService(p => p.GetRequiredService<EventAggregator>());
             collection.AddHostedService(p => p.GetRequiredService<IpcProvider>());
-            collection.AddHostedService(p => p.GetRequiredService<FileCacheManager>());
+            collection.AddHostedService(p => p.GetRequiredService<MareMediator>());
         })
         .Build()
         .RunAsync(_pluginCts.Token);

MareSynchronos/Services/DalamudUtilService.cs

@@ -284,12 +284,14 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
+        _logger.LogInformation("Starting DalamudUtilService");
         _framework.Update += FrameworkOnUpdate;
         if (IsLoggedIn)
         {
             _classJobId = _clientState.LocalPlayer!.ClassJob.Id;
         }
 
+        _logger.LogInformation("Started DalamudUtilService");
         return Task.CompletedTask;
     }
 

MareSynchronos/Services/Events/EventAggregator.cs

@@ -99,6 +99,8 @@ public class EventAggregator : MediatorSubscriberBase, IHostedService
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
+        Logger.LogInformation("Starting EventAggregatorService");
+        Logger.LogInformation("Started EventAggregatorService");
         return Task.CompletedTask;
     }
 

MareSynchronos/Services/Mediator/MareMediator.cs

@@ -54,7 +54,7 @@ public sealed class MareMediator : IHostedService
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
-        _logger.LogTrace("Starting MareMediator");
+        _logger.LogInformation("Starting MareMediator");
 
         _ = Task.Run(async () =>
         {
@@ -73,7 +73,7 @@ public sealed class MareMediator : IHostedService
             }
         });
 
-        _logger.LogTrace("Started MareMediator");
+        _logger.LogInformation("Started MareMediator");
 
         return Task.CompletedTask;
     }

MareSynchronos/Services/PerformanceCollectorService.cs

@@ -78,7 +78,9 @@ public sealed class PerformanceCollectorService : IHostedService
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
+        _logger.LogInformation("Starting PerformanceCollectorService");
         _ = Task.Run(PeriodicLogPrune, _periodicLogPruneTask.Token);
+        _logger.LogInformation("Started PerformanceCollectorService");
         return Task.CompletedTask;
     }
 

MareSynchronos/UI/DtrEntry.cs

@@ -46,7 +46,9 @@ public sealed class DtrEntry : IDisposable, IHostedService
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
+        _logger.LogInformation("Starting DtrEntry");
         _runTask = Task.Run(RunAsync, _cancellationTokenSource.Token);
+        _logger.LogInformation("Started DtrEntry");
         return Task.CompletedTask;
     }
 
@@ -127,7 +129,7 @@ public sealed class DtrEntry : IDisposable, IHostedService
                 {
                     visiblePairs = _pairManager.GetOnlineUserPairs()
                         .Where(x => x.IsVisible)
-                        .Select(x => string.Format("{0} ({1})", _configService.Current.PreferNoteInDtrTooltip ? x.GetNote() ?? x.PlayerName : x.PlayerName, x.UserData.AliasOrUID ));
+                        .Select(x => string.Format("{0} ({1})", _configService.Current.PreferNoteInDtrTooltip ? x.GetNote() ?? x.PlayerName : x.PlayerName, x.UserData.AliasOrUID));
                 }
                 else
                 {

MareSynchronos/UI/SettingsUi.cs

@@ -537,7 +537,10 @@ public class SettingsUi : WindowMediatorSubscriberBase
 
         _uiShared.DrawCacheDirectorySetting();
         ImGui.AlignTextToFramePadding();
-        ImGui.TextUnformatted($"Currently utilized local storage: {UiSharedService.ByteToString(_cacheMonitor.FileCacheSize)}");
+        if (_cacheMonitor.FileCacheSize >= 0)
+            ImGui.TextUnformatted($"Currently utilized local storage: {UiSharedService.ByteToString(_cacheMonitor.FileCacheSize)}");
+        else
+            ImGui.TextUnformatted($"Currently utilized local storage: Calculating...");
         ImGui.TextUnformatted($"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}");
         bool useFileCompactor = _configService.Current.UseCompactor;
         bool isLinux = Util.IsWine();
@@ -545,7 +548,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
         {
             UiSharedService.ColorTextWrapped("Hint: To free up space when using Mare consider enabling the File Compactor", ImGuiColors.DalamudYellow);
         }
-        if (isLinux) ImGui.BeginDisabled();
+        if (isLinux || !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled();
         if (ImGui.Checkbox("Use file compactor", ref useFileCompactor))
         {
             _configService.Current.UseCompactor = useFileCompactor;
@@ -561,7 +564,8 @@ public class SettingsUi : WindowMediatorSubscriberBase
                 _ = Task.Run(() =>
                 {
                     _fileCompactor.CompactStorage(compress: true);
-                    _ = _cacheMonitor.RecalculateFileCacheSize();
+                    CancellationTokenSource cts = new();
+                    _cacheMonitor.RecalculateFileCacheSize(cts.Token);
                 });
             }
             UiSharedService.AttachToolTip("This will run compression on all files in your current Mare Storage." + Environment.NewLine
@@ -572,7 +576,8 @@ public class SettingsUi : WindowMediatorSubscriberBase
                 _ = Task.Run(() =>
                 {
                     _fileCompactor.CompactStorage(compress: false);
-                    _ = _cacheMonitor.RecalculateFileCacheSize();
+                    CancellationTokenSource cts = new();
+                    _cacheMonitor.RecalculateFileCacheSize(cts.Token);
                 });
             }
             UiSharedService.AttachToolTip("This will run decompression on all files in your current Mare Storage.");
@@ -581,10 +586,10 @@ public class SettingsUi : WindowMediatorSubscriberBase
         {
             UiSharedService.ColorText($"File compactor currently running ({_fileCompactor.Progress})", ImGuiColors.DalamudYellow);
         }
-        if (isLinux)
+        if (isLinux || !_cacheMonitor.StorageisNTFS)
         {
             ImGui.EndDisabled();
-            ImGui.TextUnformatted("The file compactor is only available on Windows.");
+            ImGui.TextUnformatted("The file compactor is only available on Windows and NTFS drives.");
         }
         ImGuiHelpers.ScaledDummy(new Vector2(10, 10));