friendlynamazu/client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

make storage size calculation asynchronous and running in parallel

Browse Source
This commit is contained in:
rootdarkarchon
2024-02-15 02:38:41 +01:00
parent 01e36f64ff
commit 549a93654a
11 changed files with 110 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
MareSynchronos/FileCache/CacheMonitor.cs
View File

@@ -18,6 +18,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
private readonly PerformanceCollectorService _performanceCollector;
private long _currentFileProgress = 0;
private CancellationTokenSource _scanCancellationTokenSource = new();
private readonly CancellationTokenSource _periodicCalculationTokenSource = new();
private readonly string[] _allowedExtensions = [".mdl", ".tex", ".mtrl", ".tmb", ".pap", ".avfx", ".atex", ".sklb", ".eid", ".phyb", ".scd", ".skp", ".shpk"];
public CacheMonitor(ILogger<CacheMonitor> logger, IpcManager ipcManager, MareConfigService configService,
@@ -57,6 +58,25 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
{
StartMareWatcher(configService.Current.CacheFolder);
}
var token = _periodicCalculationTokenSource.Token;
_ = Task.Run(async () =>
{
Logger.LogInformation("Starting Periodic Storage Directory Calculation Task");
var token = _periodicCalculationTokenSource.Token;
while (!token.IsCancellationRequested)
{
try
{
RecalculateFileCacheSize(token);
}
catch
{
// ignore
}
await Task.Delay(TimeSpan.FromMinutes(1), token).ConfigureAwait(false);
}
}, token);
}
public long CurrentFileProgress => _currentFileProgress;
@@ -86,17 +106,21 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
PenumbraWatcher = null;
}
public bool StorageisNTFS { get; private set; } = false;
public void StartMareWatcher(string? marePath)
{
MareWatcher?.Dispose();
if (string.IsNullOrEmpty(marePath))
if (string.IsNullOrEmpty(marePath) || !Directory.Exists(marePath))
{
MareWatcher = null;
Logger.LogWarning("Mare file path is not set, cannot start the FSW for Mare.");
return;
}
RecalculateFileCacheSize();
DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase);
Logger.LogInformation("Mare Storage is on NTFS drive: {isNtfs}", StorageisNTFS);
Logger.LogDebug("Initializing Mare FSW on {path}", marePath);
MareWatcher = new()
@@ -248,9 +272,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}
}
_ = RecalculateFileCacheSize();
if (changes.Any(c => c.Value.ChangeType == WatcherChangeTypes.Deleted))
{
lock (_fileDbManager)
@@ -356,13 +377,33 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}, token);
}
public bool RecalculateFileCacheSize()
public void RecalculateFileCacheSize(CancellationToken token)
{
FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder).Sum(f =>
if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder))
{
FileCacheSize = 0;
return;
}
FileCacheSize = -1;
DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
try
{
return _fileCompactor.GetFileSizeOnDisk(f);
FileCacheDriveFree = di.AvailableFreeSpace;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder);
}
FileCacheSize = Directory.EnumerateFiles(_configService.Current.CacheFolder)
.AsParallel().Sum(f =>
{
token.ThrowIfCancellationRequested();
try
{
return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS);
}
catch
{
@@ -370,12 +411,9 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
}
});
DriveInfo di = new DriveInfo(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName);
FileCacheDriveFree = di.AvailableFreeSpace;
var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d);
if (FileCacheSize < maxCacheInBytes) return false;
if (FileCacheSize < maxCacheInBytes) return;
var allFiles = Directory.EnumerateFiles(_configService.Current.CacheFolder)
.Select(f => new FileInfo(f)).OrderBy(f => f.LastAccessTime).ToList();
@@ -387,8 +425,6 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
File.Delete(oldestFile.FullName);
allFiles.Remove(oldestFile);
}
return true;
}
public void ResetLocks()
@@ -412,6 +448,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
MareWatcher?.Dispose();
_penumbraFswCts?.CancelDispose();
_mareFswCts?.CancelDispose();
_periodicCalculationTokenSource?.CancelDispose();
}
private void FullFileScan(CancellationToken ct)

12
MareSynchronos/FileCache/FileCacheManager.cs
View File

@@ -352,12 +352,17 @@ public sealed class FileCacheManager : IHostedService
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Starting FileCacheManager");
lock (_fileWriteLock)
{
try
{
_logger.LogInformation("Checking for {bakPath}", CsvBakPath);
if (File.Exists(CsvBakPath))
{
_logger.LogInformation("{bakPath} found, moving to {csvPath}", CsvBakPath, _csvPath);
File.Move(CsvBakPath, _csvPath, overwrite: true);
}
}
@@ -378,6 +383,8 @@ public sealed class FileCacheManager : IHostedService
if (File.Exists(_csvPath))
{
_logger.LogInformation("{csvPath} found, parsing", _csvPath);
bool success = false;
string[] entries = [];
int attempts = 0;
@@ -385,6 +392,7 @@ public sealed class FileCacheManager : IHostedService
{
try
{
_logger.LogInformation("Attempting to read {csvPath}", _csvPath);
entries = File.ReadAllLines(_csvPath);
success = true;
}
@@ -401,6 +409,8 @@ public sealed class FileCacheManager : IHostedService
_logger.LogWarning("Could not load entries from {path}, continuing with empty file cache", _csvPath);
}
_logger.LogInformation("Found {amount} files in {path}", entries.Length, _csvPath);
Dictionary<string, bool> processedFiles = new(StringComparer.OrdinalIgnoreCase);
foreach (var entry in entries)
{
@@ -447,6 +457,8 @@ public sealed class FileCacheManager : IHostedService
}
}
_logger.LogInformation("Started FileCacheManager");
return Task.CompletedTask;
}

14
MareSynchronos/FileCache/FileCompactor.cs
View File

@@ -62,9 +62,11 @@ public sealed class FileCompactor
MassCompactRunning = false;
}
public long GetFileSizeOnDisk(string filePath)
public long GetFileSizeOnDisk(string filePath, bool? isNTFS = null)
{
if (Dalamud.Utility.Util.IsWine()) return new FileInfo(filePath).Length;
bool ntfs = isNTFS ?? string.Equals(new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
if (Dalamud.Utility.Util.IsWine() || !ntfs) return new FileInfo(filePath).Length;
var clusterSize = GetClusterSize(filePath);
if (clusterSize == -1) return new FileInfo(filePath).Length;
@@ -105,6 +107,14 @@ public sealed class FileCompactor
private void CompactFile(string filePath)
{
var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName);
bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase);
if (!isNTFS)
{
_logger.LogWarning("Drive for file {file} is not NTFS", filePath);
return;
}
var oldSize = new FileInfo(filePath).Length;
var clusterSize = GetClusterSize(filePath);

3
MareSynchronos/Interop/IpcProvider.cs
View File

@@ -28,11 +28,12 @@ public class IpcProvider : IHostedService
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogDebug("Starting IpcProvider Service");
_logger.LogInformation("Starting IpcProviderService");
_loadFileProvider = _pi.GetIpcProvider<string, GameObject, bool>("MareSynchronos.LoadMcdf");
_loadFileProvider.RegisterFunc(LoadMcdf);
_loadFileAsyncProvider = _pi.GetIpcProvider<string, GameObject, Task<bool>>("MareSynchronos.LoadMcdfAsync");
_loadFileAsyncProvider.RegisterFunc(LoadMcdfAsync);
_logger.LogInformation("Started IpcProviderService");
return Task.CompletedTask;
}

6
MareSynchronos/Plugin.cs
View File

@@ -137,15 +137,15 @@ public sealed class Plugin : IDalamudPlugin
s.GetRequiredService<CacheMonitor>(), s.GetRequiredService<FileDialogManager>(), s.GetRequiredService<MareConfigService>(), s.GetRequiredService<DalamudUtilService>(),
pluginInterface, s.GetRequiredService<Dalamud.Localization>(), s.GetRequiredService<ServerConfigurationManager>(), s.GetRequiredService<MareMediator>()));
collection.AddHostedService(p => p.GetRequiredService<MareMediator>());
collection.AddHostedService(p => p.GetRequiredService<MarePlugin>());
collection.AddHostedService(p => p.GetRequiredService<FileCacheManager>());
collection.AddHostedService(p => p.GetRequiredService<ConfigurationMigrator>());
collection.AddHostedService(p => p.GetRequiredService<DalamudUtilService>());
collection.AddHostedService(p => p.GetRequiredService<PerformanceCollectorService>());
collection.AddHostedService(p => p.GetRequiredService<DtrEntry>());
collection.AddHostedService(p => p.GetRequiredService<MarePlugin>());
collection.AddHostedService(p => p.GetRequiredService<EventAggregator>());
collection.AddHostedService(p => p.GetRequiredService<IpcProvider>());
collection.AddHostedService(p => p.GetRequiredService<FileCacheManager>());
collection.AddHostedService(p => p.GetRequiredService<MareMediator>());
})
.Build()
.RunAsync(_pluginCts.Token);

2
MareSynchronos/Services/DalamudUtilService.cs
View File

@@ -284,12 +284,14 @@ public class DalamudUtilService : IHostedService, IMediatorSubscriber
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Starting DalamudUtilService");
_framework.Update += FrameworkOnUpdate;
if (IsLoggedIn)
{
_classJobId = _clientState.LocalPlayer!.ClassJob.Id;
}
_logger.LogInformation("Started DalamudUtilService");
return Task.CompletedTask;
}

2
MareSynchronos/Services/Events/EventAggregator.cs
View File

@@ -99,6 +99,8 @@ public class EventAggregator : MediatorSubscriberBase, IHostedService
public Task StartAsync(CancellationToken cancellationToken)
{
Logger.LogInformation("Starting EventAggregatorService");
Logger.LogInformation("Started EventAggregatorService");
return Task.CompletedTask;
}

4
MareSynchronos/Services/Mediator/MareMediator.cs
View File

@@ -54,7 +54,7 @@ public sealed class MareMediator : IHostedService
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogTrace("Starting MareMediator");
_logger.LogInformation("Starting MareMediator");
_ = Task.Run(async () =>
{
@@ -73,7 +73,7 @@ public sealed class MareMediator : IHostedService
}
});
_logger.LogTrace("Started MareMediator");
_logger.LogInformation("Started MareMediator");
return Task.CompletedTask;
}

2
MareSynchronos/Services/PerformanceCollectorService.cs
View File

@@ -78,7 +78,9 @@ public sealed class PerformanceCollectorService : IHostedService
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Starting PerformanceCollectorService");
_ = Task.Run(PeriodicLogPrune, _periodicLogPruneTask.Token);
_logger.LogInformation("Started PerformanceCollectorService");
return Task.CompletedTask;
}

2
MareSynchronos/UI/DtrEntry.cs
View File

@@ -46,7 +46,9 @@ public sealed class DtrEntry : IDisposable, IHostedService
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Starting DtrEntry");
_runTask = Task.Run(RunAsync, _cancellationTokenSource.Token);
_logger.LogInformation("Started DtrEntry");
return Task.CompletedTask;
}

15
MareSynchronos/UI/SettingsUi.cs
View File

@@ -537,7 +537,10 @@ public class SettingsUi : WindowMediatorSubscriberBase
_uiShared.DrawCacheDirectorySetting();
ImGui.AlignTextToFramePadding();
if (_cacheMonitor.FileCacheSize >= 0)
ImGui.TextUnformatted($"Currently utilized local storage: {UiSharedService.ByteToString(_cacheMonitor.FileCacheSize)}");
else
ImGui.TextUnformatted($"Currently utilized local storage: Calculating...");
ImGui.TextUnformatted($"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}");
bool useFileCompactor = _configService.Current.UseCompactor;
bool isLinux = Util.IsWine();
@@ -545,7 +548,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
{
UiSharedService.ColorTextWrapped("Hint: To free up space when using Mare consider enabling the File Compactor", ImGuiColors.DalamudYellow);
}
if (isLinux) ImGui.BeginDisabled();
if (isLinux || !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled();
if (ImGui.Checkbox("Use file compactor", ref useFileCompactor))
{
_configService.Current.UseCompactor = useFileCompactor;
@@ -561,7 +564,8 @@ public class SettingsUi : WindowMediatorSubscriberBase
_ = Task.Run(() =>
{
_fileCompactor.CompactStorage(compress: true);
_ = _cacheMonitor.RecalculateFileCacheSize();
CancellationTokenSource cts = new();
_cacheMonitor.RecalculateFileCacheSize(cts.Token);
});
}
UiSharedService.AttachToolTip("This will run compression on all files in your current Mare Storage." + Environment.NewLine
@@ -572,7 +576,8 @@ public class SettingsUi : WindowMediatorSubscriberBase
_ = Task.Run(() =>
{
_fileCompactor.CompactStorage(compress: false);
_ = _cacheMonitor.RecalculateFileCacheSize();
CancellationTokenSource cts = new();
_cacheMonitor.RecalculateFileCacheSize(cts.Token);
});
}
UiSharedService.AttachToolTip("This will run decompression on all files in your current Mare Storage.");
@@ -581,10 +586,10 @@ public class SettingsUi : WindowMediatorSubscriberBase
{
UiSharedService.ColorText($"File compactor currently running ({_fileCompactor.Progress})", ImGuiColors.DalamudYellow);
}
if (isLinux)
if (isLinux || !_cacheMonitor.StorageisNTFS)
{
ImGui.EndDisabled();
ImGui.TextUnformatted("The file compactor is only available on Windows.");
ImGui.TextUnformatted("The file compactor is only available on Windows and NTFS drives.");
}
ImGuiHelpers.ScaledDummy(new Vector2(10, 10));