friendlynamazu/server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

speed up cleanup service

Browse Source
This commit is contained in:
Stanley Dimant
2024-05-03 16:09:37 +02:00
parent 90c9c85498
commit 6e3ff34522
1 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs
View File

@@ -3,6 +3,7 @@ using MareSynchronosShared.Data;
using MareSynchronosShared.Metrics; using MareSynchronosShared.Metrics;
using MareSynchronosShared.Models; using MareSynchronosShared.Models;
using MareSynchronosShared.Services; using MareSynchronosShared.Services;
using MareSynchronosStaticFilesServer.Utils;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
namespace MareSynchronosStaticFilesServer.Services; namespace MareSynchronosStaticFilesServer.Services;
@@ -48,7 +49,7 @@ public class MainFileCleanupService : IHostedService
var coldStorageSize = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1); var coldStorageSize = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1);
// clean up cold storage // clean up cold storage
var remainingColdFiles = await CleanUpOutdatedFiles(allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1, var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1,
isColdStorage: true, deleteFromDb: true, isColdStorage: true, deleteFromDb: true,
dbContext, ct).ConfigureAwait(false); dbContext, ct).ConfigureAwait(false);
var finalRemainingColdFiles = CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize, var finalRemainingColdFiles = CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize,
@@ -67,7 +68,7 @@ public class MainFileCleanupService : IHostedService
var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1); var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1); var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
var remainingHotFiles = await CleanUpOutdatedFiles(allFilesInHotStorage, unusedRetention, forcedDeletionAfterHours, var remainingHotFiles = await CleanUpOutdatedFiles(hotStorageDir, allFilesInHotStorage, unusedRetention, forcedDeletionAfterHours,
isColdStorage: false, deleteFromDb: !useColdStorage, isColdStorage: false, deleteFromDb: !useColdStorage,
dbContext, ct).ConfigureAwait(false); dbContext, ct).ConfigureAwait(false);
@@ -163,6 +164,7 @@ public class MainFileCleanupService : IHostedService
_metrics.DecGauge(MetricsAPI.GaugeFilesTotal); _metrics.DecGauge(MetricsAPI.GaugeFilesTotal);
file.Delete(); file.Delete();
_logger.LogInformation("File not in DB, deleting: {fileName}", file.Name); _logger.LogInformation("File not in DB, deleting: {fileName}", file.Name);
allPhysicalFiles.Remove(file);
} }
ct.ThrowIfCancellationRequested(); ct.ThrowIfCancellationRequested();
@@ -171,7 +173,7 @@ public class MainFileCleanupService : IHostedService
return allPhysicalFiles; return allPhysicalFiles;
} }
private async Task<List<FileInfo>> CleanUpOutdatedFiles(List<FileInfo> allFilesInDir, int unusedRetention, int forcedDeletionAfterHours, private async Task<List<FileInfo>> CleanUpOutdatedFiles(string dir, List<FileInfo> allFilesInDir, int unusedRetention, int forcedDeletionAfterHours,
bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct) bool isColdStorage, bool deleteFromDb, MareDbContext dbContext, CancellationToken ct)
{ {
try try
@@ -186,12 +188,13 @@ public class MainFileCleanupService : IHostedService
var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention)); var prevTime = DateTime.Now.Subtract(TimeSpan.FromDays(unusedRetention));
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
List<FileCache> allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false); List<FileCache> allDbFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false);
List<string> removedFileHashes = new List<string>();
int fileCounter = 0; int fileCounter = 0;
foreach (var fileCache in allDbFiles.Where(f => f.Uploaded)) foreach (var fileCache in allDbFiles.Where(f => f.Uploaded))
{ {
bool deleteCurrentFile = false; bool deleteCurrentFile = false;
var file = allFilesInDir.Find(f => string.Equals(f.Name, fileCache.Hash, StringComparison.OrdinalIgnoreCase)); var file = FilePathUtil.GetFileInfoForHash(dir, fileCache.Hash);
if (file == null) if (file == null)
{ {
_logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash); _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
@@ -213,7 +216,7 @@ public class MainFileCleanupService : IHostedService
{ {
if (file != null) file.Delete(); if (file != null) file.Delete();
allFilesInDir.Remove(file); removedFileHashes.Add(fileCache.Hash);
if (deleteFromDb) if (deleteFromDb)
dbContext.Files.Remove(fileCache); dbContext.Files.Remove(fileCache);
@@ -235,7 +238,7 @@ public class MainFileCleanupService : IHostedService
} }
// clean up files that are on disk but not in DB for some reason // clean up files that are on disk but not in DB for some reason
return CleanUpOrphanedFiles(allDbFiles, allFilesInDir, ct); return CleanUpOrphanedFiles(allDbFiles, allFilesInDir.Where(c => !removedFileHashes.Contains(c.Name, StringComparer.OrdinalIgnoreCase)).ToList(), ct);
} }
catch (Exception ex) catch (Exception ex)
{ {