friendlynamazu/server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

revert chunking on fileservers

Browse Source
This commit is contained in:
rootdarkarchon
2023-11-17 11:12:48 +01:00
parent a1e82bcdb6
commit df87e45e9c
1 changed files with 44 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs
View File

@@ -5,7 +5,6 @@ using MareSynchronosShared.Models;
using MareSynchronosShared.Services; using MareSynchronosShared.Services;
using MareSynchronosStaticFilesServer.Utils; using MareSynchronosStaticFilesServer.Utils;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using System.Globalization;
namespace MareSynchronosStaticFilesServer.Services; namespace MareSynchronosStaticFilesServer.Services;
@@ -63,8 +62,8 @@ public class FileCleanupService : IHostedService
var now = DateTime.Now; var now = DateTime.Now;
TimeOnly currentTime = new(now.Hour, now.Minute, now.Second); TimeOnly currentTime = new(now.Hour, now.Minute, now.Second);
TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 10, 0); TimeOnly futureTime = new(now.Hour, now.Minute - now.Minute % 30, 0);
var span = futureTime.AddMinutes(10) - currentTime; var span = futureTime.AddMinutes(30) - currentTime;
_logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span)); _logger.LogInformation("File Cleanup Complete, next run at {date}", now.Add(span));
await Task.Delay(span, ct).ConfigureAwait(false); await Task.Delay(span, ct).ConfigureAwait(false);
@@ -165,16 +164,13 @@ public class FileCleanupService : IHostedService
var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
DirectoryInfo dir = new(_cacheDir); DirectoryInfo dir = new(_cacheDir);
var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories); var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
int filesToTake = 10000;
var files = dbContext.Files.OrderBy(f => f.Hash); var files = dbContext.Files.OrderBy(f => f.Hash);
var filesChunk = await files.Take(filesToTake).ToListAsync(cancellationToken: ct).ConfigureAwait(false); List<FileCache> allFiles;
int iterations = 1; if (_isMainServer) allFiles = await dbContext.Files.ToListAsync(ct).ConfigureAwait(false);
var allFiles = new List<FileCache>(); else allFiles = await dbContext.Files.AsNoTracking().ToListAsync(ct).ConfigureAwait(false);
while (filesChunk.Any())
{
int fileCounter = 0; int fileCounter = 0;
foreach (var fileCache in filesChunk.Where(f => f.Uploaded)) foreach (var fileCache in allFiles.Where(f => f.Uploaded))
{ {
bool fileDeleted = false; bool fileDeleted = false;
@@ -223,11 +219,6 @@ public class FileCleanupService : IHostedService
ct.ThrowIfCancellationRequested(); ct.ThrowIfCancellationRequested();
} }
allFiles.AddRange(filesChunk);
filesChunk = await files.Skip(filesToTake * iterations).Take(filesToTake).ToListAsync(cancellationToken: ct).ConfigureAwait(false);
iterations++;
}
// clean up files that are on disk but not in DB for some reason // clean up files that are on disk but not in DB for some reason
CleanUpOrphanedFiles(allFiles, allFilesInDir, ct); CleanUpOrphanedFiles(allFiles, allFilesInDir, ct);
} }