diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs index cc0045f..0933365 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs @@ -16,7 +16,6 @@ using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; using System.Collections.Concurrent; using System.Security.Cryptography; -using System.Security.Policy; using System.Text.Json; using System.Text.RegularExpressions; @@ -230,6 +229,7 @@ public class ServerFilesController : ControllerBase finally { fileLock.Release(); + fileLock.Dispose(); } } @@ -305,6 +305,7 @@ public class ServerFilesController : ControllerBase finally { fileLock.Release(); + fileLock.Dispose(); } } @@ -388,6 +389,7 @@ public class ServerFilesController : ControllerBase finally { fileLock.Release(); + fileLock.Dispose(); } } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj b/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj index 86eadb0..ea363e9 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj @@ -18,6 +18,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + all diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs index c15f216..a2b83c2 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs @@ -164,54 +164,66 @@ public class FileCleanupService : IHostedService var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); DirectoryInfo dir = new(_cacheDir); var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories); - var allFiles = await dbContext.Files.ToListAsync().ConfigureAwait(false); - int fileCounter = 0; - foreach (var fileCache in allFiles.Where(f => f.Uploaded)) + int filesToTake = 10000; + var filesChunk = await dbContext.Files.Take(filesToTake).ToListAsync().ConfigureAwait(false); + int iterations = 1; + var allFiles = new List(); + while (filesChunk.Any()) { - var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash); - bool fileDeleted = false; - if (file == null && _isMainServer) + int fileCounter = 0; + + foreach (var fileCache in filesChunk.Where(f => f.Uploaded)) { - _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash); - dbContext.Files.Remove(fileCache); - fileDeleted = true; - } - else if (file != null && file.LastAccessTime < prevTime) - { - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); - _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); - file.Delete(); - if (_isMainServer) + bool fileDeleted = false; + + var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash); + if (file == null && _isMainServer) { - fileDeleted = true; + _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash); dbContext.Files.Remove(fileCache); + fileDeleted = true; } - } - else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion) - { - _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); - _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); - _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); - file.Delete(); - if (_isMainServer) + else if (file != null && file.LastAccessTime < prevTime) { - fileDeleted = true; - dbContext.Files.Remove(fileCache); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _logger.LogInformation("File outdated: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); + file.Delete(); + if (_isMainServer) + { + fileDeleted = true; + dbContext.Files.Remove(fileCache); + } } + else if (file != null && forcedDeletionAfterHours > 0 && file.LastWriteTime < prevTimeForcedDeletion) + { + _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); + file.Delete(); + if (_isMainServer) + { + fileDeleted = true; + dbContext.Files.Remove(fileCache); + } + } + + if (_isMainServer && !fileDeleted && file != null && fileCache.Size == 0) + { + _logger.LogInformation("Setting File Size of " + fileCache.Hash + " to " + file.Length); + fileCache.Size = file.Length; + // commit every 1000 files to db + if (fileCounter % 1000 == 0) await dbContext.SaveChangesAsync().ConfigureAwait(false); + } + + fileCounter++; + + ct.ThrowIfCancellationRequested(); } - if (_isMainServer && !fileDeleted && file != null && fileCache.Size == 0) - { - _logger.LogInformation("Setting File Size of " + fileCache.Hash + " to " + file.Length); - fileCache.Size = file.Length; - // commit every 1000 files to db - if (fileCounter % 1000 == 0) await dbContext.SaveChangesAsync().ConfigureAwait(false); - } - - fileCounter++; - - ct.ThrowIfCancellationRequested(); + allFiles.AddRange(filesChunk); + filesChunk = await dbContext.Files.Skip(filesToTake * iterations).Take(filesToTake).ToListAsync(cancellationToken: ct).ConfigureAwait(false); + iterations++; } // clean up files that are on disk but not in DB for some reason diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/BlockFileDataStream.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/BlockFileDataStream.cs index 27fac5d..3785698 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/BlockFileDataStream.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/BlockFileDataStream.cs @@ -40,12 +40,16 @@ public sealed class BlockFileDataStream : Stream protected override void Dispose(bool disposing) { - base.Dispose(disposing); - foreach (var substream in _substreams) + if (disposing) { - // probably unnecessary but better safe than sorry - substream.Dispose(); + foreach (var substream in _substreams) + { + // probably unnecessary but better safe than sorry + substream.Dispose(); + } } + + base.Dispose(disposing); } public override void Flush()