use chunks for cleanup

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs

@@ -16,7 +16,6 @@ using Microsoft.AspNetCore.SignalR;
 using Microsoft.EntityFrameworkCore;
 using System.Collections.Concurrent;
 using System.Security.Cryptography;
-using System.Security.Policy;
 using System.Text.Json;
 using System.Text.RegularExpressions;
 
@@ -230,6 +229,7 @@ public class ServerFilesController : ControllerBase
         finally
         {
             fileLock.Release();
+            fileLock.Dispose();
         }
     }
 
@@ -305,6 +305,7 @@ public class ServerFilesController : ControllerBase
         finally
         {
             fileLock.Release();
+            fileLock.Dispose();
         }
     }
 
@@ -388,6 +389,7 @@ public class ServerFilesController : ControllerBase
         finally
         {
             fileLock.Release();
+            fileLock.Dispose();
         }
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/MareSynchronosStaticFilesServer.csproj

@@ -18,6 +18,10 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="IDisposableAnalyzers" Version="4.0.7">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="lz4net" Version="1.0.15.93" />
     <PackageReference Include="Meziantou.Analyzer" Version="2.0.93">
       <PrivateAssets>all</PrivateAssets>

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/FileCleanupService.cs

@@ -164,12 +164,19 @@ public class FileCleanupService : IHostedService
             var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours));
             DirectoryInfo dir = new(_cacheDir);
             var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories);
-            var allFiles = await dbContext.Files.ToListAsync().ConfigureAwait(false);
+            int filesToTake = 10000;
-            int fileCounter = 0;
+            var filesChunk = await dbContext.Files.Take(filesToTake).ToListAsync().ConfigureAwait(false);
-            foreach (var fileCache in allFiles.Where(f => f.Uploaded))
+            int iterations = 1;
+            var allFiles = new List<FileCache>();
+            while (filesChunk.Any())
+            {
+                int fileCounter = 0;
+
+                foreach (var fileCache in filesChunk.Where(f => f.Uploaded))
                 {
-                var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash);
                     bool fileDeleted = false;
+
+                    var file = FilePathUtil.GetFileInfoForHash(_cacheDir, fileCache.Hash);
                     if (file == null && _isMainServer)
                     {
                         _logger.LogInformation("File does not exist anymore: {fileName}", fileCache.Hash);
@@ -214,6 +221,11 @@ public class FileCleanupService : IHostedService
                     ct.ThrowIfCancellationRequested();
                 }
 
+                allFiles.AddRange(filesChunk);
+                filesChunk = await dbContext.Files.Skip(filesToTake * iterations).Take(filesToTake).ToListAsync(cancellationToken: ct).ConfigureAwait(false);
+                iterations++;
+            }
+
             // clean up files that are on disk but not in DB for some reason
             CleanUpOrphanedFiles(allFiles, allFilesInDir, ct);
         }

MareSynchronosServer/MareSynchronosStaticFilesServer/Utils/BlockFileDataStream.cs

@@ -40,7 +40,8 @@ public sealed class BlockFileDataStream : Stream
 
     protected override void Dispose(bool disposing)
     {
-        base.Dispose(disposing);
+        if (disposing)
+        {
             foreach (var substream in _substreams)
             {
                 // probably unnecessary but better safe than sorry
@@ -48,6 +49,9 @@ public sealed class BlockFileDataStream : Stream
             }
         }
 
+        base.Dispose(disposing);
+    }
+
     public override void Flush()
     {
         throw new NotSupportedException();