some minor fixes and improvements, probably

MareSynchronosServer/MareSynchronosServer/Services/SystemInfoService.cs

@@ -11,7 +11,7 @@ using StackExchange.Redis.Extensions.Core.Abstractions;
 
 namespace MareSynchronosServer.Services;
 
-public class SystemInfoService : IHostedService, IDisposable
+public sealed class SystemInfoService : IHostedService, IDisposable
 {
     private readonly MareMetrics _mareMetrics;
     private readonly IConfigurationService<ServerConfiguration> _config;

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs

@@ -17,9 +17,9 @@ public class MainController : ControllerBase
 
     [HttpGet(MareFiles.Main_SendReady)]
     [Authorize(Policy = "Internal")]
-    public IActionResult SendReadyToClients(string uid, Guid requestId)
+    public async Task<IActionResult> SendReadyToClients(string uid, Guid requestId)
     {
-        _messageService.SendDownloadReady(uid, requestId);
+        await _messageService.SendDownloadReady(uid, requestId).ConfigureAwait(false);
         return Ok();
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs

@@ -29,13 +29,13 @@ public class ServerFilesController : ControllerBase
     private readonly CachedFileProvider _cachedFileProvider;
     private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
     private readonly IHubContext<MareHub> _hubContext;
-    private readonly MareDbContext _mareDbContext;
+    private readonly IDbContextFactory<MareDbContext> _mareDbContext;
     private readonly MareMetrics _metricsClient;
 
     public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider,
         IConfigurationService<StaticFilesServerConfiguration> configuration,
         IHubContext<MareHub> hubContext,
-        MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger)
+        IDbContextFactory<MareDbContext> mareDbContext, MareMetrics metricsClient) : base(logger)
     {
         _basePath = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)
             ? configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory))
@@ -50,7 +50,8 @@ public class ServerFilesController : ControllerBase
     [HttpPost(MareFiles.ServerFiles_DeleteAll)]
     public async Task<IActionResult> FilesDeleteAll()
     {
-        var ownFiles = await _mareDbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false);
+        using var dbContext = await _mareDbContext.CreateDbContextAsync();
+        var ownFiles = await dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false);
         bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
 
         foreach (var dbFile in ownFiles)
@@ -65,8 +66,8 @@ public class ServerFilesController : ControllerBase
             }
         }
 
-        _mareDbContext.Files.RemoveRange(ownFiles);
+        dbContext.Files.RemoveRange(ownFiles);
-        await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
+        await dbContext.SaveChangesAsync().ConfigureAwait(false);
 
         return Ok();
     }
@@ -74,11 +75,15 @@ public class ServerFilesController : ControllerBase
     [HttpGet(MareFiles.ServerFiles_GetSizes)]
     public async Task<IActionResult> FilesGetSizes([FromBody] List<string> hashes)
     {
-        var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries.
+        using var dbContext = await _mareDbContext.CreateDbContextAsync();
+        var forbiddenFiles = await dbContext.ForbiddenUploadEntries.
             Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
         List<DownloadFileDto> response = new();
 
-        var cacheFile = await _mareDbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size, k.RawSize }).AsNoTracking().ToListAsync().ConfigureAwait(false);
+        var cacheFile = await dbContext.Files.AsNoTracking()
+            .Where(f => hashes.Contains(f.Hash))
+            .Select(k => new { k.Hash, k.Size, k.RawSize })
+            .ToListAsync().ConfigureAwait(false);
 
         var allFileShards = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
 
@@ -128,10 +133,12 @@ public class ServerFilesController : ControllerBase
     [HttpPost(MareFiles.ServerFiles_FilesSend)]
     public async Task<IActionResult> FilesSend([FromBody] FilesSendDto filesSendDto)
     {
+        using var dbContext = await _mareDbContext.CreateDbContextAsync();
+
         var userSentHashes = new HashSet<string>(filesSendDto.FileHashes.Distinct(StringComparer.Ordinal).Select(s => string.Concat(s.Where(c => char.IsLetterOrDigit(c)))), StringComparer.Ordinal);
         var notCoveredFiles = new Dictionary<string, UploadFileDto>(StringComparer.Ordinal);
-        var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
+        var forbiddenFiles = await dbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
-        var existingFiles = await _mareDbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
+        var existingFiles = await dbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
 
         List<FileCache> fileCachesToUpload = new();
         foreach (var hash in userSentHashes)
@@ -171,9 +178,11 @@ public class ServerFilesController : ControllerBase
     [RequestSizeLimit(200 * 1024 * 1024)]
     public async Task<IActionResult> UploadFile(string hash, CancellationToken requestAborted)
     {
+        using var dbContext = await _mareDbContext.CreateDbContextAsync();
+
         _logger.LogInformation("{user} uploading file {file}", MareUser, hash);
         hash = hash.ToUpperInvariant();
-        var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+        var existingFile = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
         if (existingFile != null) return Ok();
 
         SemaphoreSlim? fileLock = null;
@@ -199,7 +208,7 @@ public class ServerFilesController : ControllerBase
 
         try
         {
-            var existingFileCheck2 = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+            var existingFileCheck2 = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
             if (existingFileCheck2 != null)
             {
                 return Ok();
@@ -227,7 +236,7 @@ public class ServerFilesController : ControllerBase
             await compressedFileStream.CopyToAsync(fileStream).ConfigureAwait(false);
 
             // update on db
-            await _mareDbContext.Files.AddAsync(new FileCache()
+            await dbContext.Files.AddAsync(new FileCache()
             {
                 Hash = hash,
                 UploadDate = DateTime.UtcNow,
@@ -236,7 +245,7 @@ public class ServerFilesController : ControllerBase
                 Uploaded = true,
                 RawSize = data.LongLength
             }).ConfigureAwait(false);
-            await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
+            await dbContext.SaveChangesAsync().ConfigureAwait(false);
 
             bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
 
@@ -274,9 +283,11 @@ public class ServerFilesController : ControllerBase
     [RequestSizeLimit(200 * 1024 * 1024)]
     public async Task<IActionResult> UploadFileMunged(string hash, CancellationToken requestAborted)
     {
+        using var dbContext = await _mareDbContext.CreateDbContextAsync();
+
         _logger.LogInformation("{user} uploading munged file {file}", MareUser, hash);
         hash = hash.ToUpperInvariant();
-        var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+        var existingFile = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
         if (existingFile != null) return Ok();
 
         SemaphoreSlim? fileLock = null;
@@ -302,7 +313,7 @@ public class ServerFilesController : ControllerBase
 
         try
         {
-            var existingFileCheck2 = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+            var existingFileCheck2 = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
             if (existingFileCheck2 != null)
             {
                 return Ok();
@@ -329,7 +340,7 @@ public class ServerFilesController : ControllerBase
             await fileStream.WriteAsync(unmungedFile.AsMemory()).ConfigureAwait(false);
 
             // update on db
-            await _mareDbContext.Files.AddAsync(new FileCache()
+            await dbContext.Files.AddAsync(new FileCache()
             {
                 Hash = hash,
                 UploadDate = DateTime.UtcNow,
@@ -338,7 +349,7 @@ public class ServerFilesController : ControllerBase
                 Uploaded = true,
                 RawSize = data.LongLength
             }).ConfigureAwait(false);
-            await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
+            await dbContext.SaveChangesAsync().ConfigureAwait(false);
 
             bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
 
@@ -382,9 +393,11 @@ public class ServerFilesController : ControllerBase
     [RequestSizeLimit(200 * 1024 * 1024)]
     public async Task<IActionResult> UploadFileRaw(string hash, CancellationToken requestAborted)
     {
+        using var dbContext = await _mareDbContext.CreateDbContextAsync();
+
         _logger.LogInformation("{user} uploading raw file {file}", MareUser, hash);
         hash = hash.ToUpperInvariant();
-        var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+        var existingFile = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
         if (existingFile != null) return Ok();
 
         SemaphoreSlim? fileLock = null;
@@ -410,7 +423,7 @@ public class ServerFilesController : ControllerBase
 
         try
         {
-            var existingFileCheck2 = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
+            var existingFileCheck2 = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
             if (existingFileCheck2 != null)
             {
                 return Ok();
@@ -437,7 +450,7 @@ public class ServerFilesController : ControllerBase
             await compressedStream.CopyToAsync(fileStream).ConfigureAwait(false);
 
             // update on db
-            await _mareDbContext.Files.AddAsync(new FileCache()
+            await dbContext.Files.AddAsync(new FileCache()
             {
                 Hash = hash,
                 UploadDate = DateTime.UtcNow,
@@ -446,7 +459,7 @@ public class ServerFilesController : ControllerBase
                 Uploaded = true,
                 RawSize = rawFileStream.Length
             }).ConfigureAwait(false);
-            await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
+            await dbContext.SaveChangesAsync().ConfigureAwait(false);
 
             bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
 

MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs

@@ -49,6 +49,11 @@ public class Program
 
                 config.AddEnvironmentVariables();
             })
+            .ConfigureLogging((ctx, builder) =>
+            {
+                builder.AddConfiguration(ctx.Configuration.GetSection("Logging"));
+                builder.AddFile(o => o.RootPath = AppContext.BaseDirectory);
+            })
             .ConfigureWebHostDefaults(webBuilder =>
             {
                 webBuilder.UseContentRoot(AppContext.BaseDirectory);

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs

@@ -2,5 +2,5 @@
 
 public interface IClientReadyMessageService
 {
-    void SendDownloadReady(string uid, Guid requestId);
+    Task SendDownloadReady(string uid, Guid requestId);
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs

@@ -15,12 +15,9 @@ public class MainClientReadyMessageService : IClientReadyMessageService
         _mareHub = mareHub;
     }
 
-    public void SendDownloadReady(string uid, Guid requestId)
+    public async Task SendDownloadReady(string uid, Guid requestId)
-    {
-        _ = Task.Run(async () =>
     {
         _logger.LogInformation("Sending Client Ready for {uid}:{requestId} to SignalR", uid, requestId);
         await _mareHub.Clients.User(uid).SendAsync(nameof(IMareHub.Client_DownloadReady), requestId).ConfigureAwait(false);
-        });
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs

@@ -13,18 +13,19 @@ namespace MareSynchronosStaticFilesServer.Services;
 public class MainFileCleanupService : IHostedService
 {
     private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
+    private readonly IDbContextFactory<MareDbContext> _dbContextFactory;
     private readonly ILogger<MainFileCleanupService> _logger;
     private readonly MareMetrics _metrics;
-    private readonly IServiceProvider _services;
     private CancellationTokenSource _cleanupCts;
 
     public MainFileCleanupService(MareMetrics metrics, ILogger<MainFileCleanupService> logger,
-        IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration)
+        IConfigurationService<StaticFilesServerConfiguration> configuration,
+        IDbContextFactory<MareDbContext> dbContextFactory)
     {
         _metrics = metrics;
         _logger = logger;
-        _services = services;
         _configuration = configuration;
+        _dbContextFactory = dbContextFactory;
     }
 
     public Task StartAsync(CancellationToken cancellationToken)
@@ -35,7 +36,7 @@ public class MainFileCleanupService : IHostedService
 
         _cleanupCts = new();
 
-        _ = CleanUpTask(_cleanupCts.Token);
+        _ = Task.Run(() => CleanUpTask(_cleanupCts.Token)).ConfigureAwait(false);
 
         return Task.CompletedTask;
     }
@@ -155,8 +156,9 @@ public class MainFileCleanupService : IHostedService
             bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
             var hotStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
             var coldStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
-            using var scope = _services.CreateScope();
+            using var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false);
-            using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
+
+            _logger.LogInformation("Running File Cleanup Task");
 
             try
             {
@@ -166,16 +168,19 @@ public class MainFileCleanupService : IHostedService
                 var linkedToken = linkedTokenCts.Token;
 
                 DirectoryInfo dirHotStorage = new(hotStorageDir);
+                _logger.LogInformation("File Cleanup Task gathering hot storage files");
                 var allFilesInHotStorage = dirHotStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
 
                 var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
                 var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
                 var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
 
+                _logger.LogInformation("File Cleanup Task cleaning up outdated hot storage files");
                 var remainingHotFiles = await CleanUpOutdatedFiles(hotStorageDir, allFilesInHotStorage, unusedRetention, forcedDeletionAfterHours,
                     deleteFromDb: !useColdStorage, dbContext: dbContext,
                     ct: linkedToken).ConfigureAwait(false);
 
+                _logger.LogInformation("File Cleanup Task cleaning up hot storage file beyond size limit");
                 var finalRemainingHotFiles = CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit,
                     deleteFromDb: !useColdStorage, dbContext: dbContext,
                     ct: linkedToken);
@@ -183,15 +188,18 @@ public class MainFileCleanupService : IHostedService
                 if (useColdStorage)
                 {
                     DirectoryInfo dirColdStorage = new(coldStorageDir);
+                    _logger.LogInformation("File Cleanup Task gathering cold storage files");
                     var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
 
                     var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60);
                     var coldStorageSize = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1);
 
                     // clean up cold storage
+                    _logger.LogInformation("File Cleanup Task cleaning up outdated cold storage files");
                     var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1,
                         deleteFromDb: true, dbContext: dbContext,
                         ct: linkedToken).ConfigureAwait(false);
+                    _logger.LogInformation("File Cleanup Task cleaning up cold storage file beyond size limit");
                     var finalRemainingColdFiles = CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize,
                         deleteFromDb: true, dbContext: dbContext,
                         ct: linkedToken);

MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs

@@ -22,9 +22,7 @@ public class ShardClientReadyMessageService : IClientReadyMessageService
         _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0"));
     }
 
-    public void SendDownloadReady(string uid, Guid requestId)
+    public async Task SendDownloadReady(string uid, Guid requestId)
-    {
-        _ = Task.Run(async () =>
     {
         var mainUrl = _configurationService.GetValue<Uri>(nameof(StaticFilesServerConfiguration.MainFileServerAddress));
         var path = MareFiles.MainSendReadyFullPath(mainUrl, uid, requestId);
@@ -43,6 +41,5 @@ public class ShardClientReadyMessageService : IClientReadyMessageService
         {
             _logger.LogError(ex, "Failure to send for {uid}:{requestId}", uid, requestId);
         }
-        });
     }
 }

MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs

@@ -106,6 +106,15 @@ public class Startup
                 }).UseSnakeCaseNamingConvention();
                 options.EnableThreadSafetyChecks(false);
             }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
+            services.AddDbContextFactory<MareDbContext>(options =>
+            {
+                options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
+                {
+                    builder.MigrationsHistoryTable("_efmigrationshistory", "public");
+                    builder.MigrationsAssembly("MareSynchronosShared");
+                }).UseSnakeCaseNamingConvention();
+                options.EnableThreadSafetyChecks(false);
+            });
 
             var signalRServiceBuilder = services.AddSignalR(hubOptions =>
             {