diff --git a/MareSynchronosServer/MareSynchronosServer/Services/SystemInfoService.cs b/MareSynchronosServer/MareSynchronosServer/Services/SystemInfoService.cs index d90474e..dc80392 100644 --- a/MareSynchronosServer/MareSynchronosServer/Services/SystemInfoService.cs +++ b/MareSynchronosServer/MareSynchronosServer/Services/SystemInfoService.cs @@ -11,7 +11,7 @@ using StackExchange.Redis.Extensions.Core.Abstractions; namespace MareSynchronosServer.Services; -public class SystemInfoService : IHostedService, IDisposable +public sealed class SystemInfoService : IHostedService, IDisposable { private readonly MareMetrics _mareMetrics; private readonly IConfigurationService _config; diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs index 7e43ba0..f0ceb7b 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs @@ -17,9 +17,9 @@ public class MainController : ControllerBase [HttpGet(MareFiles.Main_SendReady)] [Authorize(Policy = "Internal")] - public IActionResult SendReadyToClients(string uid, Guid requestId) + public async Task SendReadyToClients(string uid, Guid requestId) { - _messageService.SendDownloadReady(uid, requestId); + await _messageService.SendDownloadReady(uid, requestId).ConfigureAwait(false); return Ok(); } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs index 9cb3801..e457ba4 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs @@ -29,13 +29,13 @@ public class ServerFilesController : ControllerBase private readonly CachedFileProvider _cachedFileProvider; private readonly IConfigurationService _configuration; private readonly IHubContext _hubContext; - private readonly MareDbContext _mareDbContext; + private readonly IDbContextFactory _mareDbContext; private readonly MareMetrics _metricsClient; public ServerFilesController(ILogger logger, CachedFileProvider cachedFileProvider, IConfigurationService configuration, IHubContext hubContext, - MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger) + IDbContextFactory mareDbContext, MareMetrics metricsClient) : base(logger) { _basePath = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false) ? configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)) @@ -50,7 +50,8 @@ public class ServerFilesController : ControllerBase [HttpPost(MareFiles.ServerFiles_DeleteAll)] public async Task FilesDeleteAll() { - var ownFiles = await _mareDbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false); + using var dbContext = await _mareDbContext.CreateDbContextAsync(); + var ownFiles = await dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false); bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); foreach (var dbFile in ownFiles) @@ -65,8 +66,8 @@ public class ServerFilesController : ControllerBase } } - _mareDbContext.Files.RemoveRange(ownFiles); - await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); + dbContext.Files.RemoveRange(ownFiles); + await dbContext.SaveChangesAsync().ConfigureAwait(false); return Ok(); } @@ -74,11 +75,15 @@ public class ServerFilesController : ControllerBase [HttpGet(MareFiles.ServerFiles_GetSizes)] public async Task FilesGetSizes([FromBody] List hashes) { - var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries. + using var dbContext = await _mareDbContext.CreateDbContextAsync(); + var forbiddenFiles = await dbContext.ForbiddenUploadEntries. Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false); List response = new(); - var cacheFile = await _mareDbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size, k.RawSize }).AsNoTracking().ToListAsync().ConfigureAwait(false); + var cacheFile = await dbContext.Files.AsNoTracking() + .Where(f => hashes.Contains(f.Hash)) + .Select(k => new { k.Hash, k.Size, k.RawSize }) + .ToListAsync().ConfigureAwait(false); var allFileShards = new List(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List())); @@ -128,10 +133,12 @@ public class ServerFilesController : ControllerBase [HttpPost(MareFiles.ServerFiles_FilesSend)] public async Task FilesSend([FromBody] FilesSendDto filesSendDto) { + using var dbContext = await _mareDbContext.CreateDbContextAsync(); + var userSentHashes = new HashSet(filesSendDto.FileHashes.Distinct(StringComparer.Ordinal).Select(s => string.Concat(s.Where(c => char.IsLetterOrDigit(c)))), StringComparer.Ordinal); var notCoveredFiles = new Dictionary(StringComparer.Ordinal); - var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); - var existingFiles = await _mareDbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); + var forbiddenFiles = await dbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); + var existingFiles = await dbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false); List fileCachesToUpload = new(); foreach (var hash in userSentHashes) @@ -171,9 +178,11 @@ public class ServerFilesController : ControllerBase [RequestSizeLimit(200 * 1024 * 1024)] public async Task UploadFile(string hash, CancellationToken requestAborted) { + using var dbContext = await _mareDbContext.CreateDbContextAsync(); + _logger.LogInformation("{user} uploading file {file}", MareUser, hash); hash = hash.ToUpperInvariant(); - var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); + var existingFile = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); if (existingFile != null) return Ok(); SemaphoreSlim? fileLock = null; @@ -199,7 +208,7 @@ public class ServerFilesController : ControllerBase try { - var existingFileCheck2 = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); + var existingFileCheck2 = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); if (existingFileCheck2 != null) { return Ok(); @@ -227,7 +236,7 @@ public class ServerFilesController : ControllerBase await compressedFileStream.CopyToAsync(fileStream).ConfigureAwait(false); // update on db - await _mareDbContext.Files.AddAsync(new FileCache() + await dbContext.Files.AddAsync(new FileCache() { Hash = hash, UploadDate = DateTime.UtcNow, @@ -236,7 +245,7 @@ public class ServerFilesController : ControllerBase Uploaded = true, RawSize = data.LongLength }).ConfigureAwait(false); - await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); + await dbContext.SaveChangesAsync().ConfigureAwait(false); bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); @@ -274,9 +283,11 @@ public class ServerFilesController : ControllerBase [RequestSizeLimit(200 * 1024 * 1024)] public async Task UploadFileMunged(string hash, CancellationToken requestAborted) { + using var dbContext = await _mareDbContext.CreateDbContextAsync(); + _logger.LogInformation("{user} uploading munged file {file}", MareUser, hash); hash = hash.ToUpperInvariant(); - var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); + var existingFile = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); if (existingFile != null) return Ok(); SemaphoreSlim? fileLock = null; @@ -302,7 +313,7 @@ public class ServerFilesController : ControllerBase try { - var existingFileCheck2 = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); + var existingFileCheck2 = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); if (existingFileCheck2 != null) { return Ok(); @@ -329,7 +340,7 @@ public class ServerFilesController : ControllerBase await fileStream.WriteAsync(unmungedFile.AsMemory()).ConfigureAwait(false); // update on db - await _mareDbContext.Files.AddAsync(new FileCache() + await dbContext.Files.AddAsync(new FileCache() { Hash = hash, UploadDate = DateTime.UtcNow, @@ -338,7 +349,7 @@ public class ServerFilesController : ControllerBase Uploaded = true, RawSize = data.LongLength }).ConfigureAwait(false); - await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); + await dbContext.SaveChangesAsync().ConfigureAwait(false); bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); @@ -382,9 +393,11 @@ public class ServerFilesController : ControllerBase [RequestSizeLimit(200 * 1024 * 1024)] public async Task UploadFileRaw(string hash, CancellationToken requestAborted) { + using var dbContext = await _mareDbContext.CreateDbContextAsync(); + _logger.LogInformation("{user} uploading raw file {file}", MareUser, hash); hash = hash.ToUpperInvariant(); - var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); + var existingFile = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); if (existingFile != null) return Ok(); SemaphoreSlim? fileLock = null; @@ -410,7 +423,7 @@ public class ServerFilesController : ControllerBase try { - var existingFileCheck2 = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); + var existingFileCheck2 = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash); if (existingFileCheck2 != null) { return Ok(); @@ -437,7 +450,7 @@ public class ServerFilesController : ControllerBase await compressedStream.CopyToAsync(fileStream).ConfigureAwait(false); // update on db - await _mareDbContext.Files.AddAsync(new FileCache() + await dbContext.Files.AddAsync(new FileCache() { Hash = hash, UploadDate = DateTime.UtcNow, @@ -446,7 +459,7 @@ public class ServerFilesController : ControllerBase Uploaded = true, RawSize = rawFileStream.Length }).ConfigureAwait(false); - await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); + await dbContext.SaveChangesAsync().ConfigureAwait(false); bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs index 510d600..6b40958 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs @@ -49,6 +49,11 @@ public class Program config.AddEnvironmentVariables(); }) + .ConfigureLogging((ctx, builder) => + { + builder.AddConfiguration(ctx.Configuration.GetSection("Logging")); + builder.AddFile(o => o.RootPath = AppContext.BaseDirectory); + }) .ConfigureWebHostDefaults(webBuilder => { webBuilder.UseContentRoot(AppContext.BaseDirectory); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs index 932cf51..e2e3509 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs @@ -2,5 +2,5 @@ public interface IClientReadyMessageService { - void SendDownloadReady(string uid, Guid requestId); + Task SendDownloadReady(string uid, Guid requestId); } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs index cb8c98f..8aa2abc 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs @@ -15,12 +15,9 @@ public class MainClientReadyMessageService : IClientReadyMessageService _mareHub = mareHub; } - public void SendDownloadReady(string uid, Guid requestId) + public async Task SendDownloadReady(string uid, Guid requestId) { - _ = Task.Run(async () => - { - _logger.LogInformation("Sending Client Ready for {uid}:{requestId} to SignalR", uid, requestId); - await _mareHub.Clients.User(uid).SendAsync(nameof(IMareHub.Client_DownloadReady), requestId).ConfigureAwait(false); - }); + _logger.LogInformation("Sending Client Ready for {uid}:{requestId} to SignalR", uid, requestId); + await _mareHub.Clients.User(uid).SendAsync(nameof(IMareHub.Client_DownloadReady), requestId).ConfigureAwait(false); } } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs index 7666ee2..0ba1501 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs @@ -13,18 +13,19 @@ namespace MareSynchronosStaticFilesServer.Services; public class MainFileCleanupService : IHostedService { private readonly IConfigurationService _configuration; + private readonly IDbContextFactory _dbContextFactory; private readonly ILogger _logger; private readonly MareMetrics _metrics; - private readonly IServiceProvider _services; private CancellationTokenSource _cleanupCts; public MainFileCleanupService(MareMetrics metrics, ILogger logger, - IServiceProvider services, IConfigurationService configuration) + IConfigurationService configuration, + IDbContextFactory dbContextFactory) { _metrics = metrics; _logger = logger; - _services = services; _configuration = configuration; + _dbContextFactory = dbContextFactory; } public Task StartAsync(CancellationToken cancellationToken) @@ -35,7 +36,7 @@ public class MainFileCleanupService : IHostedService _cleanupCts = new(); - _ = CleanUpTask(_cleanupCts.Token); + _ = Task.Run(() => CleanUpTask(_cleanupCts.Token)).ConfigureAwait(false); return Task.CompletedTask; } @@ -155,8 +156,9 @@ public class MainFileCleanupService : IHostedService bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false); var hotStorageDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); var coldStorageDir = _configuration.GetValue(nameof(StaticFilesServerConfiguration.ColdStorageDirectory)); - using var scope = _services.CreateScope(); - using var dbContext = scope.ServiceProvider.GetService()!; + using var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); + + _logger.LogInformation("Running File Cleanup Task"); try { @@ -166,16 +168,19 @@ public class MainFileCleanupService : IHostedService var linkedToken = linkedTokenCts.Token; DirectoryInfo dirHotStorage = new(hotStorageDir); + _logger.LogInformation("File Cleanup Task gathering hot storage files"); var allFilesInHotStorage = dirHotStorage.GetFiles("*", SearchOption.AllDirectories).ToList(); var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14); var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1); var sizeLimit = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1); + _logger.LogInformation("File Cleanup Task cleaning up outdated hot storage files"); var remainingHotFiles = await CleanUpOutdatedFiles(hotStorageDir, allFilesInHotStorage, unusedRetention, forcedDeletionAfterHours, deleteFromDb: !useColdStorage, dbContext: dbContext, ct: linkedToken).ConfigureAwait(false); + _logger.LogInformation("File Cleanup Task cleaning up hot storage file beyond size limit"); var finalRemainingHotFiles = CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit, deleteFromDb: !useColdStorage, dbContext: dbContext, ct: linkedToken); @@ -183,15 +188,18 @@ public class MainFileCleanupService : IHostedService if (useColdStorage) { DirectoryInfo dirColdStorage = new(coldStorageDir); + _logger.LogInformation("File Cleanup Task gathering cold storage files"); var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList(); var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60); var coldStorageSize = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1); // clean up cold storage + _logger.LogInformation("File Cleanup Task cleaning up outdated cold storage files"); var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1, deleteFromDb: true, dbContext: dbContext, ct: linkedToken).ConfigureAwait(false); + _logger.LogInformation("File Cleanup Task cleaning up cold storage file beyond size limit"); var finalRemainingColdFiles = CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize, deleteFromDb: true, dbContext: dbContext, ct: linkedToken); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs index 655928d..0a40f03 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs @@ -22,27 +22,24 @@ public class ShardClientReadyMessageService : IClientReadyMessageService _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0")); } - public void SendDownloadReady(string uid, Guid requestId) + public async Task SendDownloadReady(string uid, Guid requestId) { - _ = Task.Run(async () => + var mainUrl = _configurationService.GetValue(nameof(StaticFilesServerConfiguration.MainFileServerAddress)); + var path = MareFiles.MainSendReadyFullPath(mainUrl, uid, requestId); + using HttpRequestMessage msg = new() { - var mainUrl = _configurationService.GetValue(nameof(StaticFilesServerConfiguration.MainFileServerAddress)); - var path = MareFiles.MainSendReadyFullPath(mainUrl, uid, requestId); - using HttpRequestMessage msg = new() - { - RequestUri = path - }; - msg.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _tokenGenerator.Token); + RequestUri = path + }; + msg.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _tokenGenerator.Token); - _logger.LogInformation("Sending Client Ready for {uid}:{requestId} to {path}", uid, requestId, path); - try - { - using var result = await _httpClient.SendAsync(msg).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failure to send for {uid}:{requestId}", uid, requestId); - } - }); + _logger.LogInformation("Sending Client Ready for {uid}:{requestId} to {path}", uid, requestId, path); + try + { + using var result = await _httpClient.SendAsync(msg).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failure to send for {uid}:{requestId}", uid, requestId); + } } } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs index 9358fc0..c49378b 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs @@ -106,6 +106,15 @@ public class Startup }).UseSnakeCaseNamingConvention(); options.EnableThreadSafetyChecks(false); }, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024)); + services.AddDbContextFactory(options => + { + options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder => + { + builder.MigrationsHistoryTable("_efmigrationshistory", "public"); + builder.MigrationsAssembly("MareSynchronosShared"); + }).UseSnakeCaseNamingConvention(); + options.EnableThreadSafetyChecks(false); + }); var signalRServiceBuilder = services.AddSignalR(hubOptions => {