friendlynamazu/server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

some minor fixes and improvements, probably

Browse Source
This commit is contained in:
Stanley Dimant
2024-11-06 12:51:23 +01:00
parent 6c5ad25d99
commit 33d5f44754
9 changed files with 85 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MareSynchronosServer/MareSynchronosServer/Services/SystemInfoService.cs
View File

@@ -11,7 +11,7 @@ using StackExchange.Redis.Extensions.Core.Abstractions;
namespace MareSynchronosServer.Services;
public class SystemInfoService : IHostedService, IDisposable
public sealed class SystemInfoService : IHostedService, IDisposable
{
private readonly MareMetrics _mareMetrics;
private readonly IConfigurationService<ServerConfiguration> _config;

4
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs
View File

@@ -17,9 +17,9 @@ public class MainController : ControllerBase
[HttpGet(MareFiles.Main_SendReady)]
[Authorize(Policy = "Internal")]
public IActionResult SendReadyToClients(string uid, Guid requestId)
public async Task<IActionResult> SendReadyToClients(string uid, Guid requestId)
{
_messageService.SendDownloadReady(uid, requestId);
await _messageService.SendDownloadReady(uid, requestId).ConfigureAwait(false);
return Ok();
}
}

55
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs
View File

@@ -29,13 +29,13 @@ public class ServerFilesController : ControllerBase
private readonly CachedFileProvider _cachedFileProvider;
private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
private readonly IHubContext<MareHub> _hubContext;
private readonly MareDbContext _mareDbContext;
private readonly IDbContextFactory<MareDbContext> _mareDbContext;
private readonly MareMetrics _metricsClient;
public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider,
IConfigurationService<StaticFilesServerConfiguration> configuration,
IHubContext<MareHub> hubContext,
MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger)
IDbContextFactory<MareDbContext> mareDbContext, MareMetrics metricsClient) : base(logger)
{
_basePath = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)
? configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory))
@@ -50,7 +50,8 @@ public class ServerFilesController : ControllerBase
[HttpPost(MareFiles.ServerFiles_DeleteAll)]
public async Task<IActionResult> FilesDeleteAll()
{
var ownFiles = await _mareDbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false);
using var dbContext = await _mareDbContext.CreateDbContextAsync();
var ownFiles = await dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == MareUser).ToListAsync().ConfigureAwait(false);
bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
foreach (var dbFile in ownFiles)
@@ -65,8 +66,8 @@ public class ServerFilesController : ControllerBase
}
}
_mareDbContext.Files.RemoveRange(ownFiles);
await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
dbContext.Files.RemoveRange(ownFiles);
await dbContext.SaveChangesAsync().ConfigureAwait(false);
return Ok();
}
@@ -74,11 +75,15 @@ public class ServerFilesController : ControllerBase
[HttpGet(MareFiles.ServerFiles_GetSizes)]
public async Task<IActionResult> FilesGetSizes([FromBody] List<string> hashes)
{
var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries.
using var dbContext = await _mareDbContext.CreateDbContextAsync();
var forbiddenFiles = await dbContext.ForbiddenUploadEntries.
Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
List<DownloadFileDto> response = new();
var cacheFile = await _mareDbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size, k.RawSize }).AsNoTracking().ToListAsync().ConfigureAwait(false);
var cacheFile = await dbContext.Files.AsNoTracking()
.Where(f => hashes.Contains(f.Hash))
.Select(k => new { k.Hash, k.Size, k.RawSize })
.ToListAsync().ConfigureAwait(false);
var allFileShards = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
@@ -128,10 +133,12 @@ public class ServerFilesController : ControllerBase
[HttpPost(MareFiles.ServerFiles_FilesSend)]
public async Task<IActionResult> FilesSend([FromBody] FilesSendDto filesSendDto)
{
using var dbContext = await _mareDbContext.CreateDbContextAsync();
var userSentHashes = new HashSet<string>(filesSendDto.FileHashes.Distinct(StringComparer.Ordinal).Select(s => string.Concat(s.Where(c => char.IsLetterOrDigit(c)))), StringComparer.Ordinal);
var notCoveredFiles = new Dictionary<string, UploadFileDto>(StringComparer.Ordinal);
var forbiddenFiles = await _mareDbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
var existingFiles = await _mareDbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
var forbiddenFiles = await dbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
var existingFiles = await dbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
List<FileCache> fileCachesToUpload = new();
foreach (var hash in userSentHashes)
@@ -171,9 +178,11 @@ public class ServerFilesController : ControllerBase
[RequestSizeLimit(200 * 1024 * 1024)]
public async Task<IActionResult> UploadFile(string hash, CancellationToken requestAborted)
{
using var dbContext = await _mareDbContext.CreateDbContextAsync();
_logger.LogInformation("{user} uploading file {file}", MareUser, hash);
hash = hash.ToUpperInvariant();
var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
var existingFile = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
if (existingFile != null) return Ok();
SemaphoreSlim? fileLock = null;
@@ -199,7 +208,7 @@ public class ServerFilesController : ControllerBase
try
{
var existingFileCheck2 = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
var existingFileCheck2 = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
if (existingFileCheck2 != null)
{
return Ok();
@@ -227,7 +236,7 @@ public class ServerFilesController : ControllerBase
await compressedFileStream.CopyToAsync(fileStream).ConfigureAwait(false);
// update on db
await _mareDbContext.Files.AddAsync(new FileCache()
await dbContext.Files.AddAsync(new FileCache()
{
Hash = hash,
UploadDate = DateTime.UtcNow,
@@ -236,7 +245,7 @@ public class ServerFilesController : ControllerBase
Uploaded = true,
RawSize = data.LongLength
}).ConfigureAwait(false);
await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
await dbContext.SaveChangesAsync().ConfigureAwait(false);
bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
@@ -274,9 +283,11 @@ public class ServerFilesController : ControllerBase
[RequestSizeLimit(200 * 1024 * 1024)]
public async Task<IActionResult> UploadFileMunged(string hash, CancellationToken requestAborted)
{
using var dbContext = await _mareDbContext.CreateDbContextAsync();
_logger.LogInformation("{user} uploading munged file {file}", MareUser, hash);
hash = hash.ToUpperInvariant();
var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
var existingFile = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
if (existingFile != null) return Ok();
SemaphoreSlim? fileLock = null;
@@ -302,7 +313,7 @@ public class ServerFilesController : ControllerBase
try
{
var existingFileCheck2 = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
var existingFileCheck2 = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
if (existingFileCheck2 != null)
{
return Ok();
@@ -329,7 +340,7 @@ public class ServerFilesController : ControllerBase
await fileStream.WriteAsync(unmungedFile.AsMemory()).ConfigureAwait(false);
// update on db
await _mareDbContext.Files.AddAsync(new FileCache()
await dbContext.Files.AddAsync(new FileCache()
{
Hash = hash,
UploadDate = DateTime.UtcNow,
@@ -338,7 +349,7 @@ public class ServerFilesController : ControllerBase
Uploaded = true,
RawSize = data.LongLength
}).ConfigureAwait(false);
await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
await dbContext.SaveChangesAsync().ConfigureAwait(false);
bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
@@ -382,9 +393,11 @@ public class ServerFilesController : ControllerBase
[RequestSizeLimit(200 * 1024 * 1024)]
public async Task<IActionResult> UploadFileRaw(string hash, CancellationToken requestAborted)
{
using var dbContext = await _mareDbContext.CreateDbContextAsync();
_logger.LogInformation("{user} uploading raw file {file}", MareUser, hash);
hash = hash.ToUpperInvariant();
var existingFile = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
var existingFile = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
if (existingFile != null) return Ok();
SemaphoreSlim? fileLock = null;
@@ -410,7 +423,7 @@ public class ServerFilesController : ControllerBase
try
{
var existingFileCheck2 = await _mareDbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
var existingFileCheck2 = await dbContext.Files.SingleOrDefaultAsync(f => f.Hash == hash);
if (existingFileCheck2 != null)
{
return Ok();
@@ -437,7 +450,7 @@ public class ServerFilesController : ControllerBase
await compressedStream.CopyToAsync(fileStream).ConfigureAwait(false);
// update on db
await _mareDbContext.Files.AddAsync(new FileCache()
await dbContext.Files.AddAsync(new FileCache()
{
Hash = hash,
UploadDate = DateTime.UtcNow,
@@ -446,7 +459,7 @@ public class ServerFilesController : ControllerBase
Uploaded = true,
RawSize = rawFileStream.Length
}).ConfigureAwait(false);
await _mareDbContext.SaveChangesAsync().ConfigureAwait(false);
await dbContext.SaveChangesAsync().ConfigureAwait(false);
bool isColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);

5
MareSynchronosServer/MareSynchronosStaticFilesServer/Program.cs
View File

@@ -49,6 +49,11 @@ public class Program
config.AddEnvironmentVariables();
})
.ConfigureLogging((ctx, builder) =>
{
builder.AddConfiguration(ctx.Configuration.GetSection("Logging"));
builder.AddFile(o => o.RootPath = AppContext.BaseDirectory);
})
.ConfigureWebHostDefaults(webBuilder =>
{
webBuilder.UseContentRoot(AppContext.BaseDirectory);

2
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/IClientReadyMessageService.cs
View File

@@ -2,5 +2,5 @@
public interface IClientReadyMessageService
{
void SendDownloadReady(string uid, Guid requestId);
Task SendDownloadReady(string uid, Guid requestId);
}

5
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainClientReadyMessageService.cs
View File

@@ -15,12 +15,9 @@ public class MainClientReadyMessageService : IClientReadyMessageService
_mareHub = mareHub;
}
public void SendDownloadReady(string uid, Guid requestId)
{
_ = Task.Run(async () =>
public async Task SendDownloadReady(string uid, Guid requestId)
{
_logger.LogInformation("Sending Client Ready for {uid}:{requestId} to SignalR", uid, requestId);
await _mareHub.Clients.User(uid).SendAsync(nameof(IMareHub.Client_DownloadReady), requestId).ConfigureAwait(false);
});
}
}

20
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs
View File

@@ -13,18 +13,19 @@ namespace MareSynchronosStaticFilesServer.Services;
public class MainFileCleanupService : IHostedService
{
private readonly IConfigurationService<StaticFilesServerConfiguration> _configuration;
private readonly IDbContextFactory<MareDbContext> _dbContextFactory;
private readonly ILogger<MainFileCleanupService> _logger;
private readonly MareMetrics _metrics;
private readonly IServiceProvider _services;
private CancellationTokenSource _cleanupCts;
public MainFileCleanupService(MareMetrics metrics, ILogger<MainFileCleanupService> logger,
IServiceProvider services, IConfigurationService<StaticFilesServerConfiguration> configuration)
IConfigurationService<StaticFilesServerConfiguration> configuration,
IDbContextFactory<MareDbContext> dbContextFactory)
{
_metrics = metrics;
_logger = logger;
_services = services;
_configuration = configuration;
_dbContextFactory = dbContextFactory;
}
public Task StartAsync(CancellationToken cancellationToken)
@@ -35,7 +36,7 @@ public class MainFileCleanupService : IHostedService
_cleanupCts = new();
_ = CleanUpTask(_cleanupCts.Token);
_ = Task.Run(() => CleanUpTask(_cleanupCts.Token)).ConfigureAwait(false);
return Task.CompletedTask;
}
@@ -155,8 +156,9 @@ public class MainFileCleanupService : IHostedService
bool useColdStorage = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false);
var hotStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
var coldStorageDir = _configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory));
using var scope = _services.CreateScope();
using var dbContext = scope.ServiceProvider.GetService<MareDbContext>()!;
using var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false);
_logger.LogInformation("Running File Cleanup Task");
try
{
@@ -166,16 +168,19 @@ public class MainFileCleanupService : IHostedService
var linkedToken = linkedTokenCts.Token;
DirectoryInfo dirHotStorage = new(hotStorageDir);
_logger.LogInformation("File Cleanup Task gathering hot storage files");
var allFilesInHotStorage = dirHotStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
var unusedRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UnusedFileRetentionPeriodInDays), 14);
var forcedDeletionAfterHours = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ForcedDeletionOfFilesAfterHours), -1);
var sizeLimit = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.CacheSizeHardLimitInGiB), -1);
_logger.LogInformation("File Cleanup Task cleaning up outdated hot storage files");
var remainingHotFiles = await CleanUpOutdatedFiles(hotStorageDir, allFilesInHotStorage, unusedRetention, forcedDeletionAfterHours,
deleteFromDb: !useColdStorage, dbContext: dbContext,
ct: linkedToken).ConfigureAwait(false);
_logger.LogInformation("File Cleanup Task cleaning up hot storage file beyond size limit");
var finalRemainingHotFiles = CleanUpFilesBeyondSizeLimit(remainingHotFiles, sizeLimit,
deleteFromDb: !useColdStorage, dbContext: dbContext,
ct: linkedToken);
@@ -183,15 +188,18 @@ public class MainFileCleanupService : IHostedService
if (useColdStorage)
{
DirectoryInfo dirColdStorage = new(coldStorageDir);
_logger.LogInformation("File Cleanup Task gathering cold storage files");
var allFilesInColdStorageDir = dirColdStorage.GetFiles("*", SearchOption.AllDirectories).ToList();
var coldStorageRetention = _configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.ColdStorageUnusedFileRetentionPeriodInDays), 60);
var coldStorageSize = _configuration.GetValueOrDefault<double>(nameof(StaticFilesServerConfiguration.ColdStorageSizeHardLimitInGiB), -1);
// clean up cold storage
_logger.LogInformation("File Cleanup Task cleaning up outdated cold storage files");
var remainingColdFiles = await CleanUpOutdatedFiles(coldStorageDir, allFilesInColdStorageDir, coldStorageRetention, forcedDeletionAfterHours: -1,
deleteFromDb: true, dbContext: dbContext,
ct: linkedToken).ConfigureAwait(false);
_logger.LogInformation("File Cleanup Task cleaning up cold storage file beyond size limit");
var finalRemainingColdFiles = CleanUpFilesBeyondSizeLimit(remainingColdFiles, coldStorageSize,
deleteFromDb: true, dbContext: dbContext,
ct: linkedToken);

5
MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardClientReadyMessageService.cs
View File

@@ -22,9 +22,7 @@ public class ShardClientReadyMessageService : IClientReadyMessageService
_httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("MareSynchronosServer", "1.0.0.0"));
}
public void SendDownloadReady(string uid, Guid requestId)
{
_ = Task.Run(async () =>
public async Task SendDownloadReady(string uid, Guid requestId)
{
var mainUrl = _configurationService.GetValue<Uri>(nameof(StaticFilesServerConfiguration.MainFileServerAddress));
var path = MareFiles.MainSendReadyFullPath(mainUrl, uid, requestId);
@@ -43,6 +41,5 @@ public class ShardClientReadyMessageService : IClientReadyMessageService
{
_logger.LogError(ex, "Failure to send for {uid}:{requestId}", uid, requestId);
}
});
}
}

9
MareSynchronosServer/MareSynchronosStaticFilesServer/Startup.cs
View File

@@ -106,6 +106,15 @@ public class Startup
}).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false);
}, mareConfig.GetValue(nameof(MareConfigurationBase.DbContextPoolSize), 1024));
services.AddDbContextFactory<MareDbContext>(options =>
{
options.UseNpgsql(Configuration.GetConnectionString("DefaultConnection"), builder =>
{
builder.MigrationsHistoryTable("_efmigrationshistory", "public");
builder.MigrationsAssembly("MareSynchronosShared");
}).UseSnakeCaseNamingConvention();
options.EnableThreadSafetyChecks(false);
});
var signalRServiceBuilder = services.AddSignalR(hubOptions =>
{