friendlynamazu/server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

let file shards register against main or so

Browse Source
This commit is contained in:
Stanley Dimant
2024-11-16 23:02:04 +01:00
parent 532c123927
commit 3a9f93b157
12 changed files with 457 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/MainController.cs
View File

@@ -1,4 +1,5 @@
using MareSynchronos.API.Routes;
using MareSynchronosShared.Utils.Configuration;
using MareSynchronosStaticFilesServer.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
@@ -6,20 +7,60 @@ using Microsoft.AspNetCore.Mvc;
namespace MareSynchronosStaticFilesServer.Controllers;
[Route(MareFiles.Main)]
[Authorize(Policy = "Internal")]
public class MainController : ControllerBase
{
private readonly IClientReadyMessageService _messageService;
private readonly MainServerShardRegistrationService _shardRegistrationService;
public MainController(ILogger<MainController> logger, IClientReadyMessageService mareHub) : base(logger)
public MainController(ILogger<MainController> logger, IClientReadyMessageService mareHub,
MainServerShardRegistrationService shardRegistrationService) : base(logger)
{
_messageService = mareHub;
_shardRegistrationService = shardRegistrationService;
}
[HttpGet(MareFiles.Main_SendReady)]
[Authorize(Policy = "Internal")]
public async Task<IActionResult> SendReadyToClients(string uid, Guid requestId)
{
await _messageService.SendDownloadReady(uid, requestId).ConfigureAwait(false);
return Ok();
}
[HttpPost("shardRegister")]
public IActionResult RegisterShard([FromBody] ShardConfiguration shardConfiguration)
{
try
{
_shardRegistrationService.RegisterShard(MareUser, shardConfiguration);
return Ok();
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Shard could not be registered {shard}", MareUser);
return BadRequest();
}
}
[HttpPost("shardUnregister")]
public IActionResult UnregisterShard()
{
_shardRegistrationService.UnregisterShard(MareUser);
return Ok();
}
[HttpPost("shardHeartbeat")]
public IActionResult ShardHeartbeat()
{
try
{
_shardRegistrationService.ShardHeartbeat(MareUser);
return Ok();
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Shard not registered: {shard}", MareUser);
return BadRequest();
}
}
}

37
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs
View File

@@ -31,11 +31,13 @@ public class ServerFilesController : ControllerBase
private readonly IHubContext<MareHub> _hubContext;
private readonly IDbContextFactory<MareDbContext> _mareDbContext;
private readonly MareMetrics _metricsClient;
private readonly MainServerShardRegistrationService _shardRegistrationService;
public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider,
IConfigurationService<StaticFilesServerConfiguration> configuration,
IHubContext<MareHub> hubContext,
IDbContextFactory<MareDbContext> mareDbContext, MareMetrics metricsClient) : base(logger)
IDbContextFactory<MareDbContext> mareDbContext, MareMetrics metricsClient,
MainServerShardRegistrationService shardRegistrationService) : base(logger)
{
_basePath = configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.UseColdStorage), false)
? configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.ColdStorageDirectory))
@@ -45,6 +47,7 @@ public class ServerFilesController : ControllerBase
_hubContext = hubContext;
_mareDbContext = mareDbContext;
_metricsClient = metricsClient;
_shardRegistrationService = shardRegistrationService;
}
[HttpPost(MareFiles.ServerFiles_DeleteAll)]
@@ -85,7 +88,7 @@ public class ServerFilesController : ControllerBase
.Select(k => new { k.Hash, k.Size, k.RawSize })
.ToListAsync().ConfigureAwait(false);
var allFileShards = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
var allFileShards = _shardRegistrationService.GetConfigurationsByContinent(Continent);
foreach (var file in cacheFile)
{
@@ -94,25 +97,12 @@ public class ServerFilesController : ControllerBase
if (forbiddenFile == null)
{
List<CdnShardConfiguration> selectedShards = new();
var matchingShards = allFileShards.Where(f => new Regex(f.FileMatch).IsMatch(file.Hash)).ToList();
if (string.Equals(Continent, "*", StringComparison.Ordinal))
{
selectedShards = matchingShards;
}
else
{
selectedShards = matchingShards.Where(c => c.Continents.Contains(Continent, StringComparer.OrdinalIgnoreCase)).ToList();
if (!selectedShards.Any()) selectedShards = matchingShards;
}
var shard = matchingShards.SelectMany(g => g.RegionUris)
.OrderBy(g => Guid.NewGuid()).FirstOrDefault();
var shard = selectedShards
.OrderBy(s => !s.Continents.Any() ? 0 : 1)
.ThenBy(s => s.Continents.Contains("*", StringComparer.Ordinal) ? 0 : 1)
.ThenBy(g => Guid.NewGuid()).FirstOrDefault();
baseUrl = shard?.CdnFullUrl ?? _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
baseUrl = shard.Value ?? _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
}
response.Add(new DownloadFileDto
@@ -133,15 +123,8 @@ public class ServerFilesController : ControllerBase
[HttpGet(MareFiles.ServerFiles_DownloadServers)]
public async Task<IActionResult> GetDownloadServers()
{
var allFileShards = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()))
.DistinctBy(f => f.CdnFullUrl).ToList();
if (!allFileShards.Any())
{
return Ok(JsonSerializer.Serialize(new List<string> { _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl)).ToString() }));
}
var selectedShards = allFileShards.Where(c => c.Continents.Contains(Continent, StringComparer.OrdinalIgnoreCase)).ToList();
if (!selectedShards.Any()) selectedShards = allFileShards.Where(c => c.Continents.Contains("*", StringComparer.Ordinal)).ToList();
return Ok(JsonSerializer.Serialize(selectedShards.Select(t => t.CdnFullUrl.ToString())));
var allFileShards = _shardRegistrationService.GetConfigurationsByContinent(Continent);
return Ok(JsonSerializer.Serialize(allFileShards.SelectMany(t => t.RegionUris.Select(v => v.Value.ToString()))));
}
[HttpPost(MareFiles.ServerFiles_FilesSend)]