add horizontal file sharding based on filename matches

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs

@@ -6,6 +6,7 @@ using MareSynchronos.API;
 using MareSynchronosServer.Utils;
 using MareSynchronosShared.Models;
 using MareSynchronosShared.Protos;
+using MareSynchronosShared.Utils;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.EntityFrameworkCore;
 
@@ -52,10 +53,14 @@ public partial class MareHub
 
         var cacheFile = await _dbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false);
 
+        var shardConfig = _configurationService.GetValueOrDefault(nameof(ServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>());
+
         foreach (var file in cacheFile)
         {
             var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
-            var downloadFile = allFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
+
+            var matchedShardConfig = shardConfig.Find(f => f.FileMatchRegex.Match(file.Hash).Success);
+            var baseUrl = matchedShardConfig?.CdnFullUrl ?? _mainCdnFullUrl;
 
             response.Add(new DownloadFileDto
             {
@@ -64,7 +69,7 @@ public partial class MareHub
                 IsForbidden = forbiddenFile != null,
                 Hash = file.Hash,
                 Size = file.Size,
-                Url = new Uri(_cdnFullUri, file.Hash.ToUpperInvariant()).ToString()
+                Url = new Uri(baseUrl, file.Hash.ToUpperInvariant()).ToString()
             });
         }
 

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs

@@ -23,11 +23,12 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
     private readonly IClientIdentificationService _clientIdentService;
     private readonly MareHubLogger _logger;
     private readonly MareDbContext _dbContext;
-    private readonly Uri _cdnFullUri;
+    private readonly Uri _mainCdnFullUrl;
     private readonly string _shardName;
     private readonly int _maxExistingGroupsByUser;
     private readonly int _maxJoinedGroupsByUser;
     private readonly int _maxGroupUserCount;
+    private IConfigurationService<ServerConfiguration> _configurationService;
 
     public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
         MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService,
@@ -37,7 +38,8 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
         _mareMetrics = mareMetrics;
         _fileServiceClient = fileServiceClient;
         _systemInfoService = systemInfoService;
-        _cdnFullUri = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
+        _configurationService = configuration;
+        _mainCdnFullUrl = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
         _shardName = configuration.GetValue<string>(nameof(ServerConfiguration.ShardName));
         _maxExistingGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3);
         _maxJoinedGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxJoinedGroupsByUser), 6);

MareSynchronosServer/MareSynchronosServer/Program.cs

@@ -30,6 +30,8 @@ public class Program
                 context.RemoveRange(unfinishedRegistrations);
                 context.RemoveRange(looseFiles);
                 context.SaveChanges();
+
+                logger.LogInformation(options.ToString());
             }
 
             var metrics = services.GetRequiredService<MareMetrics>();

MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceClient.cs

@@ -4,11 +4,13 @@ using MareSynchronosShared.Utils;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using System.Collections;
 using System.Collections.Concurrent;
 using System.Globalization;
 using System.Reflection;
 using System.Text;
 using System.Text.Json;
+using System.Text.RegularExpressions;
 using static MareSynchronosShared.Protos.ConfigurationService;
 
 namespace MareSynchronosShared.Services;
@@ -75,9 +77,17 @@ public class MareConfigurationServiceClient<T> : IHostedService, IConfigurationS
         foreach (var prop in props)
         {
             var isRemote = prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any();
-            var mi = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType);
-            var val = mi.Invoke(this, new[] { prop.Name });
-            var value = isRemote ? val : prop.GetValue(_config);
+            var getValueMethod = GetType().GetMethod(nameof(GetValue)).MakeGenericMethod(prop.PropertyType);
+            var value = isRemote ? getValueMethod.Invoke(this, new[] { prop.Name }) : prop.GetValue(_config);
+            if (typeof(IEnumerable).IsAssignableFrom(prop.PropertyType) && !typeof(string).IsAssignableFrom(prop.PropertyType))
+            {
+                var enumVal = (IEnumerable)value;
+                value = string.Empty;
+                foreach (var listVal in enumVal)
+                {
+                    value += listVal.ToString() + ", ";
+                }
+            }
             sb.AppendLine($"{prop.Name} (IsRemote: {isRemote}) => {value}");
         }
         return sb.ToString();

MareSynchronosServer/MareSynchronosShared/Services/MareConfigurationServiceServer.cs

@@ -32,6 +32,9 @@ public class MareConfigurationServiceServer<T> : IConfigurationService<T> where
         {
             sb.AppendLine($"{prop.Name} (IsRemote: {prop.GetCustomAttributes(typeof(RemoteConfigurationAttribute), true).Any()}) => {prop.GetValue(_config)}");
         }
+
+        sb.AppendLine(_config.ToString());
+
         return sb.ToString();
     }
 }

MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs

@@ -0,0 +1,17 @@
+using System.Text.Json.Serialization;
+using System.Text.RegularExpressions;
+
+namespace MareSynchronosShared.Utils;
+
+public class CdnShardConfiguration
+{
+    public string FileMatch { get; set; }
+    [JsonIgnore]
+    public Regex FileMatchRegex => new Regex(FileMatch);
+    public Uri CdnFullUrl { get; set; }
+
+    public override string ToString()
+    {
+        return CdnFullUrl.ToString() + " == " + FileMatch;
+    }
+}

MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs

@@ -20,6 +20,8 @@ public class ServerConfiguration : MareConfigurationAuthBase
     public bool PurgeUnusedAccounts { get; set; } = false;
     [RemoteConfiguration]
     public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
+    [RemoteConfiguration]
+    public List<CdnShardConfiguration> CdnShardConfiguration { get; set; } = new();
 
     public override string ToString()
     {
@@ -27,6 +29,7 @@ public class ServerConfiguration : MareConfigurationAuthBase
         sb.AppendLine(base.ToString());
         sb.AppendLine($"{nameof(MainServerGrpcAddress)} => {MainServerGrpcAddress}");
         sb.AppendLine($"{nameof(CdnFullUrl)} => {CdnFullUrl}");
+        sb.AppendLine($"{nameof(CdnShardConfiguration)} => {string.Join(", ", CdnShardConfiguration.Select(c => c.ToString()))}");
         sb.AppendLine($"{nameof(StaticFileServiceAddress)} => {StaticFileServiceAddress}");
         sb.AppendLine($"{nameof(RedisConnectionString)} => {RedisConnectionString}");
         sb.AppendLine($"{nameof(MaxExistingGroupsByUser)} => {MaxExistingGroupsByUser}");