From 53e96d9318e7cc43ebe791ed4d138785483d8590 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Fri, 12 Jan 2024 13:10:14 +0100 Subject: [PATCH] add geoip service for file shard matching --- .../Controllers/JwtController.cs | 14 +- .../MareSynchronosServer.csproj | 1 + .../Services/GeoIPService.cs | 123 ++++++++++++++++++ .../MareSynchronosServer/Startup.cs | 2 + .../Utils/CdnShardConfiguration.cs | 3 +- .../Utils/MareClaimTypes.cs | 1 + .../Utils/MareConfigurationAuthBase.cs | 3 + .../Utils/ServerConfiguration.cs | 2 + .../Controllers/ControllerBase.cs | 1 + .../Controllers/ServerFilesController.cs | 22 +++- 10 files changed, 162 insertions(+), 10 deletions(-) create mode 100644 MareSynchronosServer/MareSynchronosServer/Services/GeoIPService.cs diff --git a/MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs b/MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs index 3489dff..a0cfa60 100644 --- a/MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs +++ b/MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs @@ -1,5 +1,6 @@ using MareSynchronos.API.Routes; using MareSynchronosServer.Authentication; +using MareSynchronosServer.Services; using MareSynchronosShared; using MareSynchronosShared.Data; using MareSynchronosShared.Models; @@ -26,17 +27,19 @@ public class JwtController : Controller private readonly IConfigurationService _configuration; private readonly MareDbContext _mareDbContext; private readonly IRedisDatabase _redis; + private readonly GeoIPService _geoIPProvider; private readonly SecretKeyAuthenticatorService _secretKeyAuthenticatorService; public JwtController(ILogger logger, IHttpContextAccessor accessor, MareDbContext mareDbContext, SecretKeyAuthenticatorService secretKeyAuthenticatorService, IConfigurationService configuration, - IRedisDatabase redisDb) + IRedisDatabase redisDb, GeoIPService geoIPProvider) { _logger = logger; _accessor = accessor; _redis = redisDb; + _geoIPProvider = geoIPProvider; _mareDbContext = mareDbContext; _secretKeyAuthenticatorService = secretKeyAuthenticatorService; _configuration = configuration; @@ -71,7 +74,7 @@ public class JwtController : Controller } _logger.LogInformation("RenewToken:SUCCESS:{id}:{ident}", uid, ident); - return CreateJwtFromId(uid, ident); + return await CreateJwtFromId(uid, ident); } catch (Exception ex) { @@ -123,7 +126,7 @@ public class JwtController : Controller } _logger.LogInformation("Authenticate:SUCCESS:{id}:{ident}", authResult.Uid, charaIdent); - return CreateJwtFromId(authResult.Uid, charaIdent); + return await CreateJwtFromId(authResult.Uid, charaIdent); } catch (Exception ex) { @@ -147,13 +150,14 @@ public class JwtController : Controller return handler.CreateJwtSecurityToken(token); } - private IActionResult CreateJwtFromId(string uid, string charaIdent) + private async Task CreateJwtFromId(string uid, string charaIdent) { var token = CreateJwt(new List() { new Claim(MareClaimTypes.Uid, uid), new Claim(MareClaimTypes.CharaIdent, charaIdent), - new Claim(MareClaimTypes.Expires, DateTime.UtcNow.AddHours(6).Ticks.ToString(CultureInfo.InvariantCulture)) + new Claim(MareClaimTypes.Expires, DateTime.UtcNow.AddHours(6).Ticks.ToString(CultureInfo.InvariantCulture)), + new Claim(MareClaimTypes.Continent, await _geoIPProvider.GetCountryFromIP(_accessor)) }); return Content(token.RawData); diff --git a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj index 38293d9..4cd4019 100644 --- a/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj +++ b/MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj @@ -28,6 +28,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/MareSynchronosServer/MareSynchronosServer/Services/GeoIPService.cs b/MareSynchronosServer/MareSynchronosServer/Services/GeoIPService.cs new file mode 100644 index 0000000..eaa34b3 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosServer/Services/GeoIPService.cs @@ -0,0 +1,123 @@ +using MareSynchronosShared; +using MareSynchronosShared.Services; +using MareSynchronosShared.Utils; +using MaxMind.GeoIP2; + +namespace MareSynchronosServer.Services; + +public class GeoIPService : IHostedService +{ + private readonly ILogger _logger; + private readonly IConfigurationService _mareConfiguration; + private bool _useGeoIP = false; + private string _countryFile = string.Empty; + private DatabaseReader? _dbReader; + private DateTime _dbLastWriteTime = DateTime.Now; + private CancellationTokenSource _fileWriteTimeCheckCts = new(); + private bool _processingReload = false; + + public GeoIPService(ILogger logger, + IConfigurationService mareConfiguration) + { + _logger = logger; + _mareConfiguration = mareConfiguration; + } + + public async Task GetCountryFromIP(IHttpContextAccessor httpContextAccessor) + { + if (!_useGeoIP) + { + return "*"; + } + + try + { + var ip = httpContextAccessor.GetIpAddress(); + + using CancellationTokenSource waitCts = new(); + waitCts.CancelAfter(TimeSpan.FromSeconds(5)); + while (_processingReload) await Task.Delay(100, waitCts.Token).ConfigureAwait(false); + + if (_dbReader.TryCountry(ip, out var response)) + { + return response.Continent.Code; + } + + return "*"; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error handling Geo IP country in request"); + return "*"; + } + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("GeoIP module starting update task"); + + var token = _fileWriteTimeCheckCts.Token; + _ = PeriodicReloadTask(token); + + return Task.CompletedTask; + } + + private async Task PeriodicReloadTask(CancellationToken token) + { + while (!token.IsCancellationRequested) + { + try + { + _processingReload = true; + + var useGeoIP = _mareConfiguration.GetValueOrDefault(nameof(ServerConfiguration.UseGeoIP), false); + var countryFile = _mareConfiguration.GetValueOrDefault(nameof(ServerConfiguration.GeoIPDbCountryFile), string.Empty); + var lastWriteTime = new FileInfo(countryFile).LastWriteTimeUtc; + if (useGeoIP && (!string.Equals(countryFile, _countryFile, StringComparison.OrdinalIgnoreCase) || lastWriteTime != _dbLastWriteTime)) + { + _countryFile = countryFile; + if (!File.Exists(_countryFile)) throw new FileNotFoundException($"Could not open GeoIP Country Database, path does not exist: {_countryFile}"); + _dbReader?.Dispose(); + _dbReader = null; + _dbReader = new DatabaseReader(_countryFile); + _dbLastWriteTime = lastWriteTime; + + _ = _dbReader.Country("8.8.8.8").Continent; + + _logger.LogInformation($"Loaded GeoIP country file from {_countryFile}"); + + if (_useGeoIP != useGeoIP) + { + _logger.LogInformation("GeoIP module is now enabled"); + _useGeoIP = useGeoIP; + } + } + + if (_useGeoIP != useGeoIP && !useGeoIP) + { + _logger.LogInformation("GeoIP module is now disabled"); + _useGeoIP = useGeoIP; + } + } + catch (Exception e) + { + _logger.LogWarning(e, "Error during periodic GeoIP module reload task, disabling GeoIP"); + _useGeoIP = false; + } + finally + { + _processingReload = false; + } + + await Task.Delay(TimeSpan.FromMinutes(1)).ConfigureAwait(false); + } + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _fileWriteTimeCheckCts.Cancel(); + _fileWriteTimeCheckCts.Dispose(); + _dbReader.Dispose(); + return Task.CompletedTask; + } +} diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index 553dc57..0146ed3 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -100,8 +100,10 @@ public class Startup if (isMainServer) { + services.AddSingleton(); services.AddSingleton(); services.AddHostedService(provider => provider.GetService()); + services.AddHostedService(provider => provider.GetService()); } } diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs index 4b9d694..4883f83 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs @@ -2,11 +2,12 @@ public class CdnShardConfiguration { + public List Continents { get; set; } = new List() { "*" }; public string FileMatch { get; set; } public Uri CdnFullUrl { get; set; } public override string ToString() { - return CdnFullUrl.ToString() + " == " + FileMatch; + return CdnFullUrl.ToString() + "[" + string.Join(',', Continents) + "] == " + FileMatch; } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs b/MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs index 13f6d1f..efb7adf 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs @@ -6,4 +6,5 @@ public static class MareClaimTypes public const string CharaIdent = "character_identification"; public const string Internal = "internal"; public const string Expires = "expiration_date"; + public const string Continent = "continent"; } diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs b/MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs index 966f956..749fbd6 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs @@ -10,6 +10,8 @@ public class MareConfigurationAuthBase : MareConfigurationBase public int TempBanDurationInMinutes { get; set; } = 5; [RemoteConfiguration] public List WhitelistedIps { get; set; } = new(); + [RemoteConfiguration] + public bool UseGeoIP { get; set; } = false; public override string ToString() { @@ -19,6 +21,7 @@ public class MareConfigurationAuthBase : MareConfigurationBase sb.AppendLine($"{nameof(TempBanDurationInMinutes)} => {TempBanDurationInMinutes}"); sb.AppendLine($"{nameof(Jwt)} => {Jwt}"); sb.AppendLine($"{nameof(WhitelistedIps)} => {string.Join(", ", WhitelistedIps)}"); + sb.AppendLine($"{nameof(UseGeoIP)} => {UseGeoIP}"); return sb.ToString(); } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs b/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs index 2b8d668..8f54d0c 100644 --- a/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs +++ b/MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs @@ -24,6 +24,7 @@ public class ServerConfiguration : MareConfigurationAuthBase [RemoteConfiguration] public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14; + public string GeoIPDbCountryFile { get; set; } = string.Empty; public int RedisPool { get; set; } = 50; @@ -40,6 +41,7 @@ public class ServerConfiguration : MareConfigurationAuthBase sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}"); sb.AppendLine($"{nameof(PurgeUnusedAccounts)} => {PurgeUnusedAccounts}"); sb.AppendLine($"{nameof(PurgeUnusedAccountsPeriodInDays)} => {PurgeUnusedAccountsPeriodInDays}"); + sb.AppendLine($"{nameof(GeoIPDbCountryFile)} => {GeoIPDbCountryFile}"); return sb.ToString(); } } \ No newline at end of file diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs index 4ccbd64..630a741 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs @@ -13,4 +13,5 @@ public class ControllerBase : Controller } protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value; + protected string Continent => HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, MareClaimTypes.Continent, StringComparison.Ordinal))?.Value ?? "*"; } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs index f0414af..94b4b68 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs @@ -35,7 +35,7 @@ public class ServerFilesController : ControllerBase public ServerFilesController(ILogger logger, CachedFileProvider cachedFileProvider, IConfigurationService configuration, - IHubContext hubContext, + IHubContext hubContext, MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger) { _basePath = configuration.GetValue(nameof(StaticFilesServerConfiguration.CacheDirectory)); @@ -78,14 +78,28 @@ public class ServerFilesController : ControllerBase var cacheFile = await _mareDbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false); - var shardConfig = new List(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List())); + var allFileShards = new List(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List())); foreach (var file in cacheFile) { var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase)); - var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash)); - var baseUrl = matchedShardConfig?.CdnFullUrl ?? _configuration.GetValue(nameof(StaticFilesServerConfiguration.CdnFullUrl)); + List selectedShards = new(); + var matchingShards = allFileShards.Where(f => new Regex(f.FileMatch).IsMatch(file.Hash)).ToList(); + + if (string.Equals(Continent, "*", StringComparison.Ordinal)) + { + selectedShards = matchingShards; + } + else + { + selectedShards = matchingShards.Where(c => c.Continents.Contains(Continent, StringComparer.OrdinalIgnoreCase)).ToList(); + if (!selectedShards.Any()) selectedShards = matchingShards; + } + + var shard = selectedShards.OrderBy(g => Guid.NewGuid()).FirstOrDefault(); + + var baseUrl = shard?.CdnFullUrl ?? _configuration.GetValue(nameof(StaticFilesServerConfiguration.CdnFullUrl)); response.Add(new DownloadFileDto {