friendlynamazu/server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add geoip service for file shard matching

Browse Source
This commit is contained in:
rootdarkarchon
2024-01-12 13:10:14 +01:00
parent c6d3ac1c51
commit 53e96d9318
10 changed files with 162 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
MareSynchronosServer/MareSynchronosServer/Controllers/JwtController.cs
View File

@@ -1,5 +1,6 @@
using MareSynchronos.API.Routes;
using MareSynchronosServer.Authentication;
using MareSynchronosServer.Services;
using MareSynchronosShared;
using MareSynchronosShared.Data;
using MareSynchronosShared.Models;
@@ -26,17 +27,19 @@ public class JwtController : Controller
private readonly IConfigurationService<MareConfigurationAuthBase> _configuration;
private readonly MareDbContext _mareDbContext;
private readonly IRedisDatabase _redis;
private readonly GeoIPService _geoIPProvider;
private readonly SecretKeyAuthenticatorService _secretKeyAuthenticatorService;
public JwtController(ILogger<JwtController> logger,
IHttpContextAccessor accessor, MareDbContext mareDbContext,
SecretKeyAuthenticatorService secretKeyAuthenticatorService,
IConfigurationService<MareConfigurationAuthBase> configuration,
IRedisDatabase redisDb)
IRedisDatabase redisDb, GeoIPService geoIPProvider)
{
_logger = logger;
_accessor = accessor;
_redis = redisDb;
_geoIPProvider = geoIPProvider;
_mareDbContext = mareDbContext;
_secretKeyAuthenticatorService = secretKeyAuthenticatorService;
_configuration = configuration;
@@ -71,7 +74,7 @@ public class JwtController : Controller
}
_logger.LogInformation("RenewToken:SUCCESS:{id}:{ident}", uid, ident);
return CreateJwtFromId(uid, ident);
return await CreateJwtFromId(uid, ident);
}
catch (Exception ex)
{
@@ -123,7 +126,7 @@ public class JwtController : Controller
}
_logger.LogInformation("Authenticate:SUCCESS:{id}:{ident}", authResult.Uid, charaIdent);
return CreateJwtFromId(authResult.Uid, charaIdent);
return await CreateJwtFromId(authResult.Uid, charaIdent);
}
catch (Exception ex)
{
@@ -147,13 +150,14 @@ public class JwtController : Controller
return handler.CreateJwtSecurityToken(token);
}
private IActionResult CreateJwtFromId(string uid, string charaIdent)
private async Task<IActionResult> CreateJwtFromId(string uid, string charaIdent)
{
var token = CreateJwt(new List<Claim>()
{
new Claim(MareClaimTypes.Uid, uid),
new Claim(MareClaimTypes.CharaIdent, charaIdent),
new Claim(MareClaimTypes.Expires, DateTime.UtcNow.AddHours(6).Ticks.ToString(CultureInfo.InvariantCulture))
new Claim(MareClaimTypes.Expires, DateTime.UtcNow.AddHours(6).Ticks.ToString(CultureInfo.InvariantCulture)),
new Claim(MareClaimTypes.Continent, await _geoIPProvider.GetCountryFromIP(_accessor))
});
return Content(token.RawData);

1
MareSynchronosServer/MareSynchronosServer/MareSynchronosServer.csproj
View File

@@ -28,6 +28,7 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="lz4net" Version="1.0.15.93" />
<PackageReference Include="MaxMind.GeoIP2" Version="5.2.0" />
<PackageReference Include="Meziantou.Analyzer" Version="2.0.93">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

123
MareSynchronosServer/MareSynchronosServer/Services/GeoIPService.cs Normal file
View File

@@ -0,0 +1,123 @@
using MareSynchronosShared;
using MareSynchronosShared.Services;
using MareSynchronosShared.Utils;
using MaxMind.GeoIP2;
namespace MareSynchronosServer.Services;
public class GeoIPService : IHostedService
{
private readonly ILogger<GeoIPService> _logger;
private readonly IConfigurationService<ServerConfiguration> _mareConfiguration;
private bool _useGeoIP = false;
private string _countryFile = string.Empty;
private DatabaseReader? _dbReader;
private DateTime _dbLastWriteTime = DateTime.Now;
private CancellationTokenSource _fileWriteTimeCheckCts = new();
private bool _processingReload = false;
public GeoIPService(ILogger<GeoIPService> logger,
IConfigurationService<ServerConfiguration> mareConfiguration)
{
_logger = logger;
_mareConfiguration = mareConfiguration;
}
public async Task<string> GetCountryFromIP(IHttpContextAccessor httpContextAccessor)
{
if (!_useGeoIP)
{
return "*";
}
try
{
var ip = httpContextAccessor.GetIpAddress();
using CancellationTokenSource waitCts = new();
waitCts.CancelAfter(TimeSpan.FromSeconds(5));
while (_processingReload) await Task.Delay(100, waitCts.Token).ConfigureAwait(false);
if (_dbReader.TryCountry(ip, out var response))
{
return response.Continent.Code;
}
return "*";
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error handling Geo IP country in request");
return "*";
}
}
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("GeoIP module starting update task");
var token = _fileWriteTimeCheckCts.Token;
_ = PeriodicReloadTask(token);
return Task.CompletedTask;
}
private async Task PeriodicReloadTask(CancellationToken token)
{
while (!token.IsCancellationRequested)
{
try
{
_processingReload = true;
var useGeoIP = _mareConfiguration.GetValueOrDefault(nameof(ServerConfiguration.UseGeoIP), false);
var countryFile = _mareConfiguration.GetValueOrDefault(nameof(ServerConfiguration.GeoIPDbCountryFile), string.Empty);
var lastWriteTime = new FileInfo(countryFile).LastWriteTimeUtc;
if (useGeoIP && (!string.Equals(countryFile, _countryFile, StringComparison.OrdinalIgnoreCase) || lastWriteTime != _dbLastWriteTime))
{
_countryFile = countryFile;
if (!File.Exists(_countryFile)) throw new FileNotFoundException($"Could not open GeoIP Country Database, path does not exist: {_countryFile}");
_dbReader?.Dispose();
_dbReader = null;
_dbReader = new DatabaseReader(_countryFile);
_dbLastWriteTime = lastWriteTime;
_ = _dbReader.Country("8.8.8.8").Continent;
_logger.LogInformation($"Loaded GeoIP country file from {_countryFile}");
if (_useGeoIP != useGeoIP)
{
_logger.LogInformation("GeoIP module is now enabled");
_useGeoIP = useGeoIP;
}
}
if (_useGeoIP != useGeoIP && !useGeoIP)
{
_logger.LogInformation("GeoIP module is now disabled");
_useGeoIP = useGeoIP;
}
}
catch (Exception e)
{
_logger.LogWarning(e, "Error during periodic GeoIP module reload task, disabling GeoIP");
_useGeoIP = false;
}
finally
{
_processingReload = false;
}
await Task.Delay(TimeSpan.FromMinutes(1)).ConfigureAwait(false);
}
}
public Task StopAsync(CancellationToken cancellationToken)
{
_fileWriteTimeCheckCts.Cancel();
_fileWriteTimeCheckCts.Dispose();
_dbReader.Dispose();
return Task.CompletedTask;
}
}

2
MareSynchronosServer/MareSynchronosServer/Startup.cs
View File

@@ -100,8 +100,10 @@ public class Startup
if (isMainServer)
{
services.AddSingleton<GeoIPService>();
services.AddSingleton<UserCleanupService>();
services.AddHostedService(provider => provider.GetService<UserCleanupService>());
services.AddHostedService(provider => provider.GetService<GeoIPService>());
}
}

3
MareSynchronosServer/MareSynchronosShared/Utils/CdnShardConfiguration.cs
View File

@@ -2,11 +2,12 @@
public class CdnShardConfiguration
{
public List<string> Continents { get; set; } = new List<string>() { "*" };
public string FileMatch { get; set; }
public Uri CdnFullUrl { get; set; }
public override string ToString()
{
return CdnFullUrl.ToString() + " == " + FileMatch;
return CdnFullUrl.ToString() + "[" + string.Join(',', Continents) + "] == " + FileMatch;
}
}

1
MareSynchronosServer/MareSynchronosShared/Utils/MareClaimTypes.cs
View File

@@ -6,4 +6,5 @@ public static class MareClaimTypes
public const string CharaIdent = "character_identification";
public const string Internal = "internal";
public const string Expires = "expiration_date";
public const string Continent = "continent";
}

3
MareSynchronosServer/MareSynchronosShared/Utils/MareConfigurationAuthBase.cs
View File

@@ -10,6 +10,8 @@ public class MareConfigurationAuthBase : MareConfigurationBase
public int TempBanDurationInMinutes { get; set; } = 5;
[RemoteConfiguration]
public List<string> WhitelistedIps { get; set; } = new();
[RemoteConfiguration]
public bool UseGeoIP { get; set; } = false;
public override string ToString()
{
@@ -19,6 +21,7 @@ public class MareConfigurationAuthBase : MareConfigurationBase
sb.AppendLine($"{nameof(TempBanDurationInMinutes)} => {TempBanDurationInMinutes}");
sb.AppendLine($"{nameof(Jwt)} => {Jwt}");
sb.AppendLine($"{nameof(WhitelistedIps)} => {string.Join(", ", WhitelistedIps)}");
sb.AppendLine($"{nameof(UseGeoIP)} => {UseGeoIP}");
return sb.ToString();
}
}

2
MareSynchronosServer/MareSynchronosShared/Utils/ServerConfiguration.cs
View File

@@ -24,6 +24,7 @@ public class ServerConfiguration : MareConfigurationAuthBase
[RemoteConfiguration]
public int PurgeUnusedAccountsPeriodInDays { get; set; } = 14;
public string GeoIPDbCountryFile { get; set; } = string.Empty;
public int RedisPool { get; set; } = 50;
@@ -40,6 +41,7 @@ public class ServerConfiguration : MareConfigurationAuthBase
sb.AppendLine($"{nameof(MaxGroupUserCount)} => {MaxGroupUserCount}");
sb.AppendLine($"{nameof(PurgeUnusedAccounts)} => {PurgeUnusedAccounts}");
sb.AppendLine($"{nameof(PurgeUnusedAccountsPeriodInDays)} => {PurgeUnusedAccountsPeriodInDays}");
sb.AppendLine($"{nameof(GeoIPDbCountryFile)} => {GeoIPDbCountryFile}");
return sb.ToString();
}
}

1
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ControllerBase.cs
View File

@@ -13,4 +13,5 @@ public class ControllerBase : Controller
}
protected string MareUser => HttpContext.User.Claims.First(f => string.Equals(f.Type, MareClaimTypes.Uid, StringComparison.Ordinal)).Value;
protected string Continent => HttpContext.User.Claims.FirstOrDefault(f => string.Equals(f.Type, MareClaimTypes.Continent, StringComparison.Ordinal))?.Value ?? "*";
}

22
MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs
View File

@@ -35,7 +35,7 @@ public class ServerFilesController : ControllerBase
public ServerFilesController(ILogger<ServerFilesController> logger, CachedFileProvider cachedFileProvider,
IConfigurationService<StaticFilesServerConfiguration> configuration,
IHubContext<MareSynchronosServer.Hubs.MareHub> hubContext,
IHubContext<MareHub> hubContext,
MareDbContext mareDbContext, MareMetrics metricsClient) : base(logger)
{
_basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
@@ -78,14 +78,28 @@ public class ServerFilesController : ControllerBase
var cacheFile = await _mareDbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false);
var shardConfig = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
var allFileShards = new List<CdnShardConfiguration>(_configuration.GetValueOrDefault(nameof(StaticFilesServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
foreach (var file in cacheFile)
{
var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash));
var baseUrl = matchedShardConfig?.CdnFullUrl ?? _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
List<CdnShardConfiguration> selectedShards = new();
var matchingShards = allFileShards.Where(f => new Regex(f.FileMatch).IsMatch(file.Hash)).ToList();
if (string.Equals(Continent, "*", StringComparison.Ordinal))
{
selectedShards = matchingShards;
}
else
{
selectedShards = matchingShards.Where(c => c.Continents.Contains(Continent, StringComparer.OrdinalIgnoreCase)).ToList();
if (!selectedShards.Any()) selectedShards = matchingShards;
}
var shard = selectedShards.OrderBy(g => Guid.NewGuid()).FirstOrDefault();
var baseUrl = shard?.CdnFullUrl ?? _configuration.GetValue<Uri>(nameof(StaticFilesServerConfiguration.CdnFullUrl));
response.Add(new DownloadFileDto
{