From a1e82bcdb642365c77728ebeadb1165da9d69b4d Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Fri, 17 Nov 2023 02:10:45 +0100 Subject: [PATCH] census update --- Docker/run/compose/mare-standalone.yml | 2 + .../config/standalone/server-standalone.json | 3 +- MareAPI | 2 +- .../MareSynchronosServer/Hubs/MareHub.User.cs | 6 +- .../MareSynchronosServer/Hubs/MareHub.cs | 6 +- .../Services/MareCensus.cs | 161 ++++++++++++++++++ .../MareSynchronosServer/Startup.cs | 3 + 7 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 MareSynchronosServer/MareSynchronosServer/Services/MareCensus.cs diff --git a/Docker/run/compose/mare-standalone.yml b/Docker/run/compose/mare-standalone.yml index 1eb866c..d7f3f17 100644 --- a/Docker/run/compose/mare-standalone.yml +++ b/Docker/run/compose/mare-standalone.yml @@ -29,8 +29,10 @@ services: restart: on-failure ports: - 6000:6000/tcp + - 6050:6050/tcp environment: MareSynchronos__CdnFullUrl: "${DEV_MARE_CDNURL}" + MareSynchronos__XIVAPIKey: "${DEV_MARE_XIVAPIKEY}" DOTNET_USE_POLLING_FILE_WATCHER: 1 volumes: - ../config/standalone/server-standalone.json:/opt/MareSynchronosServer/appsettings.json diff --git a/Docker/run/config/standalone/server-standalone.json b/Docker/run/config/standalone/server-standalone.json index 1bc4aa5..72cc626 100644 --- a/Docker/run/config/standalone/server-standalone.json +++ b/Docker/run/config/standalone/server-standalone.json @@ -44,7 +44,8 @@ "MaxGroupUserCount": 100, "PurgeUnusedAccounts": false, "PurgeUnusedAccountsPeriodInDays": 14, - "ExpectedClientVersion": "0.8.0" + "ExpectedClientVersion": "0.9.0", + "XIVAPIKey": "" }, "AllowedHosts": "*", "Kestrel": { diff --git a/MareAPI b/MareAPI index edaa0dd..47ff523 160000 --- a/MareAPI +++ b/MareAPI @@ -1 +1 @@ -Subproject commit edaa0ddb5123b2eb78d694b844bb7ad6cc13192d +Subproject commit 47ff5235d697eda92eac1171d29c7a8ba25e7841 diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs index 8ab48c8..af101fa 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs @@ -152,7 +152,7 @@ public partial class MareHub } [Authorize(Policy = "Identified")] - public async Task> UserGetOnlinePairs() + public async Task> UserGetOnlinePairs(CensusDataDto? censusData) { _logger.LogCallInfo(); @@ -161,6 +161,8 @@ public partial class MareHub await SendOnlineToAllPairedUsers().ConfigureAwait(false); + _mareCensus.PublishStatistics(UserUID, censusData); + return pairs.Select(p => new OnlineUserIdentDto(new UserData(p.Key), p.Value)).ToList(); } @@ -278,6 +280,8 @@ public partial class MareHub await Clients.Users(recipientUids).Client_UserReceiveCharacterData(new OnlineUserCharaDataDto(new UserData(UserUID), dto.CharaData)).ConfigureAwait(false); + _mareCensus.PublishStatistics(UserUID, dto.CensusDataDto); + _mareMetrics.IncCounter(MetricsAPI.CounterUserPushData); _mareMetrics.IncCounter(MetricsAPI.CounterUserPushDataTo, recipientUids.Count); } diff --git a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs index 2c188c7..1c90849 100644 --- a/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs +++ b/MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs @@ -30,6 +30,7 @@ public partial class MareHub : Hub, IMareHub private readonly int _maxGroupUserCount; private readonly IRedisDatabase _redis; private readonly OnlineSyncedPairCacheService _onlineSyncedPairCacheService; + private readonly MareCensus _mareCensus; private readonly Uri _fileServerAddress; private readonly Version _expectedClientVersion; private readonly Lazy _dbContextLazy; @@ -38,7 +39,7 @@ public partial class MareHub : Hub, IMareHub public MareHub(MareMetrics mareMetrics, IDbContextFactory mareDbContextFactory, ILogger logger, SystemInfoService systemInfoService, IConfigurationService configuration, IHttpContextAccessor contextAccessor, - IRedisDatabase redisDb, OnlineSyncedPairCacheService onlineSyncedPairCacheService) + IRedisDatabase redisDb, OnlineSyncedPairCacheService onlineSyncedPairCacheService, MareCensus mareCensus) { _mareMetrics = mareMetrics; _systemInfoService = systemInfoService; @@ -51,6 +52,7 @@ public partial class MareHub : Hub, IMareHub _contextAccessor = contextAccessor; _redis = redisDb; _onlineSyncedPairCacheService = onlineSyncedPairCacheService; + _mareCensus = mareCensus; _logger = new MareHubLogger(this, logger); _dbContextLazy = new Lazy(() => mareDbContextFactory.CreateDbContext()); } @@ -158,6 +160,8 @@ public partial class MareHub : Hub, IMareHub await RemoveUserFromRedis().ConfigureAwait(false); + _mareCensus.ClearStatistics(UserUID); + await SendOfflineToAllPairedUsers().ConfigureAwait(false); DbContext.RemoveRange(DbContext.Files.Where(f => !f.Uploaded && f.UploaderUID == UserUID)); diff --git a/MareSynchronosServer/MareSynchronosServer/Services/MareCensus.cs b/MareSynchronosServer/MareSynchronosServer/Services/MareCensus.cs new file mode 100644 index 0000000..d40783c --- /dev/null +++ b/MareSynchronosServer/MareSynchronosServer/Services/MareCensus.cs @@ -0,0 +1,161 @@ +using MareSynchronos.API.Dto.User; +using Prometheus; +using System.Collections.Concurrent; +using System.Globalization; +using System.Text.Json; + +namespace MareSynchronosServer.Services; + +public class MareCensus : IHostedService +{ + private record CensusEntry(ushort WorldId, short Race, short Subrace, short Gender) + { + public static CensusEntry FromDto(CensusDataDto dto) + { + return new CensusEntry(dto.WorldId, dto.RaceId, dto.TribeId, dto.Gender); + } + } + + private readonly ConcurrentDictionary _censusEntries = new(StringComparer.Ordinal); + private readonly Dictionary _dcs = new(); + private readonly Dictionary _gender = new(); + private readonly ILogger _logger; + private readonly Dictionary _races = new(); + private readonly Dictionary _tribes = new(); + private readonly Dictionary _worlds = new(); + private readonly string _xivApiKey; + private Gauge? _gauge; + + public MareCensus(ILogger logger, string xivApiKey) + { + _logger = logger; + _xivApiKey = xivApiKey; + } + + private bool Initialized => _gauge != null; + + public void ClearStatistics(string uid) + { + if (!Initialized) return; + + if (_censusEntries.Remove(uid, out var censusEntry)) + { + ModifyGauge(censusEntry, increase: false); + } + } + + public void PublishStatistics(string uid, CensusDataDto? censusDataDto) + { + if (!Initialized || censusDataDto == null) return; + + var newEntry = CensusEntry.FromDto(censusDataDto); + + if (_censusEntries.TryGetValue(uid, out var entry)) + { + if (entry != newEntry) + { + ModifyGauge(entry, increase: false); + ModifyGauge(newEntry, increase: true); + _censusEntries[uid] = newEntry; + } + } + else + { + _censusEntries[uid] = newEntry; + ModifyGauge(newEntry, increase: true); + } + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(_xivApiKey)) return; + + _logger.LogInformation("Loading XIVAPI data"); + + using HttpClient client = new HttpClient(); + + Dictionary worldDcs = new(); + var dcs = await client.GetStringAsync("https://xivapi.com/worlddcgrouptype?private_key" + _xivApiKey, cancellationToken).ConfigureAwait(false); + using var dcsJson = JsonSerializer.Deserialize(dcs).GetProperty("Results").EnumerateArray(); + + foreach (var dcValue in dcsJson) + { + var id = dcValue.GetProperty("ID").GetInt16(); + var name = dcValue.GetProperty("Name").GetString(); + _dcs[id] = name; + _logger.LogInformation("DC: ID: {id}, Name: {name}", id, name); + var dcData = await client.GetStringAsync("https://xivapi.com/worlddcgrouptype/" + id.ToString(CultureInfo.InvariantCulture) + "?private_key=" + _xivApiKey, cancellationToken).ConfigureAwait(false); + if (JsonSerializer.Deserialize(dcData).TryGetProperty("GameContentLinks", out var gameContentLinks)) + { + if (gameContentLinks.ValueKind == JsonValueKind.Object && gameContentLinks.TryGetProperty("World", out var worldProp)) + { + using var json = worldProp.GetProperty("DataCenter").EnumerateArray(); + foreach (var world in json) + { + worldDcs[(ushort)world.GetInt32()] = id; + } + } + } + + await Task.Delay(TimeSpan.FromMilliseconds(100)).ConfigureAwait(false); + } + + var worlds = await client.GetStringAsync("https://xivapi.com/world?limit=1000&private_key" + _xivApiKey, cancellationToken).ConfigureAwait(false); + using var worldJson = JsonSerializer.Deserialize(worlds).GetProperty("Results").EnumerateArray(); + foreach (var worldValue in worldJson) + { + var id = (ushort)worldValue.GetProperty("ID").GetInt32(); + var name = worldValue.GetProperty("Name").GetString(); + if (worldDcs.TryGetValue(id, out var dc)) + { + _worlds[(ushort)id] = (name, dc); + _logger.LogInformation("World: ID: {id}, Name: {name}, DC: {dc}", id, name, dc); + } + } + + var races = await client.GetStringAsync("https://xivapi.com/race?private_key" + _xivApiKey, cancellationToken).ConfigureAwait(false); + using var racesJson = JsonSerializer.Deserialize(races).GetProperty("Results").EnumerateArray(); + foreach (var racesValue in racesJson) + { + var id = racesValue.GetProperty("ID").GetInt16(); + var name = racesValue.GetProperty("Name").GetString(); + _races[id] = name; + _logger.LogInformation("Race: ID: {id}, Name: {name}", id, name); + } + + var tribe = await client.GetStringAsync("https://xivapi.com/tribe?private_key=" + _xivApiKey, cancellationToken).ConfigureAwait(false); + using var tribeJson = JsonSerializer.Deserialize(tribe).GetProperty("Results").EnumerateArray(); + foreach (var tribeValue in tribeJson) + { + var id = tribeValue.GetProperty("ID").GetInt16(); + var name = tribeValue.GetProperty("Name").GetString(); + _tribes[id] = name; + _logger.LogInformation("Tribe: ID: {id}, Name: {name}", id, name); + } + + _gender[0] = "Male"; + _gender[1] = "Female"; + + _gauge = Metrics.CreateGauge("mare_census", "mare informational census data", new[] { "dc", "world", "gender", "race", "subrace" }); + } + + public Task StopAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + private void ModifyGauge(CensusEntry censusEntry, bool increase) + { + var subraceSuccess = _tribes.TryGetValue(censusEntry.Subrace, out var subrace); + var raceSuccess = _races.TryGetValue(censusEntry.Race, out var race); + var worldSuccess = _worlds.TryGetValue(censusEntry.WorldId, out var world); + var genderSuccess = _gender.TryGetValue(censusEntry.Gender, out var gender); + if (subraceSuccess && raceSuccess && worldSuccess && genderSuccess && _dcs.TryGetValue(world.Item2, out var dc)) + { + if (increase) + _gauge.WithLabels(dc, world.Item1, gender, race, subrace).Inc(); + else + _gauge.WithLabels(dc, world.Item1, gender, race, subrace).Dec(); + } + } +} diff --git a/MareSynchronosServer/MareSynchronosServer/Startup.cs b/MareSynchronosServer/MareSynchronosServer/Startup.cs index bc5c795..553dc57 100644 --- a/MareSynchronosServer/MareSynchronosServer/Startup.cs +++ b/MareSynchronosServer/MareSynchronosServer/Startup.cs @@ -95,6 +95,9 @@ public class Startup // configure services based on main server status ConfigureServicesBasedOnShardType(services, mareConfig, isMainServer); + services.AddSingleton(s => new MareCensus(s.GetRequiredService>(), mareConfig.GetValue("XIVAPIKey", string.Empty))); + services.AddHostedService(p => p.GetRequiredService()); + if (isMainServer) { services.AddSingleton();