[Draft] Update 0.8 (#25)

* get rid of file handling through grpc and signalr

* fix upload on controller

* adapt usersetpairpermissions

* send user perms

* server-side fixes

* rework file upload

* adjust log level to debug in docker standalone json

* update dependencies

---------

Co-authored-by: rootdarkarchon <root.darkarchon@outlook.com>

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Files.cs

@@ -1,253 +0,0 @@
-using System.Security.Cryptography;
-using System.Text.RegularExpressions;
-using Google.Protobuf;
-using Grpc.Core;
-using MareSynchronos.API.Dto.Files;
-using MareSynchronosServer.Utils;
-using MareSynchronosShared.Models;
-using MareSynchronosShared.Protos;
-using MareSynchronosShared.Utils;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.EntityFrameworkCore;
-
-namespace MareSynchronosServer.Hubs;
-
-public partial class MareHub
-{
-    private static readonly SemaphoreSlim _uploadSemaphore = new(20);
-
-    [Authorize(Policy = "Identified")]
-    public async Task FilesAbortUpload()
-    {
-        _logger.LogCallInfo();
-        var notUploadedFiles = await _dbContext.Files.Where(f => !f.Uploaded && f.Uploader.UID == UserUID).ToListAsync();
-        if (notUploadedFiles.Any())
-        {
-            _dbContext.RemoveRange(notUploadedFiles);
-            await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-        }
-    }
-
-    [Authorize(Policy = "Identified")]
-    public async Task FilesDeleteAll()
-    {
-        _logger.LogCallInfo();
-
-        var ownFiles = await _dbContext.Files.Where(f => f.Uploaded && f.Uploader.UID == UserUID).ToListAsync().ConfigureAwait(false);
-        var request = new DeleteFilesRequest();
-        request.Hash.AddRange(ownFiles.Select(f => f.Hash));
-        Metadata headers = new Metadata()
-            {
-                { "Authorization", "Bearer " + _generator.Token },
-            };
-        _ = await _fileServiceClient.DeleteFilesAsync(request, headers).ConfigureAwait(false);
-    }
-
-    [Authorize(Policy = "Identified")]
-    public async Task<List<DownloadFileDto>> FilesGetSizes(List<string> hashes)
-    {
-        _logger.LogCallInfo(MareHubLogger.Args(hashes.Count.ToString()));
-
-        var allFiles = await _dbContext.Files.Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
-        var forbiddenFiles = await _dbContext.ForbiddenUploadEntries.
-            Where(f => hashes.Contains(f.Hash)).ToListAsync().ConfigureAwait(false);
-        List<DownloadFileDto> response = new();
-
-        var cacheFile = await _dbContext.Files.AsNoTracking().Where(f => hashes.Contains(f.Hash)).AsNoTracking().Select(k => new { k.Hash, k.Size }).AsNoTracking().ToListAsync().ConfigureAwait(false);
-
-        var shardConfig = new List<CdnShardConfiguration>(_configurationService.GetValueOrDefault(nameof(ServerConfiguration.CdnShardConfiguration), new List<CdnShardConfiguration>()));
-
-        foreach (var file in cacheFile)
-        {
-            var forbiddenFile = forbiddenFiles.SingleOrDefault(f => string.Equals(f.Hash, file.Hash, StringComparison.OrdinalIgnoreCase));
-
-            var matchedShardConfig = shardConfig.OrderBy(g => Guid.NewGuid()).FirstOrDefault(f => new Regex(f.FileMatch).IsMatch(file.Hash));
-            var baseUrl = matchedShardConfig?.CdnFullUrl ?? _mainCdnFullUrl;
-
-            response.Add(new DownloadFileDto
-            {
-                FileExists = file.Size > 0,
-                ForbiddenBy = forbiddenFile?.ForbiddenBy ?? string.Empty,
-                IsForbidden = forbiddenFile != null,
-                Hash = file.Hash,
-                Size = file.Size,
-                Url = baseUrl.ToString(),
-            });
-        }
-
-        return response;
-    }
-
-    [Authorize(Policy = "Identified")]
-    public async Task<bool> FilesIsUploadFinished()
-    {
-        _logger.LogCallInfo();
-        return await _dbContext.Files.AsNoTracking()
-            .AnyAsync(f => f.Uploader.UID == UserUID && !f.Uploaded).ConfigureAwait(false);
-    }
-
-    [Authorize(Policy = "Identified")]
-    public async Task<List<UploadFileDto>> FilesSend(List<string> fileListHashes)
-    {
-        var userSentHashes = new HashSet<string>(fileListHashes.Distinct(StringComparer.Ordinal).Select(s => string.Concat(s.Where(c => char.IsLetterOrDigit(c)))), StringComparer.Ordinal);
-        _logger.LogCallInfo(MareHubLogger.Args(userSentHashes.Count.ToString()));
-        var notCoveredFiles = new Dictionary<string, UploadFileDto>(StringComparer.Ordinal);
-        var forbiddenFiles = await _dbContext.ForbiddenUploadEntries.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
-        var existingFiles = await _dbContext.Files.AsNoTracking().Where(f => userSentHashes.Contains(f.Hash)).AsNoTracking().ToDictionaryAsync(f => f.Hash, f => f).ConfigureAwait(false);
-        var uploader = await _dbContext.Users.SingleAsync(u => u.UID == UserUID).ConfigureAwait(false);
-
-        List<FileCache> fileCachesToUpload = new();
-        foreach (var hash in userSentHashes)
-        {
-            // Skip empty file hashes, duplicate file hashes, forbidden file hashes and existing file hashes
-            if (string.IsNullOrEmpty(hash)) { continue; }
-            if (notCoveredFiles.ContainsKey(hash)) { continue; }
-            if (forbiddenFiles.ContainsKey(hash))
-            {
-                notCoveredFiles[hash] = new UploadFileDto()
-                {
-                    ForbiddenBy = forbiddenFiles[hash].ForbiddenBy,
-                    Hash = hash,
-                    IsForbidden = true,
-                };
-
-                continue;
-            }
-            if (existingFiles.TryGetValue(hash, out var file) && file.Uploaded) { continue; }
-
-            _logger.LogCallInfo(MareHubLogger.Args(hash, "Missing"));
-
-            if (file == null)
-            {
-                fileCachesToUpload.Add(new FileCache()
-                {
-                    Hash = hash,
-                    Uploaded = false,
-                    Uploader = uploader,
-                    UploadDate = DateTime.UtcNow,
-                });
-            }
-
-            notCoveredFiles[hash] = new UploadFileDto()
-            {
-                Hash = hash,
-            };
-        }
-        //Save bulk
-        await _dbContext.Files.AddRangeAsync(fileCachesToUpload).ConfigureAwait(false);
-        await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-        return notCoveredFiles.Values.ToList();
-    }
-
-    [Authorize(Policy = "Identified")]
-    public async Task FilesUploadStreamAsync(string hash, IAsyncEnumerable<byte[]> fileContent)
-    {
-        _logger.LogCallInfo(MareHubLogger.Args(hash));
-
-        await _uploadSemaphore.WaitAsync(Context.ConnectionAborted).ConfigureAwait(false);
-
-        var relatedFile = _dbContext.Files.SingleOrDefault(f => f.Hash == hash && f.Uploader.UID == UserUID && !f.Uploaded);
-        if (relatedFile == null)
-        {
-            _uploadSemaphore.Release();
-            return;
-        }
-        var forbiddenFile = _dbContext.ForbiddenUploadEntries.SingleOrDefault(f => f.Hash == hash);
-        if (forbiddenFile != null)
-        {
-            _uploadSemaphore.Release();
-            return;
-        }
-
-        var tempFileName = Path.GetTempFileName();
-        using var fileStream = new FileStream(tempFileName, FileMode.OpenOrCreate);
-        long length = 0;
-        try
-        {
-            await foreach (var chunk in fileContent.ConfigureAwait(false))
-            {
-                length += chunk.Length;
-                await fileStream.WriteAsync(chunk).ConfigureAwait(false);
-            }
-
-            await fileStream.FlushAsync().ConfigureAwait(false);
-            await fileStream.DisposeAsync().ConfigureAwait(false);
-        }
-        catch
-        {
-            try
-            {
-                await fileStream.FlushAsync().ConfigureAwait(false);
-                await fileStream.DisposeAsync().ConfigureAwait(false);
-                _dbContext.Files.Remove(relatedFile);
-                await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-            }
-            catch
-            {
-                // already removed
-            }
-            finally
-            {
-                File.Delete(tempFileName);
-            }
-
-            _uploadSemaphore.Release();
-            return;
-        }
-
-        _logger.LogCallInfo(MareHubLogger.Args(hash, "Uploaded"));
-
-        try
-        {
-            var decodedFile = LZ4.LZ4Codec.Unwrap(await File.ReadAllBytesAsync(tempFileName).ConfigureAwait(false));
-            using var sha1 = SHA1.Create();
-            using var ms = new MemoryStream(decodedFile);
-            var computedHash = await sha1.ComputeHashAsync(ms).ConfigureAwait(false);
-            var computedHashString = BitConverter.ToString(computedHash).Replace("-", "", StringComparison.Ordinal);
-            if (!string.Equals(hash, computedHashString, StringComparison.Ordinal))
-            {
-                _logger.LogCallWarning(MareHubLogger.Args(hash, "Invalid", computedHashString));
-                _dbContext.Remove(relatedFile);
-                await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-
-                _uploadSemaphore.Release();
-                return;
-            }
-
-            Metadata headers = new Metadata()
-            {
-                { "Authorization", "Bearer " + _generator.Token },
-            };
-            var streamingCall = _fileServiceClient.UploadFile(headers);
-            using var tempFileStream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read);
-            int size = 1024 * 1024;
-            byte[] data = new byte[size];
-            int readBytes;
-            while ((readBytes = tempFileStream.Read(data, 0, size)) > 0)
-            {
-                await streamingCall.RequestStream.WriteAsync(new UploadFileRequest()
-                {
-                    FileData = ByteString.CopyFrom(data, 0, readBytes),
-                    Hash = computedHashString,
-                    Uploader = UserUID,
-                }).ConfigureAwait(false);
-            }
-            await streamingCall.RequestStream.CompleteAsync().ConfigureAwait(false);
-            tempFileStream.Close();
-            await tempFileStream.DisposeAsync().ConfigureAwait(false);
-
-            _logger.LogCallInfo(MareHubLogger.Args(hash, "Pushed"));
-        }
-        catch (Exception ex)
-        {
-            _logger.LogCallWarning(MareHubLogger.Args("Failed", hash, ex.Message));
-            _dbContext.Remove(relatedFile);
-            await _dbContext.SaveChangesAsync().ConfigureAwait(false);
-        }
-        finally
-        {
-            _uploadSemaphore.Release();
-            File.Delete(tempFileName);
-        }
-    }
-}

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.Functions.cs

@@ -30,7 +30,7 @@ public partial class MareHub
 
     private async Task<Dictionary<string, string>> GetOnlineUsers(List<string> uids)
     {
-        var result = await _redis.GetAllAsync<string>(uids.Select(u => "UID:" + u).ToArray()).ConfigureAwait(false);
+        var result = await _redis.GetAllAsync<string>(uids.Select(u => "UID:" + u).ToHashSet(StringComparer.Ordinal)).ConfigureAwait(false);
         return uids.Where(u => result.TryGetValue("UID:" + u, out var ident) && !string.IsNullOrEmpty(ident)).ToDictionary(u => u, u => result["UID:" + u], StringComparer.Ordinal);
     }
 

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.User.cs

@@ -68,6 +68,10 @@ public partial class MareHub
                 OtherIsPaused = otherEntry != null && otherEntry.IsPaused,
                 userToOther.OtherUserUID,
                 IsSynced = otherEntry != null,
+                DisableOwnAnimations = userToOther.DisableAnimations,
+                DisableOwnSounds = userToOther.DisableSounds,
+                DisableOtherAnimations = otherEntry == null ? false : otherEntry.DisableAnimations,
+                DisableOtherSounds = otherEntry == null ? false : otherEntry.DisableSounds
             };
 
         var results = await query.AsNoTracking().ToListAsync().ConfigureAwait(false);
@@ -76,9 +80,13 @@ public partial class MareHub
         {
             var ownPerm = UserPermissions.Paired;
             ownPerm.SetPaused(c.IsPaused);
+            ownPerm.SetDisableAnimations(c.DisableOwnAnimations);
+            ownPerm.SetDisableSounds(c.DisableOwnSounds);
             var otherPerm = UserPermissions.NoneSet;
             otherPerm.SetPaired(c.IsSynced);
             otherPerm.SetPaused(c.OtherIsPaused);
+            otherPerm.SetDisableAnimations(c.DisableOtherAnimations);
+            otherPerm.SetDisableSounds(c.DisableOtherSounds);
             return new UserPairDto(new(c.OtherUserUID, c.Alias), ownPerm, otherPerm);
         }).ToList();
     }
@@ -215,7 +223,11 @@ public partial class MareHub
         ClientPair pair = await _dbContext.ClientPairs.SingleOrDefaultAsync(w => w.UserUID == UserUID && w.OtherUserUID == dto.User.UID).ConfigureAwait(false);
         if (pair == null) return;
 
+        var pauseChange = pair.IsPaused != dto.Permissions.IsPaused();
+
         pair.IsPaused = dto.Permissions.IsPaused();
+        pair.DisableAnimations = dto.Permissions.IsDisableAnimations();
+        pair.DisableSounds = dto.Permissions.IsDisableSounds();
         _dbContext.Update(pair);
         await _dbContext.SaveChangesAsync().ConfigureAwait(false);
 
@@ -229,19 +241,22 @@ public partial class MareHub
         {
             await Clients.User(dto.User.UID).Client_UserUpdateOtherPairPermissions(new UserPermissionsDto(new UserData(UserUID), dto.Permissions)).ConfigureAwait(false);
 
-            var otherCharaIdent = await GetUserIdent(pair.OtherUserUID).ConfigureAwait(false);
-
-            if (UserCharaIdent == null || otherCharaIdent == null || otherEntry.IsPaused) return;
-
-            if (dto.Permissions.IsPaused())
+            if (pauseChange)
             {
-                await Clients.User(UserUID).Client_UserSendOffline(dto).ConfigureAwait(false);
-                await Clients.User(dto.User.UID).Client_UserSendOffline(new(new(UserUID))).ConfigureAwait(false);
-            }
-            else
-            {
-                await Clients.User(UserUID).Client_UserSendOnline(new(dto.User, otherCharaIdent)).ConfigureAwait(false);
-                await Clients.User(dto.User.UID).Client_UserSendOnline(new(new(UserUID), UserCharaIdent)).ConfigureAwait(false);
+                var otherCharaIdent = await GetUserIdent(pair.OtherUserUID).ConfigureAwait(false);
+
+                if (UserCharaIdent == null || otherCharaIdent == null || otherEntry.IsPaused) return;
+
+                if (dto.Permissions.IsPaused())
+                {
+                    await Clients.User(UserUID).Client_UserSendOffline(dto).ConfigureAwait(false);
+                    await Clients.User(dto.User.UID).Client_UserSendOffline(new(new(UserUID))).ConfigureAwait(false);
+                }
+                else
+                {
+                    await Clients.User(UserUID).Client_UserSendOnline(new(dto.User, otherCharaIdent)).ConfigureAwait(false);
+                    await Clients.User(dto.User.UID).Client_UserSendOnline(new(new(UserUID), UserCharaIdent)).ConfigureAwait(false);
+                }
             }
         }
     }

MareSynchronosServer/MareSynchronosServer/Hubs/MareHub.cs

@@ -7,7 +7,6 @@ using MareSynchronosServer.Utils;
 using MareSynchronosShared;
 using MareSynchronosShared.Data;
 using MareSynchronosShared.Metrics;
-using MareSynchronosShared.Protos;
 using MareSynchronosShared.Services;
 using MareSynchronosShared.Utils;
 using Microsoft.AspNetCore.Authorization;
@@ -20,37 +19,33 @@ namespace MareSynchronosServer.Hubs;
 public partial class MareHub : Hub<IMareHub>, IMareHub
 {
     private readonly MareMetrics _mareMetrics;
-    private readonly FileService.FileServiceClient _fileServiceClient;
     private readonly SystemInfoService _systemInfoService;
     private readonly IHttpContextAccessor _contextAccessor;
     private readonly MareHubLogger _logger;
     private readonly MareDbContext _dbContext;
-    private readonly Uri _mainCdnFullUrl;
     private readonly string _shardName;
     private readonly int _maxExistingGroupsByUser;
     private readonly int _maxJoinedGroupsByUser;
     private readonly int _maxGroupUserCount;
-    private readonly IConfigurationService<ServerConfiguration> _configurationService;
     private readonly IRedisDatabase _redis;
-    private readonly ServerTokenGenerator _generator;
+    private readonly Uri _fileServerAddress;
+    private readonly Version _expectedClientVersion;
 
-    public MareHub(MareMetrics mareMetrics, FileService.FileServiceClient fileServiceClient,
+    public MareHub(MareMetrics mareMetrics,
         MareDbContext mareDbContext, ILogger<MareHub> logger, SystemInfoService systemInfoService,
         IConfigurationService<ServerConfiguration> configuration, IHttpContextAccessor contextAccessor,
-        IRedisDatabase redisDb, ServerTokenGenerator generator)
+        IRedisDatabase redisDb)
     {
         _mareMetrics = mareMetrics;
-        _fileServiceClient = fileServiceClient;
         _systemInfoService = systemInfoService;
-        _configurationService = configuration;
-        _mainCdnFullUrl = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
         _shardName = configuration.GetValue<string>(nameof(ServerConfiguration.ShardName));
         _maxExistingGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxExistingGroupsByUser), 3);
         _maxJoinedGroupsByUser = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxJoinedGroupsByUser), 6);
         _maxGroupUserCount = configuration.GetValueOrDefault(nameof(ServerConfiguration.MaxGroupUserCount), 100);
+        _fileServerAddress = configuration.GetValue<Uri>(nameof(ServerConfiguration.CdnFullUrl));
+        _expectedClientVersion = configuration.GetValueOrDefault(nameof(ServerConfiguration.ExpectedClientVersion), new Version(0, 0, 0));
         _contextAccessor = contextAccessor;
         _redis = redisDb;
-        _generator = generator;
         _logger = new MareHubLogger(this, logger);
         _dbContext = mareDbContext;
     }
@@ -73,6 +68,7 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
 
         return new ConnectionDto(new UserData(dbUser.UID, string.IsNullOrWhiteSpace(dbUser.Alias) ? null : dbUser.Alias))
         {
+            CurrentClientVersion = _expectedClientVersion,
             ServerVersion = IMareHub.ApiVersion,
             IsAdmin = dbUser.IsAdmin,
             IsModerator = dbUser.IsModerator,
@@ -82,6 +78,7 @@ public partial class MareHub : Hub<IMareHub>, IMareHub
                 ShardName = _shardName,
                 MaxGroupsJoinedByUser = _maxJoinedGroupsByUser,
                 MaxGroupUserCount = _maxGroupUserCount,
+                FileServerAddress = _fileServerAddress
             },
         };
     }