From efe7fe98f2ca802d509816d2dc974ed1c5157113 Mon Sep 17 00:00:00 2001 From: Stanley Dimant Date: Wed, 4 Sep 2024 16:53:05 +0200 Subject: [PATCH] add raw filesize to db --- .../20240904144711_AddRawFileSize.Designer.cs | 761 ++++++++++++++++++ .../20240904144711_AddRawFileSize.cs | 29 + .../Migrations/MareDbContextModelSnapshot.cs | 4 + .../MareSynchronosShared/Models/FileCache.cs | 1 + .../Controllers/ServerFilesController.cs | 9 +- .../Services/MainFileCleanupService.cs | 11 + 6 files changed, 812 insertions(+), 3 deletions(-) create mode 100644 MareSynchronosServer/MareSynchronosShared/Migrations/20240904144711_AddRawFileSize.Designer.cs create mode 100644 MareSynchronosServer/MareSynchronosShared/Migrations/20240904144711_AddRawFileSize.cs diff --git a/MareSynchronosServer/MareSynchronosShared/Migrations/20240904144711_AddRawFileSize.Designer.cs b/MareSynchronosServer/MareSynchronosShared/Migrations/20240904144711_AddRawFileSize.Designer.cs new file mode 100644 index 0000000..2e06ddf --- /dev/null +++ b/MareSynchronosServer/MareSynchronosShared/Migrations/20240904144711_AddRawFileSize.Designer.cs @@ -0,0 +1,761 @@ +// +using System; +using MareSynchronosShared.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace MareSynchronosServer.Migrations +{ + [DbContext(typeof(MareDbContext))] + [Migration("20240904144711_AddRawFileSize")] + partial class AddRawFileSize + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.4") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("MareSynchronosShared.Models.Auth", b => + { + b.Property("HashedKey") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("hashed_key"); + + b.Property("IsBanned") + .HasColumnType("boolean") + .HasColumnName("is_banned"); + + b.Property("MarkForBan") + .HasColumnType("boolean") + .HasColumnName("mark_for_ban"); + + b.Property("PrimaryUserUID") + .HasColumnType("character varying(10)") + .HasColumnName("primary_user_uid"); + + b.Property("UserUID") + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.HasKey("HashedKey") + .HasName("pk_auth"); + + b.HasIndex("PrimaryUserUID") + .HasDatabaseName("ix_auth_primary_user_uid"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_auth_user_uid"); + + b.ToTable("auth", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Banned", b => + { + b.Property("CharacterIdentification") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("character_identification"); + + b.Property("Reason") + .HasColumnType("text") + .HasColumnName("reason"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("CharacterIdentification") + .HasName("pk_banned_users"); + + b.ToTable("banned_users", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.BannedRegistrations", b => + { + b.Property("DiscordIdOrLodestoneAuth") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("discord_id_or_lodestone_auth"); + + b.HasKey("DiscordIdOrLodestoneAuth") + .HasName("pk_banned_registrations"); + + b.ToTable("banned_registrations", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b => + { + b.Property("UserUID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.Property("OtherUserUID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("other_user_uid"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("UserUID", "OtherUserUID") + .HasName("pk_client_pairs"); + + b.HasIndex("OtherUserUID") + .HasDatabaseName("ix_client_pairs_other_user_uid"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_client_pairs_user_uid"); + + b.ToTable("client_pairs", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b => + { + b.Property("Hash") + .HasMaxLength(40) + .HasColumnType("character varying(40)") + .HasColumnName("hash"); + + b.Property("RawSize") + .HasColumnType("bigint") + .HasColumnName("raw_size"); + + b.Property("Size") + .HasColumnType("bigint") + .HasColumnName("size"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.Property("UploadDate") + .HasColumnType("timestamp with time zone") + .HasColumnName("upload_date"); + + b.Property("Uploaded") + .HasColumnType("boolean") + .HasColumnName("uploaded"); + + b.Property("UploaderUID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("uploader_uid"); + + b.HasKey("Hash") + .HasName("pk_file_caches"); + + b.HasIndex("UploaderUID") + .HasDatabaseName("ix_file_caches_uploader_uid"); + + b.ToTable("file_caches", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.ForbiddenUploadEntry", b => + { + b.Property("Hash") + .HasMaxLength(40) + .HasColumnType("character varying(40)") + .HasColumnName("hash"); + + b.Property("ForbiddenBy") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("forbidden_by"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("Hash") + .HasName("pk_forbidden_upload_entries"); + + b.ToTable("forbidden_upload_entries", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Group", b => + { + b.Property("GID") + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("gid"); + + b.Property("Alias") + .HasMaxLength(50) + .HasColumnType("character varying(50)") + .HasColumnName("alias"); + + b.Property("HashedPassword") + .HasColumnType("text") + .HasColumnName("hashed_password"); + + b.Property("InvitesEnabled") + .HasColumnType("boolean") + .HasColumnName("invites_enabled"); + + b.Property("OwnerUID") + .HasColumnType("character varying(10)") + .HasColumnName("owner_uid"); + + b.Property("PreferDisableAnimations") + .HasColumnType("boolean") + .HasColumnName("prefer_disable_animations"); + + b.Property("PreferDisableSounds") + .HasColumnType("boolean") + .HasColumnName("prefer_disable_sounds"); + + b.Property("PreferDisableVFX") + .HasColumnType("boolean") + .HasColumnName("prefer_disable_vfx"); + + b.HasKey("GID") + .HasName("pk_groups"); + + b.HasIndex("OwnerUID") + .HasDatabaseName("ix_groups_owner_uid"); + + b.ToTable("groups", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b => + { + b.Property("GroupGID") + .HasColumnType("character varying(20)") + .HasColumnName("group_gid"); + + b.Property("BannedUserUID") + .HasColumnType("character varying(10)") + .HasColumnName("banned_user_uid"); + + b.Property("BannedByUID") + .HasColumnType("character varying(10)") + .HasColumnName("banned_by_uid"); + + b.Property("BannedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("banned_on"); + + b.Property("BannedReason") + .HasColumnType("text") + .HasColumnName("banned_reason"); + + b.HasKey("GroupGID", "BannedUserUID") + .HasName("pk_group_bans"); + + b.HasIndex("BannedByUID") + .HasDatabaseName("ix_group_bans_banned_by_uid"); + + b.HasIndex("BannedUserUID") + .HasDatabaseName("ix_group_bans_banned_user_uid"); + + b.HasIndex("GroupGID") + .HasDatabaseName("ix_group_bans_group_gid"); + + b.ToTable("group_bans", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b => + { + b.Property("GroupGID") + .HasColumnType("character varying(20)") + .HasColumnName("group_gid"); + + b.Property("GroupUserUID") + .HasColumnType("character varying(10)") + .HasColumnName("group_user_uid"); + + b.Property("IsModerator") + .HasColumnType("boolean") + .HasColumnName("is_moderator"); + + b.Property("IsPinned") + .HasColumnType("boolean") + .HasColumnName("is_pinned"); + + b.HasKey("GroupGID", "GroupUserUID") + .HasName("pk_group_pairs"); + + b.HasIndex("GroupGID") + .HasDatabaseName("ix_group_pairs_group_gid"); + + b.HasIndex("GroupUserUID") + .HasDatabaseName("ix_group_pairs_group_user_uid"); + + b.ToTable("group_pairs", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupPairPreferredPermission", b => + { + b.Property("UserUID") + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.Property("GroupGID") + .HasColumnType("character varying(20)") + .HasColumnName("group_gid"); + + b.Property("DisableAnimations") + .HasColumnType("boolean") + .HasColumnName("disable_animations"); + + b.Property("DisableSounds") + .HasColumnType("boolean") + .HasColumnName("disable_sounds"); + + b.Property("DisableVFX") + .HasColumnType("boolean") + .HasColumnName("disable_vfx"); + + b.Property("IsPaused") + .HasColumnType("boolean") + .HasColumnName("is_paused"); + + b.HasKey("UserUID", "GroupGID") + .HasName("pk_group_pair_preferred_permissions"); + + b.HasIndex("GroupGID") + .HasDatabaseName("ix_group_pair_preferred_permissions_group_gid"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_group_pair_preferred_permissions_user_uid"); + + b.ToTable("group_pair_preferred_permissions", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b => + { + b.Property("GroupGID") + .HasColumnType("character varying(20)") + .HasColumnName("group_gid"); + + b.Property("Invite") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("invite"); + + b.Property("ExpirationDate") + .HasColumnType("timestamp with time zone") + .HasColumnName("expiration_date"); + + b.HasKey("GroupGID", "Invite") + .HasName("pk_group_temp_invites"); + + b.HasIndex("GroupGID") + .HasDatabaseName("ix_group_temp_invites_group_gid"); + + b.HasIndex("Invite") + .HasDatabaseName("ix_group_temp_invites_invite"); + + b.ToTable("group_temp_invites", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b => + { + b.Property("DiscordId") + .ValueGeneratedOnAdd() + .HasColumnType("numeric(20,0)") + .HasColumnName("discord_id"); + + b.Property("HashedLodestoneId") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("hashed_lodestone_id"); + + b.Property("LodestoneAuthString") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("lodestone_auth_string"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("UserUID") + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.HasKey("DiscordId") + .HasName("pk_lodestone_auth"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_lodestone_auth_user_uid"); + + b.ToTable("lodestone_auth", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.User", b => + { + b.Property("UID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("uid"); + + b.Property("Alias") + .HasMaxLength(15) + .HasColumnType("character varying(15)") + .HasColumnName("alias"); + + b.Property("IsAdmin") + .HasColumnType("boolean") + .HasColumnName("is_admin"); + + b.Property("IsModerator") + .HasColumnType("boolean") + .HasColumnName("is_moderator"); + + b.Property("LastLoggedIn") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_logged_in"); + + b.Property("Timestamp") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("bytea") + .HasColumnName("timestamp"); + + b.HasKey("UID") + .HasName("pk_users"); + + b.ToTable("users", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.UserDefaultPreferredPermission", b => + { + b.Property("UserUID") + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.Property("DisableGroupAnimations") + .HasColumnType("boolean") + .HasColumnName("disable_group_animations"); + + b.Property("DisableGroupSounds") + .HasColumnType("boolean") + .HasColumnName("disable_group_sounds"); + + b.Property("DisableGroupVFX") + .HasColumnType("boolean") + .HasColumnName("disable_group_vfx"); + + b.Property("DisableIndividualAnimations") + .HasColumnType("boolean") + .HasColumnName("disable_individual_animations"); + + b.Property("DisableIndividualSounds") + .HasColumnType("boolean") + .HasColumnName("disable_individual_sounds"); + + b.Property("DisableIndividualVFX") + .HasColumnType("boolean") + .HasColumnName("disable_individual_vfx"); + + b.Property("IndividualIsSticky") + .HasColumnType("boolean") + .HasColumnName("individual_is_sticky"); + + b.HasKey("UserUID") + .HasName("pk_user_default_preferred_permissions"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_user_default_preferred_permissions_user_uid"); + + b.ToTable("user_default_preferred_permissions", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.UserPermissionSet", b => + { + b.Property("UserUID") + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.Property("OtherUserUID") + .HasColumnType("character varying(10)") + .HasColumnName("other_user_uid"); + + b.Property("DisableAnimations") + .HasColumnType("boolean") + .HasColumnName("disable_animations"); + + b.Property("DisableSounds") + .HasColumnType("boolean") + .HasColumnName("disable_sounds"); + + b.Property("DisableVFX") + .HasColumnType("boolean") + .HasColumnName("disable_vfx"); + + b.Property("IsPaused") + .HasColumnType("boolean") + .HasColumnName("is_paused"); + + b.Property("Sticky") + .HasColumnType("boolean") + .HasColumnName("sticky"); + + b.HasKey("UserUID", "OtherUserUID") + .HasName("pk_user_permission_sets"); + + b.HasIndex("OtherUserUID") + .HasDatabaseName("ix_user_permission_sets_other_user_uid"); + + b.HasIndex("UserUID") + .HasDatabaseName("ix_user_permission_sets_user_uid"); + + b.HasIndex("UserUID", "OtherUserUID", "IsPaused") + .HasDatabaseName("ix_user_permission_sets_user_uid_other_user_uid_is_paused"); + + b.ToTable("user_permission_sets", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.UserProfileData", b => + { + b.Property("UserUID") + .HasColumnType("character varying(10)") + .HasColumnName("user_uid"); + + b.Property("Base64ProfileImage") + .HasColumnType("text") + .HasColumnName("base64profile_image"); + + b.Property("FlaggedForReport") + .HasColumnType("boolean") + .HasColumnName("flagged_for_report"); + + b.Property("IsNSFW") + .HasColumnType("boolean") + .HasColumnName("is_nsfw"); + + b.Property("ProfileDisabled") + .HasColumnType("boolean") + .HasColumnName("profile_disabled"); + + b.Property("UserDescription") + .HasColumnType("text") + .HasColumnName("user_description"); + + b.HasKey("UserUID") + .HasName("pk_user_profile_data"); + + b.ToTable("user_profile_data", (string)null); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Auth", b => + { + b.HasOne("MareSynchronosShared.Models.User", "PrimaryUser") + .WithMany() + .HasForeignKey("PrimaryUserUID") + .HasConstraintName("fk_auth_users_primary_user_uid"); + + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .HasConstraintName("fk_auth_users_user_uid"); + + b.Navigation("PrimaryUser"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.ClientPair", b => + { + b.HasOne("MareSynchronosShared.Models.User", "OtherUser") + .WithMany() + .HasForeignKey("OtherUserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_client_pairs_users_other_user_uid"); + + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_client_pairs_users_user_uid"); + + b.Navigation("OtherUser"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.FileCache", b => + { + b.HasOne("MareSynchronosShared.Models.User", "Uploader") + .WithMany() + .HasForeignKey("UploaderUID") + .HasConstraintName("fk_file_caches_users_uploader_uid"); + + b.Navigation("Uploader"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.Group", b => + { + b.HasOne("MareSynchronosShared.Models.User", "Owner") + .WithMany() + .HasForeignKey("OwnerUID") + .HasConstraintName("fk_groups_users_owner_uid"); + + b.Navigation("Owner"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupBan", b => + { + b.HasOne("MareSynchronosShared.Models.User", "BannedBy") + .WithMany() + .HasForeignKey("BannedByUID") + .HasConstraintName("fk_group_bans_users_banned_by_uid"); + + b.HasOne("MareSynchronosShared.Models.User", "BannedUser") + .WithMany() + .HasForeignKey("BannedUserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_bans_users_banned_user_uid"); + + b.HasOne("MareSynchronosShared.Models.Group", "Group") + .WithMany() + .HasForeignKey("GroupGID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_bans_groups_group_gid"); + + b.Navigation("BannedBy"); + + b.Navigation("BannedUser"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupPair", b => + { + b.HasOne("MareSynchronosShared.Models.Group", "Group") + .WithMany() + .HasForeignKey("GroupGID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_pairs_groups_group_gid"); + + b.HasOne("MareSynchronosShared.Models.User", "GroupUser") + .WithMany() + .HasForeignKey("GroupUserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_pairs_users_group_user_uid"); + + b.Navigation("Group"); + + b.Navigation("GroupUser"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupPairPreferredPermission", b => + { + b.HasOne("MareSynchronosShared.Models.Group", "Group") + .WithMany() + .HasForeignKey("GroupGID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_pair_preferred_permissions_groups_group_gid"); + + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_pair_preferred_permissions_users_user_uid"); + + b.Navigation("Group"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.GroupTempInvite", b => + { + b.HasOne("MareSynchronosShared.Models.Group", "Group") + .WithMany() + .HasForeignKey("GroupGID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_group_temp_invites_groups_group_gid"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.LodeStoneAuth", b => + { + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .HasConstraintName("fk_lodestone_auth_users_user_uid"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.UserDefaultPreferredPermission", b => + { + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_default_preferred_permissions_users_user_uid"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.UserPermissionSet", b => + { + b.HasOne("MareSynchronosShared.Models.User", "OtherUser") + .WithMany() + .HasForeignKey("OtherUserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_permission_sets_users_other_user_uid"); + + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_permission_sets_users_user_uid"); + + b.Navigation("OtherUser"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("MareSynchronosShared.Models.UserProfileData", b => + { + b.HasOne("MareSynchronosShared.Models.User", "User") + .WithMany() + .HasForeignKey("UserUID") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_profile_data_users_user_uid"); + + b.Navigation("User"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/MareSynchronosServer/MareSynchronosShared/Migrations/20240904144711_AddRawFileSize.cs b/MareSynchronosServer/MareSynchronosShared/Migrations/20240904144711_AddRawFileSize.cs new file mode 100644 index 0000000..1bb3cd3 --- /dev/null +++ b/MareSynchronosServer/MareSynchronosShared/Migrations/20240904144711_AddRawFileSize.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace MareSynchronosServer.Migrations +{ + /// + public partial class AddRawFileSize : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "raw_size", + table: "file_caches", + type: "bigint", + nullable: false, + defaultValue: 0L); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "raw_size", + table: "file_caches"); + } + } +} diff --git a/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs b/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs index bf07616..4290c5a 100644 --- a/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs +++ b/MareSynchronosServer/MareSynchronosShared/Migrations/MareDbContextModelSnapshot.cs @@ -130,6 +130,10 @@ namespace MareSynchronosServer.Migrations .HasColumnType("character varying(40)") .HasColumnName("hash"); + b.Property("RawSize") + .HasColumnType("bigint") + .HasColumnName("raw_size"); + b.Property("Size") .HasColumnType("bigint") .HasColumnName("size"); diff --git a/MareSynchronosServer/MareSynchronosShared/Models/FileCache.cs b/MareSynchronosServer/MareSynchronosShared/Models/FileCache.cs index 3ffd2c2..d0a4345 100644 --- a/MareSynchronosServer/MareSynchronosShared/Models/FileCache.cs +++ b/MareSynchronosServer/MareSynchronosShared/Models/FileCache.cs @@ -15,4 +15,5 @@ public class FileCache [Timestamp] public byte[] Timestamp { get; set; } public long Size { get; set; } + public long RawSize { get; set; } } diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs index 9f4e7ed..1003ef8 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Controllers/ServerFilesController.cs @@ -232,7 +232,8 @@ public class ServerFilesController : ControllerBase UploadDate = DateTime.UtcNow, UploaderUID = MareUser, Size = compressedFileStream.Length, - Uploaded = true + Uploaded = true, + RawSize = data.LongLength }).ConfigureAwait(false); await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); @@ -333,7 +334,8 @@ public class ServerFilesController : ControllerBase UploadDate = DateTime.UtcNow, UploaderUID = MareUser, Size = compressedMungedStream.Length, - Uploaded = true + Uploaded = true, + RawSize = data.Length }).ConfigureAwait(false); await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); @@ -440,7 +442,8 @@ public class ServerFilesController : ControllerBase UploadDate = DateTime.UtcNow, UploaderUID = MareUser, Size = compressedStream.Length, - Uploaded = true + Uploaded = true, + RawSize = rawFileStream.Length }).ConfigureAwait(false); await _mareDbContext.SaveChangesAsync().ConfigureAwait(false); diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs index 835352e..1144363 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/MainFileCleanupService.cs @@ -1,4 +1,5 @@ using ByteSizeLib; +using LZ4; using MareSynchronosShared.Data; using MareSynchronosShared.Metrics; using MareSynchronosShared.Models; @@ -262,6 +263,16 @@ public class MainFileCleanupService : IHostedService await dbContext.SaveChangesAsync().ConfigureAwait(false); } + // only used if file in db has no raw size for whatever reason + if (!deleteCurrentFile && file != null && fileCache.RawSize == 0) + { + var length = LZ4Codec.Unwrap(File.ReadAllBytes(file.FullName)).LongLength; + _logger.LogInformation("Setting Raw File Size of " + fileCache.Hash + " to " + length); + fileCache.RawSize = length; + if (fileCounter % 1000 == 0) + await dbContext.SaveChangesAsync().ConfigureAwait(false); + } + fileCounter++; ct.ThrowIfCancellationRequested();