From 9f82f8a25acafb3dc787906e8cc3c86a6c6e8e77 Mon Sep 17 00:00:00 2001 From: rootdarkarchon Date: Thu, 18 Jan 2024 21:00:14 +0100 Subject: [PATCH] handle download errors on shards better --- .../Services/CachedFileProvider.cs | 7 +++++-- .../Services/ShardFileCleanupService.cs | 10 +++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs index ddf84c3..ef8d942 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/CachedFileProvider.cs @@ -69,7 +69,8 @@ public sealed class CachedFileProvider : IDisposable } var fileName = FilePathUtil.GetFilePath(_basePath, hash); - using var fileStream = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite); + var tempFileName = fileName + ".dl"; + var fileStream = new FileStream(tempFileName, FileMode.Create, FileAccess.ReadWrite); var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024; var buffer = new byte[bufferSize]; @@ -79,6 +80,8 @@ public sealed class CachedFileProvider : IDisposable { await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false); } + await fileStream.DisposeAsync().ConfigureAwait(false); + File.Move(tempFileName, fileName, true); _metrics.IncGauge(MetricsAPI.GaugeFilesTotal); _metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length); @@ -90,7 +93,7 @@ public sealed class CachedFileProvider : IDisposable if (fi == null && IsMainServer) return; await _downloadSemaphore.WaitAsync().ConfigureAwait(false); - if (fi == null && !_currentTransfers.ContainsKey(hash)) + if ((fi.Length == 0 || fi == null) && !_currentTransfers.ContainsKey(hash)) { _currentTransfers[hash] = Task.Run(async () => { diff --git a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs index 4f4401a..0c58fcc 100644 --- a/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs +++ b/MareSynchronosServer/MareSynchronosStaticFilesServer/Services/ShardFileCleanupService.cs @@ -123,7 +123,6 @@ public class ShardFileCleanupService : IHostedService var prevTimeForcedDeletion = DateTime.Now.Subtract(TimeSpan.FromHours(forcedDeletionAfterHours)); DirectoryInfo dir = new(_cacheDir); var allFilesInDir = dir.GetFiles("*", SearchOption.AllDirectories); - int fileCounter = 0; foreach (var file in allFilesInDir) { @@ -141,8 +140,13 @@ public class ShardFileCleanupService : IHostedService _logger.LogInformation("File forcefully deleted: {fileName}, {fileSize}MiB", file.Name, ByteSize.FromBytes(file.Length).MebiBytes); file.Delete(); } - - fileCounter++; + else if (file.Length == 0 && !string.Equals(file.Extension, ".dl", StringComparison.OrdinalIgnoreCase)) + { + _metrics.DecGauge(MetricsAPI.GaugeFilesTotalSize, file.Length); + _metrics.DecGauge(MetricsAPI.GaugeFilesTotal); + _logger.LogInformation("File with size 0 deleted: {filename}", file.Name); + file.Delete(); + } ct.ThrowIfCancellationRequested(); }