updated to accept only .tar.zstd uploads

ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs

@@ -6,11 +6,13 @@ namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
 public class BackgroundJobManager
 {
     private readonly ILogger _parentLogger;
+    private readonly Config.SaveUploadConfig _uploadConfig;
     private long _lastJobId;
 
-    public BackgroundJobManager(ILogger logger)
+    public BackgroundJobManager(ILogger logger, Config.SaveUploadConfig uploadConfig)
     {
         _parentLogger = logger;
+        _uploadConfig = uploadConfig;
     }
     
     public SaveParsingOperation StartNewParsingOperation(
@@ -18,7 +20,8 @@ public class BackgroundJobManager
     {
         long nextId = Interlocked.Increment(ref _lastJobId);
         var contextLogger = new ContextLogger($"BackgroundJob-{nextId}", _parentLogger);
-        var op = new SaveParsingOperation(nextId, meta, filter, contextLogger, ct);
+        var op = new SaveParsingOperation(nextId, meta, filter,
+            _uploadConfig.MaxSizePlainTextBytes, contextLogger, ct);
         op.StartAsync();
         return op;
     }

ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs

@@ -1,6 +1,4 @@
 using System.IO;
-using System.IO.Compression;
-using System.Linq;
 using System.Text.Encodings.Web;
 using System.Text.Json.Serialization;
 using ParadoxSaveParser.WebAPI.Database;
@@ -14,6 +12,7 @@ public class SaveParsingOperation
     
     private readonly SaveFileMetadata _meta;
     private readonly ISaveDataFilter _filter;
+    private readonly long _maxGameDataBytes;
     private readonly ContextLogger _logger;
     private readonly CancellationToken _ct;
 
@@ -26,11 +25,12 @@ public class SaveParsingOperation
     };
 
     public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISaveDataFilter filter,
-        ContextLogger logger, CancellationToken ct)
+        long maxGameDataBytes, ContextLogger logger, CancellationToken ct)
     {
         OperationId = operationId;
         _meta = meta;
         _filter = filter;
+        _maxGameDataBytes = maxGameDataBytes;
         _logger = logger;
         _ct = ct;
     }
@@ -76,17 +76,10 @@ public class SaveParsingOperation
         Dictionary<string, object> parsedData;
         await using (var gamestateStream = PathHelper.CreateTempFile())
         {
-            // TODO: ckeck if it is zip archive or plain text like in Search.cs
-            using (var zipArchive = ZipFile.Open(_meta.GetSaveFilePath().Str, ZipArchiveMode.Read))
-            {
-                var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
-                if (zipEntry is null)
-                    throw new Exception("Invalid save format: no 'gamestate' file found");
+            await SaveArchive.ExtractGameData(_meta.GetSaveFilePath(), _meta.game, gamestateStream,
+                _maxGameDataBytes, _ct);
+            gamestateStream.Seek(0, SeekOrigin.Begin);
 
-                await zipEntry.Open().CopyToAsync(gamestateStream, _ct);
-                gamestateStream.Seek(0, SeekOrigin.Begin);
-            }
-            
             await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.Parsing);
             var parser = new SaveParserEU4(gamestateStream, _filter.SearchExpression);
             parsedData = parser.Parse();

ParadoxSaveParser.WebAPI/Config.cs

@@ -10,6 +10,15 @@ public class Config
     public string BaseUrl { get; set; } = "http://127.0.0.1:5226/";
     public int UploadLifetimeDays { get; set; } = 30;
     public string DbPath { get; set; } = "database.sqlite";
+    public SaveUploadConfig SaveUpload { get; set; } = new();
+
+    public class SaveUploadConfig
+    {
+        /// max size of game data entry after decompression
+        public long MaxSizePlainTextBytes { get; set; } = 200 * 1024 * 1024;
+        /// max size of uploaded .tar.zst
+        public long MaxSizeCompressedBytes { get; set; } = 60 * 1024 * 1024;
+    }
 
     public static JsonSerializerOptions JsonOptions = new()
     {

ParadoxSaveParser.WebAPI/Database/DatabaseConnector.cs

@@ -28,14 +28,17 @@ public class DatabaseConnector
             .FirstOrDefault();
     }
     
-    public async Task<SaveFileMetadata> CreateMetadata(Game game)
+    public async Task<SaveFileMetadata> CreateMetadata(Game game,
+        OriginalFileType originalFileType, string originalFileName)
     {
         var meta = new SaveFileMetadata
         {
             id = Guid.NewGuid().ToString(),
             game = game,
             status = SaveFileProcessingStatus.Initialized,
-            uploadDateTime = DateTime.Now
+            uploadDateTime = DateTime.Now,
+            originalFileType = originalFileType,
+            originalFileName = originalFileName,
         };
         await _db.InsertAsync(meta);
         return meta;

ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs

@@ -12,6 +12,14 @@ public enum SaveFileProcessingStatus
     Error,
 }
 
+public enum OriginalFileType
+{
+    Text,
+    Zip,
+}
+
+public class LowerCaseEnumConverter<T>() : JsonStringEnumConverter<T>(JsonNamingPolicy.SnakeCaseLower) where T : struct, Enum;
+
 public class SaveFileMetadata
 {
     [SQLite.PrimaryKey]
@@ -22,19 +30,26 @@ public class SaveFileMetadata
     [SQLite.NotNull]
     public Game game { get; init; }
 
-    [JsonConverter(typeof(JsonStringEnumConverter))]
+    [JsonConverter(typeof(LowerCaseEnumConverter<SaveFileProcessingStatus>))]
     [SQLite.NotNull]
     public SaveFileProcessingStatus status { get; set; }
-    
+
     [SQLite.NotNull]
     public DateTime uploadDateTime { get; set; }
-    
+
+    [JsonConverter(typeof(LowerCaseEnumConverter<OriginalFileType>))]
+    [SQLite.NotNull]
+    public OriginalFileType originalFileType { get; init; }
+
+    [SQLite.NotNull]
+    public string originalFileName { get; init; } = null!;
+
     // if error occured during parsing, it's message is saved here
     // status stays the same as when error occured
     public string? errorMessage { get; set; }
-    
-    
-    public IOPath GetSaveFilePath() => Path.Concat(PathHelper.SAVES_DIR, id + ".eu4");
+
+
+    public IOPath GetSaveFilePath() => Path.Concat(PathHelper.SAVES_DIR, id + ".tar.zst");
 
     public IOPath GetParsedDataPath() => Path.Concat(PathHelper.PARSED_DIR, id + ".parsed.json");
 
@@ -42,4 +57,4 @@ public class SaveFileMetadata
     {
         return File.Exists(GetSaveFilePath()) && File.Exists(GetParsedDataPath());
     }
-}
+}

ParadoxSaveParser.WebAPI/Game.cs

@@ -3,5 +3,16 @@
 public enum Game
 {
     Unknown,
-    EU4
-}
+    EU4,
+    Stellaris,
+}
+
+public static class GameSaveEntryNames
+{
+    /// name of the entry in uploaded .tar.zst which contains game data
+    public static readonly IReadOnlyDictionary<Game, string> Map = new Dictionary<Game, string>
+    {
+        { Game.EU4, "gamestate" },
+        { Game.Stellaris, "gamestate" },
+    };
+}

ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.csproj

@@ -9,11 +9,12 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj"/>
+        <ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj" />
     </ItemGroup>
 
     <ItemGroup>
         <PackageReference Include="DTLib.Web" Version="1.4.0" />
-        <PackageReference Include="sqlite-net-pcl" Version="1.9.172" />
+        <PackageReference Include="sqlite-net-pcl" Version="1.11.285" />
+        <PackageReference Include="ZstdSharp.Port" Version="0.8.8" />
     </ItemGroup>
 </Project>

ParadoxSaveParser.WebAPI/Program.cs

@@ -87,7 +87,7 @@ public static class Program
             await DB.InitializeAsync();
             await DB.DeleteInvalidData();
 
-            var bgJobManager = new BackgroundJobManager(loggerRoot);
+            var bgJobManager = new BackgroundJobManager(loggerRoot, config.SaveUpload);
             var saveFilters = new Dictionary<Game, ISaveDataFilter>
             {
                 { Game.EU4, new SaveDataFilterEU4() },
@@ -98,11 +98,13 @@ public static class Program
             router.DefaultRoute = new SimpleRouter.RouteWithMethod(HttpMethod.GET,
                 new ServeFilesRouteHandler(PathHelper.PUBLIC_DIR));
             router.MapRoute("/uploadSave", HttpMethod.POST,
-                new UploadSaveHandler(mainCancel.Token, bgJobManager, saveFilters));
+                new UploadSaveHandler(mainCancel.Token, bgJobManager, saveFilters, config.SaveUpload));
             router.MapRoute("/getSaveStatus", HttpMethod.GET,
                 new GetSaveStatusHandler(mainCancel.Token));
             router.MapRoute("/getSaveData", HttpMethod.GET,
                 new GetSaveDataHandler(mainCancel.Token));
+            router.MapRoute("/getSaveFile", HttpMethod.GET,
+                new GetSaveFileHandler(mainCancel.Token));
 
             var app = new WebApp(config.BaseUrl, loggerRoot, router, mainCancel.Token);
             await app.Run();

ParadoxSaveParser.WebAPI/README.md

@@ -5,30 +5,50 @@ Simple web application created using DTLib.Web.
 ## Important
 Restart web application once per day to delete outdated data and clean RAM.
 
+## Config
+`SaveUpload` section of `config.json`:
+- `MaxSizeCompressedBytes` - max size of uploaded `.tar.zst` (default 60 MiB)
+- `MaxSizePlainTextBytes` - max size of the game data entry after decompression (default 200 MiB)
+
 ## Routes
 ### POST `/uploadSave`
 - **Query Params:**
   - `game` - short name of the game (see [../README.md](../README.md))
-- **Request Body:** `application/octet-stream` - .eu4 file
+  - `originalFileType` - `text` | `zip`
+  - `originalFileName` - name of the original save file (max 255 characters)
+- **Request Body:** `application/zstd` - single zstd frame containing a tar with files of the original save.
+  Game data is read from the entry named in [GameSaveEntryNames](./Game.cs) (`gamestate` for EU4).
+  A plain text save is a tar with just that one entry.
+  The blob is stored byte-for-byte and can be downloaded with `/getSaveFile`.
 - **Response:**
     ```json
-    { "saveId": "string" }
+    { "id": "string" }
     ```
-    or `{ "errorMessage": "string" }`
+    or `{ "errorMessage": "string" }` 
+    (status 413 if upload or uncompressed game data exceeds the limits from config)
 
 ### GET `/getSaveStatus`
 - **Query Params:**
     - `id` - id of uploaded save file
-- **Response:** [SaveFileMetadata](./SaveFileMetadata.cs)
+- **Response:** [SaveFileMetadata](./Database/SaveFileMetadata.cs)
     ```json
     {
         "id": "string",
         "game": "string",
-        "status": "string",
+        "status": "initialized | uploaded | parsing | saving_results | done | error",
+        "uploadDateTime": "string",
+        "originalFileType": "text | zip",
+        "originalFileName": "string",
         "errorMessage": "string?"
     }
     ```
 
+### GET `/getSaveFile`
+- **Query Params:**
+    - `id` - id of uploaded save file
+- **Response:** `application/zstd` - uploaded blob, unchanged
+    or `{ "errorMessage": "string" }`
+
 ### GET `/getSaveData`
 - **Query Params:**
     - `id` - id of uploaded save file

ParadoxSaveParser.WebAPI/Routes/GetSaveFileHandler.cs

@@ -0,0 +1,45 @@
+using System.Net;
+using ParadoxSaveParser.WebAPI.Database;
+using ParadoxSaveParser.WebAPI.HttpHelpers;
+
+namespace ParadoxSaveParser.WebAPI.Routes;
+
+internal class GetSaveFileHandler : RouteHandlerBase
+{
+    public GetSaveFileHandler(CancellationToken cancelAllToken)
+        : base(cancelAllToken)
+    {
+    }
+
+    public override async Task<HttpStatusCode> HandleRequest(
+        HttpListenerContext ctx, ContextLogger requestLogger)
+    {
+        var idOrError = RequestHelper.GetQueryValue(ctx, "id");
+        if (idOrError.HasError)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, idOrError.Error!);
+        string id = idOrError.Value!;
+
+        var meta = await Program.DB.GetMetadata(id);
+        if(meta is null)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                new ErrorMessage(HttpStatusCode.NotFound,
+                    $"Save with id {id} not found"));
+        // file is being uploaded (Initialized) or was deleted (Error)
+        if(meta.status is SaveFileProcessingStatus.Initialized or SaveFileProcessingStatus.Error)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                new ErrorMessage(HttpStatusCode.BadRequest,
+                    $"Save with id {id} has status {meta.status}"));
+
+        IOPath saveFilePath = meta.GetSaveFilePath();
+        if (!File.Exists(saveFilePath))
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                new ErrorMessage(HttpStatusCode.InternalServerError,
+                    $"Save with id {id} not found")
+            );
+
+        await using var saveFile = File.OpenRead(saveFilePath);
+
+        return await ReturnHelper.ResponseStream(ctx, requestLogger, _cancelAllToken,
+            saveFile, contentType: "application/zstd");
+    }
+}

ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs

@@ -1,4 +1,4 @@
-using System.Linq;
+using System.IO;
 using System.Net;
 using ParadoxSaveParser.WebAPI.BackgroundTasks;
 using ParadoxSaveParser.WebAPI.Database;
@@ -9,24 +9,30 @@ namespace ParadoxSaveParser.WebAPI.Routes;
 
 public class UploadSaveHandler : RouteHandlerBase
 {
+    private const int MaxOriginalFileNameLength = 255;
+
     private readonly BackgroundJobManager _bgJobManager;
     private readonly Dictionary<Game, ISaveDataFilter> _saveFilters;
+    private readonly Config.SaveUploadConfig _uploadConfig;
 
     public UploadSaveHandler(
-        CancellationToken cancelAllToken, 
+        CancellationToken cancelAllToken,
         BackgroundJobManager bgJobManager,
-        Dictionary<Game, ISaveDataFilter> saveFilters) 
+        Dictionary<Game, ISaveDataFilter> saveFilters,
+        Config.SaveUploadConfig uploadConfig)
         : base(cancelAllToken)
     {
         _bgJobManager = bgJobManager;
         _saveFilters = saveFilters;
+        _uploadConfig = uploadConfig;
     }
 
     public override async Task<HttpStatusCode> HandleRequest(
         HttpListenerContext ctx, ContextLogger requestLogger)
     {
-        string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault();
-        if (contentType != "application/octet-stream")
+        string? contentType = ctx.Request.ContentType;
+        string? mediaType = contentType?.Split(';')[0].Trim();
+        if (!string.Equals(mediaType, "application/zstd", StringComparison.OrdinalIgnoreCase))
             return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
                     new ErrorMessage(HttpStatusCode.BadRequest,
                 $"Invalid request Content-Type: '{contentType}'"));
@@ -34,18 +40,82 @@ public class UploadSaveHandler : RouteHandlerBase
         var gameStrOrError = RequestHelper.GetQueryValue(ctx, "game");
         if (gameStrOrError.HasError)
             return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, gameStrOrError.Error!);
-
         if (!Enum.TryParse(gameStrOrError.Value, ignoreCase: true, out Game game))
             return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
                 new ErrorMessage(HttpStatusCode.BadRequest,
-                    $"Invalid requested game: '{gameStrOrError.Value}'"));  
-        var meta = await Program.DB.CreateMetadata(game);
+                    $"Invalid requested game: '{gameStrOrError.Value}'"));
+        if (!_saveFilters.ContainsKey(game) || !GameSaveEntryNames.Map.ContainsKey(game))
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                new ErrorMessage(HttpStatusCode.BadRequest,
+                    $"Game {game} is not supported"));
 
-        var saveFilePath = meta.GetSaveFilePath();
-        await using (var saveFile = File.OpenWrite(saveFilePath))
+        var fileTypeStrOrError = RequestHelper.GetQueryValue(ctx, "originalFileType");
+        if (fileTypeStrOrError.HasError)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, fileTypeStrOrError.Error!);
+        OriginalFileType? originalFileType = fileTypeStrOrError.Value!.ToLowerInvariant() switch
         {
-            await using (var remoteStream = ctx.Request.InputStream)
-                await remoteStream.CopyToAsync(saveFile, _cancelAllToken);
+            "text" => OriginalFileType.Text,
+            "zip" => OriginalFileType.Zip,
+            _ => null
+        };
+        if (originalFileType is null)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                new ErrorMessage(HttpStatusCode.BadRequest,
+                    $"Invalid originalFileType: '{fileTypeStrOrError.Value}'"));
+
+        var fileNameOrError = RequestHelper.GetQueryValue(ctx, "originalFileName");
+        if (fileNameOrError.HasError)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, fileNameOrError.Error!);
+        string originalFileName = fileNameOrError.Value!;
+        if (originalFileName.Length > MaxOriginalFileNameLength)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                new ErrorMessage(HttpStatusCode.BadRequest,
+                    $"originalFileName is longer than {MaxOriginalFileNameLength} characters"));
+
+        // Content-Length is unknown (-1) for chunked requests, so body size is checked during copying too
+        if (ctx.Request.ContentLength64 > _uploadConfig.MaxSizeCompressedBytes)
+            return await ResponseUploadTooLarge(ctx, requestLogger);
+
+        var meta = await Program.DB.CreateMetadata(game, originalFileType.Value, originalFileName);
+        try
+        {
+            bool fitsLimit;
+            await using (var saveFile = File.OpenWrite(meta.GetSaveFilePath()))
+            {
+                await using (var remoteStream = ctx.Request.InputStream)
+                    fitsLimit = await CopyWithLimit(remoteStream, saveFile, _uploadConfig.MaxSizeCompressedBytes);
+            }
+            if (!fitsLimit)
+            {
+                await Program.DB.DeleteMetadata(meta, "upload is too large");
+                return await ResponseUploadTooLarge(ctx, requestLogger);
+            }
+
+            long gameDataSize;
+            try
+            {
+                gameDataSize = await SaveArchive.GetGameDataSize(meta.GetSaveFilePath(), game, _cancelAllToken);
+            }
+            catch (Exception ex) when (ex is not OperationCanceledException)
+            {
+                await Program.DB.DeleteMetadata(meta, "invalid archive");
+                return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                    new ErrorMessage(HttpStatusCode.BadRequest,
+                        $"Invalid save archive: {ex.Message}"));
+            }
+            if (gameDataSize > _uploadConfig.MaxSizePlainTextBytes)
+            {
+                await Program.DB.DeleteMetadata(meta, "game data is too large");
+                return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                    new ErrorMessage(HttpStatusCode.RequestEntityTooLarge,
+                        $"Uncompressed game data size ({gameDataSize} bytes) exceeds limit "
+                        + $"({_uploadConfig.MaxSizePlainTextBytes} bytes)"));
+            }
+        }
+        catch
+        {
+            await Program.DB.DeleteMetadata(meta, "upload failed");
+            throw;
         }
         await Program.DB.UpdateMetadataStatus(meta, SaveFileProcessingStatus.Uploaded);
 
@@ -53,4 +123,27 @@ public class UploadSaveHandler : RouteHandlerBase
         dynamic responseData = new { meta.id };
         return await ReturnHelper.ResponseJson(ctx, requestLogger, _cancelAllToken, responseData);
     }
-}
+
+    private Task<HttpStatusCode> ResponseUploadTooLarge(HttpListenerContext ctx, ContextLogger requestLogger)
+    {
+        return ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+            new ErrorMessage(HttpStatusCode.RequestEntityTooLarge,
+                $"Upload size exceeds limit ({_uploadConfig.MaxSizeCompressedBytes} bytes)"));
+    }
+
+    /// <returns>false if source is longer than maxBytes</returns>
+    private async Task<bool> CopyWithLimit(Stream source, Stream dest, long maxBytes)
+    {
+        byte[] buffer = new byte[81920];
+        long totalBytes = 0;
+        int readBytes;
+        while ((readBytes = await source.ReadAsync(buffer, _cancelAllToken)) > 0)
+        {
+            totalBytes += readBytes;
+            if (totalBytes > maxBytes)
+                return false;
+            await dest.WriteAsync(buffer.AsMemory(0, readBytes), _cancelAllToken);
+        }
+        return true;
+    }
+}

ParadoxSaveParser.WebAPI/SaveArchive.cs

@@ -0,0 +1,52 @@
+using System.Formats.Tar;
+using System.IO;
+using ZstdSharp;
+
+namespace ParadoxSaveParser.WebAPI;
+
+/// <summary>
+/// Uploaded save is a zstd-compressed tar with files of the original save inside.
+/// Game data is stored in entry with name from <see cref="GameSaveEntryNames"/>.
+/// </summary>
+public static class SaveArchive
+{
+    public static Task<long> GetGameDataSize(IOPath archivePath, Game game, CancellationToken ct)
+    {
+        return WithGameDataEntry(archivePath, game, ct,
+            entry => Task.FromResult(entry.Length));
+    }
+
+    public static Task ExtractGameData(IOPath archivePath, Game game, Stream dest, long maxBytes,
+        CancellationToken ct)
+    {
+        return WithGameDataEntry(archivePath, game, ct, async entry =>
+        {
+            if (entry.Length > maxBytes)
+                throw new InvalidDataException(
+                    $"Game data size ({entry.Length} bytes) exceeds limit ({maxBytes} bytes)");
+            if (entry.DataStream is not null)
+                await entry.DataStream.CopyToAsync(dest, ct);
+            return true;
+        });
+    }
+
+    private static async Task<T> WithGameDataEntry<T>(IOPath archivePath, Game game, CancellationToken ct,
+        Func<TarEntry, Task<T>> action)
+    {
+        if (!GameSaveEntryNames.Map.TryGetValue(game, out string? entryName))
+            throw new ArgumentOutOfRangeException(nameof(game), game, "No game data entry name for game");
+
+        await using var file = File.OpenRead(archivePath);
+        await using var zstd = new DecompressionStream(file);
+        await using var tar = new TarReader(zstd);
+        while (await tar.GetNextEntryAsync(copyData: false, ct) is { } entry)
+        {
+            if (entry.EntryType is not (TarEntryType.RegularFile or TarEntryType.V7RegularFile))
+                continue;
+            if (System.IO.Path.GetFileName(entry.Name) == entryName)
+                return await action(entry);
+        }
+
+        throw new InvalidDataException($"Invalid save format: no '{entryName}' entry found");
+    }
+}