compress parsed data to .json.zst

ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs

@@ -7,12 +7,14 @@ public class BackgroundJobManager
 {
     private readonly ILogger _parentLogger;
     private readonly Config.SaveUploadConfig _uploadConfig;
+    private readonly int _zstdCompressionLevel;
     private long _lastJobId;
 
-    public BackgroundJobManager(ILogger logger, Config.SaveUploadConfig uploadConfig)
+    public BackgroundJobManager(ILogger logger, Config.SaveUploadConfig uploadConfig, int zstdCompressionLevel)
     {
         _parentLogger = logger;
         _uploadConfig = uploadConfig;
+        _zstdCompressionLevel = zstdCompressionLevel;
     }
     
     public SaveParsingOperation StartNewParsingOperation(
@@ -21,7 +23,7 @@ public class BackgroundJobManager
         long nextId = Interlocked.Increment(ref _lastJobId);
         var contextLogger = new ContextLogger($"BackgroundJob-{nextId}", _parentLogger);
         var op = new SaveParsingOperation(nextId, meta, filter,
-            _uploadConfig.MaxSizePlainTextBytes, contextLogger, ct);
+            _uploadConfig.MaxSizePlainTextBytes, _zstdCompressionLevel, contextLogger, ct);
         op.StartAsync();
         return op;
     }

ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs

@@ -3,6 +3,7 @@ using System.Text.Encodings.Web;
 using System.Text.Json.Serialization;
 using ParadoxSaveParser.WebAPI.Database;
 using ParadoxSaveParser.WebAPI.SaveDataFilters;
+using ZstdSharp;
 
 namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
 
@@ -13,6 +14,7 @@ public class SaveParsingOperation
     private readonly SaveFileMetadata _meta;
     private readonly ISaveDataFilter _filter;
     private readonly long _maxGameDataBytes;
+    private readonly int _zstdCompressionLevel;
     private readonly ContextLogger _logger;
     private readonly CancellationToken _ct;
 
@@ -25,12 +27,13 @@ public class SaveParsingOperation
     };
 
     public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISaveDataFilter filter,
-        long maxGameDataBytes, ContextLogger logger, CancellationToken ct)
+        long maxGameDataBytes, int zstdCompressionLevel, ContextLogger logger, CancellationToken ct)
     {
         OperationId = operationId;
         _meta = meta;
         _filter = filter;
         _maxGameDataBytes = maxGameDataBytes;
+        _zstdCompressionLevel = zstdCompressionLevel;
         _logger = logger;
         _ct = ct;
     }
@@ -90,7 +93,11 @@ public class SaveParsingOperation
         await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.SavingResults);
         var resultFilePath = _meta.GetParsedDataPath();
         await using (var resultFile = File.OpenWrite(resultFilePath))
-            await JsonSerializer.SerializeAsync(resultFile, parsedData, _jsonOptions, _ct);
+        {
+            // the zstd frame is finished only when compression stream is disposed, so it must happen before Done
+            await using var zstd = new CompressionStream(resultFile, _zstdCompressionLevel);
+            await JsonSerializer.SerializeAsync(zstd, parsedData, _jsonOptions, _ct);
+        }
         await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.Done);
     }
 }

ParadoxSaveParser.WebAPI/Config.cs

@@ -1,4 +1,5 @@
 using System.Text.Json.Serialization;
+using ZstdSharp;
 
 namespace ParadoxSaveParser.WebAPI;
 
@@ -11,13 +12,21 @@ public class Config
     public int UploadLifetimeDays { get; set; } = 30;
     public string DbPath { get; set; } = "database.sqlite";
     public SaveUploadConfig SaveUpload { get; set; } = new();
+    /// level used to compress parsed save data
+    public int ZstdCompressionLevel { get; set; } = 10;
 
     public class SaveUploadConfig
     {
-        /// max size of game data entry after decompression
-        public long MaxSizePlainTextBytes { get; set; } = 200 * 1024 * 1024;
-        /// max size of uploaded .tar.zst
-        public long MaxSizeCompressedBytes { get; set; } = 60 * 1024 * 1024;
+        /// max size of game data entry after decompression, in MiB
+        public int MaxSizePlainTextMegabytes { get; set; } = 200;
+        /// max size of uploaded .tar.zst, in MiB
+        public int MaxSizeCompressedMegabytes { get; set; } = 60;
+
+        // long, because int overflows at 2048 MiB
+        [JsonIgnore]
+        public long MaxSizePlainTextBytes => MaxSizePlainTextMegabytes * 1024L * 1024L;
+        [JsonIgnore]
+        public long MaxSizeCompressedBytes => MaxSizeCompressedMegabytes * 1024L * 1024L;
     }
 
     public static JsonSerializerOptions JsonOptions = new()
@@ -35,6 +44,11 @@ public class Config
             throw new Exception($"config is obsolete (config v{cfg.Version} < program v{ActualVersion})");
         if (cfg.Version > ActualVersion)
             throw new Exception($"program is obsolete (config v{cfg.Version} > program v{ActualVersion})");
+        // fail on start instead of on first parsed save
+        if (cfg.ZstdCompressionLevel < Compressor.MinCompressionLevel
+            || cfg.ZstdCompressionLevel > Compressor.MaxCompressionLevel)
+            throw new Exception($"{nameof(ZstdCompressionLevel)} must be in range "
+                + $"[{Compressor.MinCompressionLevel}, {Compressor.MaxCompressionLevel}]");
         return cfg;
     }
 

ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs

@@ -51,7 +51,7 @@ public class SaveFileMetadata
 
     public IOPath GetSaveFilePath() => Path.Concat(PathHelper.SAVES_DIR, id + ".tar.zst");
 
-    public IOPath GetParsedDataPath() => Path.Concat(PathHelper.PARSED_DIR, id + ".parsed.json");
+    public IOPath GetParsedDataPath() => Path.Concat(PathHelper.PARSED_DIR, id + ".parsed.json.zstd");
 
     public bool AssociatedFilesExist()
     {

ParadoxSaveParser.WebAPI/Program.cs

@@ -87,7 +87,7 @@ public static class Program
             await DB.InitializeAsync();
             await DB.DeleteInvalidData();
 
-            var bgJobManager = new BackgroundJobManager(loggerRoot, config.SaveUpload);
+            var bgJobManager = new BackgroundJobManager(loggerRoot, config.SaveUpload, config.ZstdCompressionLevel);
             var saveFilters = new Dictionary<Game, ISaveDataFilter>
             {
                 { Game.EU4, new SaveDataFilterEU4() },

ParadoxSaveParser.WebAPI/README.md

@@ -6,9 +6,11 @@ Simple web application created using DTLib.Web.
 Restart web application once per day to delete outdated data and clean RAM.
 
 ## Config
+`ZstdCompressionLevel` in `config.json` - zstd level of stored parsed save data (default 10)
+
 `SaveUpload` section of `config.json`:
-- `MaxSizeCompressedBytes` - max size of uploaded `.tar.zst` (default 60 MiB)
-- `MaxSizePlainTextBytes` - max size of the game data entry after decompression (default 200 MiB)
+- `MaxSizeCompressedMegabytes` - max size of uploaded `.tar.zst` in MiB (default 60)
+- `MaxSizePlainTextMegabytes` - max size of the game data entry after decompression in MiB (default 200)
 
 ## Routes
 ### POST `/uploadSave`
@@ -52,12 +54,11 @@ Restart web application once per day to delete outdated data and clean RAM.
 ### GET `/getSaveData`
 - **Query Params:**
     - `id` - id of uploaded save file
-- **Response:**
-    ```json5
+- **Response:** `application/zstd` - single zstd frame containing json:
+    ```json
     {
         "key0": [ "objects" ],
         "key1": [ "objects" ],
-        //...
     }
     ```
     or `{ "errorMessage": "string" }`

ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs

@@ -36,9 +36,10 @@ internal class GetSaveDataHandler : RouteHandlerBase
                     $"Save with id {id} not found")
             );
 
+        // json is stored compressed and is sent as is, the client decompresses it
         await using var dataFile = File.OpenRead(dataFilePath);
-        
+
         return await ReturnHelper.ResponseStream(ctx, requestLogger, _cancelAllToken,
-            dataFile, contentType: "application/json");
+            dataFile, contentType: "application/zstd");
     }
 }