From 55f7d453b10bc3005ebfbec062b0d59491ea8392 Mon Sep 17 00:00:00 2001 From: Timerix Date: Wed, 23 Sep 2026 18:25:15 +0200 Subject: [PATCH] compress parsed data to .json.zst --- AGENTS.md | 42 +++++++++++-------- .../BackgroundTasks/BackgroundJobManager.cs | 6 ++- .../BackgroundTasks/SaveParsingOperation.cs | 11 ++++- ParadoxSaveParser.WebAPI/Config.cs | 22 ++++++++-- .../Database/SaveFileMetadata.cs | 2 +- ParadoxSaveParser.WebAPI/Program.cs | 2 +- ParadoxSaveParser.WebAPI/README.md | 11 ++--- .../Routes/GetSaveDataHandler.cs | 5 ++- TODO.md | 1 - 9 files changed, 67 insertions(+), 35 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cef3d2a..322e60d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,7 +51,7 @@ POST /uploadSave -> store blob, status "uploaded" -> SaveParsingOperation (background: parsing, saving_results, done) GET /getSaveStatus (polled by the frontend until status is "done" or "error") -GET /getSaveData (filtered parse result as JSON) +GET /getSaveData (filtered parse result as zstd-compressed JSON) GET /getSaveFile (the uploaded blob, unchanged) ``` @@ -79,11 +79,13 @@ archived save -> tar { "meta", "gamestate", ... } -> zstd -> upload [SaveArchive.cs](ParadoxSaveParser.WebAPI/SaveArchive.cs), which streams through `DecompressionStream` and `TarReader`. The whole archive is never loaded into memory. -- Upload limits come from the `SaveUpload` config section. - `MaxSizeCompressedBytes` is checked against `Content-Length` and then again - while the body is copied, because chunked requests have no length. - `MaxSizePlainTextBytes` is checked against the tar entry size once the upload - has finished. Every rejected upload deletes its database row and file. +- Upload limits come from the `SaveUpload` config section, in MiB + (`MaxSizeCompressedMegabytes`, `MaxSizePlainTextMegabytes`). The code uses + their byte equivalents, the `…Bytes` properties, which are not saved to the + config. The compressed limit is checked against `Content-Length` and then + again while the body is copied, because chunked requests have no length. The + plain text limit is checked against the tar entry size once the upload has + finished. Every rejected upload deletes its database row and file. ### Runtime files @@ -101,8 +103,11 @@ lands in the project folders. - **Nothing is deployed and there is no data worth keeping.** Do not write database migrations or backward compatibility code for old layouts; change the schema directly. -- Open items are in [TODO.md](TODO.md): compressing parsed results, - re-parsing when the query changes, automatic database cleanup. +- Open items are in [TODO.md](TODO.md): re-parsing when the query changes, + automatic database cleanup. +- Parsed results are stored as `.parsed.json.zstd` and `/getSaveData` + serves that file unchanged as `application/zstd`. The client decompresses + it; the server never does. - The WebAPI exits with code 1 after Ctrl+C even though it logs "server stopped". This has not been investigated yet. @@ -152,17 +157,20 @@ A one-off C# check can be a .NET 10 file-based app in `TEMP/` ### Checking your work -**Always build the solution and run the tests after changing code.** Do not -report a change as finished until both pass. +**Always build the solution after changing code.** `dotnet build +ParadoxSaveParser.sln` must finish with zero errors and no **new** warnings. +Three warnings are known and come from sqlite-net-pcl's +`SQLitePCLRaw.lib.e_sqlite3` dependency (NU1903 twice, NETSDK1206 once). +Everything else is yours. + +**Never run tests or benchmarks unless the user asks for it.** This is a +strict ban. It covers `dotnet test`, the benchmarks, and starting the WebAPI +to exercise routes with `curl`. When a change needs checking that way, say +what should be run and wait for the user to ask. When they do: -- `dotnet build ParadoxSaveParser.sln` must finish with zero errors and no - **new** warnings. Three warnings are known and come from - sqlite-net-pcl's `SQLitePCLRaw.lib.e_sqlite3` dependency (NU1903 twice, - NETSDK1206 once). Everything else is yours. - `dotnet test ParadoxSaveParser.Lib.Tests` must pass completely. -- A WebAPI change is not verified until it has been run: start the server - from `TEMP/` and exercise the route with `curl`, including the error cases. - Stop the server afterwards. +- For a WebAPI check, start the server from `TEMP/`, exercise the route with + `curl` including the error cases, then stop the server. - For a parser performance change, run the relevant benchmark before and after, and report both numbers. diff --git a/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs b/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs index c234c75..4b5e2ce 100644 --- a/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs +++ b/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs @@ -7,12 +7,14 @@ public class BackgroundJobManager { private readonly ILogger _parentLogger; private readonly Config.SaveUploadConfig _uploadConfig; + private readonly int _zstdCompressionLevel; private long _lastJobId; - public BackgroundJobManager(ILogger logger, Config.SaveUploadConfig uploadConfig) + public BackgroundJobManager(ILogger logger, Config.SaveUploadConfig uploadConfig, int zstdCompressionLevel) { _parentLogger = logger; _uploadConfig = uploadConfig; + _zstdCompressionLevel = zstdCompressionLevel; } public SaveParsingOperation StartNewParsingOperation( @@ -21,7 +23,7 @@ public class BackgroundJobManager long nextId = Interlocked.Increment(ref _lastJobId); var contextLogger = new ContextLogger($"BackgroundJob-{nextId}", _parentLogger); var op = new SaveParsingOperation(nextId, meta, filter, - _uploadConfig.MaxSizePlainTextBytes, contextLogger, ct); + _uploadConfig.MaxSizePlainTextBytes, _zstdCompressionLevel, contextLogger, ct); op.StartAsync(); return op; } diff --git a/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs b/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs index dc66b0c..96dce7d 100644 --- a/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs +++ b/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs @@ -3,6 +3,7 @@ using System.Text.Encodings.Web; using System.Text.Json.Serialization; using ParadoxSaveParser.WebAPI.Database; using ParadoxSaveParser.WebAPI.SaveDataFilters; +using ZstdSharp; namespace ParadoxSaveParser.WebAPI.BackgroundTasks; @@ -13,6 +14,7 @@ public class SaveParsingOperation private readonly SaveFileMetadata _meta; private readonly ISaveDataFilter _filter; private readonly long _maxGameDataBytes; + private readonly int _zstdCompressionLevel; private readonly ContextLogger _logger; private readonly CancellationToken _ct; @@ -25,12 +27,13 @@ public class SaveParsingOperation }; public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISaveDataFilter filter, - long maxGameDataBytes, ContextLogger logger, CancellationToken ct) + long maxGameDataBytes, int zstdCompressionLevel, ContextLogger logger, CancellationToken ct) { OperationId = operationId; _meta = meta; _filter = filter; _maxGameDataBytes = maxGameDataBytes; + _zstdCompressionLevel = zstdCompressionLevel; _logger = logger; _ct = ct; } @@ -90,7 +93,11 @@ public class SaveParsingOperation await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.SavingResults); var resultFilePath = _meta.GetParsedDataPath(); await using (var resultFile = File.OpenWrite(resultFilePath)) - await JsonSerializer.SerializeAsync(resultFile, parsedData, _jsonOptions, _ct); + { + // the zstd frame is finished only when compression stream is disposed, so it must happen before Done + await using var zstd = new CompressionStream(resultFile, _zstdCompressionLevel); + await JsonSerializer.SerializeAsync(zstd, parsedData, _jsonOptions, _ct); + } await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.Done); } } \ No newline at end of file diff --git a/ParadoxSaveParser.WebAPI/Config.cs b/ParadoxSaveParser.WebAPI/Config.cs index f000b0f..4a95101 100644 --- a/ParadoxSaveParser.WebAPI/Config.cs +++ b/ParadoxSaveParser.WebAPI/Config.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using ZstdSharp; namespace ParadoxSaveParser.WebAPI; @@ -11,13 +12,21 @@ public class Config public int UploadLifetimeDays { get; set; } = 30; public string DbPath { get; set; } = "database.sqlite"; public SaveUploadConfig SaveUpload { get; set; } = new(); + /// level used to compress parsed save data + public int ZstdCompressionLevel { get; set; } = 10; public class SaveUploadConfig { - /// max size of game data entry after decompression - public long MaxSizePlainTextBytes { get; set; } = 200 * 1024 * 1024; - /// max size of uploaded .tar.zst - public long MaxSizeCompressedBytes { get; set; } = 60 * 1024 * 1024; + /// max size of game data entry after decompression, in MiB + public int MaxSizePlainTextMegabytes { get; set; } = 200; + /// max size of uploaded .tar.zst, in MiB + public int MaxSizeCompressedMegabytes { get; set; } = 60; + + // long, because int overflows at 2048 MiB + [JsonIgnore] + public long MaxSizePlainTextBytes => MaxSizePlainTextMegabytes * 1024L * 1024L; + [JsonIgnore] + public long MaxSizeCompressedBytes => MaxSizeCompressedMegabytes * 1024L * 1024L; } public static JsonSerializerOptions JsonOptions = new() @@ -35,6 +44,11 @@ public class Config throw new Exception($"config is obsolete (config v{cfg.Version} < program v{ActualVersion})"); if (cfg.Version > ActualVersion) throw new Exception($"program is obsolete (config v{cfg.Version} > program v{ActualVersion})"); + // fail on start instead of on first parsed save + if (cfg.ZstdCompressionLevel < Compressor.MinCompressionLevel + || cfg.ZstdCompressionLevel > Compressor.MaxCompressionLevel) + throw new Exception($"{nameof(ZstdCompressionLevel)} must be in range " + + $"[{Compressor.MinCompressionLevel}, {Compressor.MaxCompressionLevel}]"); return cfg; } diff --git a/ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs b/ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs index 782741e..c948158 100644 --- a/ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs +++ b/ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs @@ -51,7 +51,7 @@ public class SaveFileMetadata public IOPath GetSaveFilePath() => Path.Concat(PathHelper.SAVES_DIR, id + ".tar.zst"); - public IOPath GetParsedDataPath() => Path.Concat(PathHelper.PARSED_DIR, id + ".parsed.json"); + public IOPath GetParsedDataPath() => Path.Concat(PathHelper.PARSED_DIR, id + ".parsed.json.zstd"); public bool AssociatedFilesExist() { diff --git a/ParadoxSaveParser.WebAPI/Program.cs b/ParadoxSaveParser.WebAPI/Program.cs index 468bf53..17d28ca 100644 --- a/ParadoxSaveParser.WebAPI/Program.cs +++ b/ParadoxSaveParser.WebAPI/Program.cs @@ -87,7 +87,7 @@ public static class Program await DB.InitializeAsync(); await DB.DeleteInvalidData(); - var bgJobManager = new BackgroundJobManager(loggerRoot, config.SaveUpload); + var bgJobManager = new BackgroundJobManager(loggerRoot, config.SaveUpload, config.ZstdCompressionLevel); var saveFilters = new Dictionary { { Game.EU4, new SaveDataFilterEU4() }, diff --git a/ParadoxSaveParser.WebAPI/README.md b/ParadoxSaveParser.WebAPI/README.md index 815394b..ea80dfa 100644 --- a/ParadoxSaveParser.WebAPI/README.md +++ b/ParadoxSaveParser.WebAPI/README.md @@ -6,9 +6,11 @@ Simple web application created using DTLib.Web. Restart web application once per day to delete outdated data and clean RAM. ## Config +`ZstdCompressionLevel` in `config.json` - zstd level of stored parsed save data (default 10) + `SaveUpload` section of `config.json`: -- `MaxSizeCompressedBytes` - max size of uploaded `.tar.zst` (default 60 MiB) -- `MaxSizePlainTextBytes` - max size of the game data entry after decompression (default 200 MiB) +- `MaxSizeCompressedMegabytes` - max size of uploaded `.tar.zst` in MiB (default 60) +- `MaxSizePlainTextMegabytes` - max size of the game data entry after decompression in MiB (default 200) ## Routes ### POST `/uploadSave` @@ -52,12 +54,11 @@ Restart web application once per day to delete outdated data and clean RAM. ### GET `/getSaveData` - **Query Params:** - `id` - id of uploaded save file -- **Response:** - ```json5 +- **Response:** `application/zstd` - single zstd frame containing json: + ```json { "key0": [ "objects" ], "key1": [ "objects" ], - //... } ``` or `{ "errorMessage": "string" }` diff --git a/ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs b/ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs index 1b2dd4d..acb223c 100644 --- a/ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs +++ b/ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs @@ -36,9 +36,10 @@ internal class GetSaveDataHandler : RouteHandlerBase $"Save with id {id} not found") ); + // json is stored compressed and is sent as is, the client decompresses it await using var dataFile = File.OpenRead(dataFilePath); - + return await ReturnHelper.ResponseStream(ctx, requestLogger, _cancelAllToken, - dataFile, contentType: "application/json"); + dataFile, contentType: "application/zstd"); } } \ No newline at end of file diff --git a/TODO.md b/TODO.md index 112d174..8e681b5 100644 --- a/TODO.md +++ b/TODO.md @@ -1,4 +1,3 @@ ## WebAPI.SaveParsingOperation: - Save parsed data in zstd archive Re-parse if saved data was parsed with another query Implement automatic database cleanup \ No newline at end of file