From c6b27cd3c9e0df89b686b0162400819a359a9fcd Mon Sep 17 00:00:00 2001 From: Timerix Date: Wed, 23 Sep 2026 17:48:42 +0200 Subject: [PATCH] updated to accept only .tar.zstd uploads --- .gitignore | 5 +- AGENTS.md | 256 ++++++++++++++++++ .../ParadoxSaveParser.Lib.Tests.csproj | 2 +- .../BackgroundTasks/BackgroundJobManager.cs | 7 +- .../BackgroundTasks/SaveParsingOperation.cs | 19 +- ParadoxSaveParser.WebAPI/Config.cs | 9 + .../Database/DatabaseConnector.cs | 7 +- .../Database/SaveFileMetadata.cs | 29 +- ParadoxSaveParser.WebAPI/Game.cs | 15 +- .../ParadoxSaveParser.WebAPI.csproj | 5 +- ParadoxSaveParser.WebAPI/Program.cs | 6 +- ParadoxSaveParser.WebAPI/README.md | 30 +- .../Routes/GetSaveFileHandler.cs | 45 +++ .../Routes/UploadSaveHandler.cs | 119 +++++++- ParadoxSaveParser.WebAPI/SaveArchive.cs | 52 ++++ 15 files changed, 556 insertions(+), 50 deletions(-) create mode 100644 AGENTS.md create mode 100644 ParadoxSaveParser.WebAPI/Routes/GetSaveFileHandler.cs create mode 100644 ParadoxSaveParser.WebAPI/SaveArchive.cs diff --git a/.gitignore b/.gitignore index dd2ca48..85c7ec8 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,7 @@ config.dtsod *.user #backups -.old*/ \ No newline at end of file +.old*/ + +# temporary files +TEMP/ \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..cef3d2a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,256 @@ +# AGENTS.md + +Guidance for AI coding agents working in this repository. + +## Project + +A parser for Paradox game save files, and the backend of a save analytics +website. Only EU4 is supported so far. + +- **.NET 10**, C# `latest`, nullable reference types enabled everywhere +- **DTLib** / **DTLib.Web** for filesystem helpers, logging and the HTTP server +- **sqlite-net-pcl** for the upload database, **ZstdSharp.Port** for zstd + +Projects in the solution ([ParadoxSaveParser.sln](ParadoxSaveParser.sln)): + +- `ParadoxSaveParser.Lib`: the parser itself (`Tokenizer`, `SaveParserEU4`, + `SearchExpression/`) +- `ParadoxSaveParser.Lib.Tests`: NUnit tests for the parser +- `ParadoxSaveParser.CLI`: command line tool, with an interactive mode +- `ParadoxSaveParser.WebAPI`: HTTP backend for the website +- `ParadoxSaveParser.Benchmarks`: BenchmarkDotNet comparison against regex + engines and jq, see [its README](ParadoxSaveParser.Benchmarks/README.md) + +## Architecture + +### Parser + +`SaveParserEU4` reads the plain text game state and produces nested +`Dictionary` values. It takes a compiled `SearchExpression`, +a small path query such as `countries.*.technology`. **The query is what +makes the parser fast.** `Tokenizer.SkipBlock` throws away a rejected `{...}` +block by scanning raw bytes for brace depth, so rejected data is never turned +into tokens or strings. Full parse: 1.2 s and 962 MB allocated; the same file +with a query: 50 ms and 1.5 MB. Keep any parser change from bringing back +per-byte virtual calls or eager string decoding. Check with the benchmarks +before and after. + +### WebAPI + +The frontend is a **separate Next.js project in its own repository**. It calls +this service through a same-origin proxy and expects it at +`http://127.0.0.1:5226` (`BaseUrl` in `config.json`). Routes are documented +in [ParadoxSaveParser.WebAPI/README.md](ParadoxSaveParser.WebAPI/README.md); +keep that file in sync when a route or a response shape changes, because the +frontend is written against it. + +The flow is upload, poll, fetch: + +``` +POST /uploadSave -> store blob, status "uploaded" -> SaveParsingOperation + (background: parsing, + saving_results, done) +GET /getSaveStatus (polled by the frontend until status is "done" or "error") +GET /getSaveData (filtered parse result as JSON) +GET /getSaveFile (the uploaded blob, unchanged) +``` + +### Save files on the wire + +The server is a cheap two core machine with little disk space, so the browser +re-compresses every save before uploading it: + +``` +plain text save -> tar { "gamestate" } -> zstd -> upload +archived save -> tar { "meta", "gamestate", ... } -> zstd -> upload +``` + +- `.tar.zst` is **the only accepted format.** Zip handling was removed on + purpose; do not add it back. The original type and name come in as query + params (`originalFileType=text|zip`, `originalFileName`), and they are stored + so the browser can rebuild the original file from `/getSaveFile`. A + rebuilt EU4 zip was checked entry by entry against the original and is byte + identical. +- **The blob is stored byte-for-byte.** Never re-encode it on the server. +- The game data entry name comes from `GameSaveEntryNames.Map` in + [Game.cs](ParadoxSaveParser.WebAPI/Game.cs). That map mirrors the frontend's + `GAME_SAVE_ENTRY_NAMES`. Do not hard-code `"gamestate"`. +- All tar and zstd handling lives in + [SaveArchive.cs](ParadoxSaveParser.WebAPI/SaveArchive.cs), which streams + through `DecompressionStream` and `TarReader`. The whole archive is never + loaded into memory. +- Upload limits come from the `SaveUpload` config section. + `MaxSizeCompressedBytes` is checked against `Content-Length` and then again + while the body is copied, because chunked requests have no length. + `MaxSizePlainTextBytes` is checked against the tar entry size once the upload + has finished. Every rejected upload deletes its database row and file. + +### Runtime files + +The WebAPI creates everything relative to its **working directory**: +`config.json` (written with defaults if missing), `database.sqlite`, `logs/`, +`temp/` (emptied on start), and `public/data/{saves,parsed}/`. When running it +for a test, start it with `TEMP/` as the working directory, so none of this +lands in the project folders. + +### Current status + +- **Only EU4 works.** `Game.Stellaris` exists and has an entry name, but it has + no `ISaveDataFilter` and no parser, so uploads for it are rejected with 400. + `SaveParsingOperation` is EU4 specific. +- **Nothing is deployed and there is no data worth keeping.** Do not write + database migrations or backward compatibility code for old layouts; change + the schema directly. +- Open items are in [TODO.md](TODO.md): compressing parsed results, + re-parsing when the query changes, automatic database cleanup. +- The WebAPI exits with code 1 after Ctrl+C even though it logs "server + stopped". This has not been investigated yet. + +## Tools and files + +**Prefer the editor's own tools over shell commands that do the same thing.** +Reading, editing, writing and searching files have dedicated tools. Using +`cat`, `sed`, `grep` or a throwaway script instead does the same work, +spends more tokens, and may need its own approval each time. Use the shell for +what only the shell can do: building, running tests, git, `curl`, `tar`, +`zstd`, or running the program itself. + +**Work inside the project directory.** For temporary files (scratch scripts, +intermediate output, test data, a running WebAPI's working directory), create +a `TEMP/` directory in the project root and keep everything there. It is +gitignored. Delete that directory once the work is done. Ask before reading or +writing anything outside the project directory, including `/tmp` and the home +folder. A save file the user names explicitly may be read, but never modified. + +## Terminal + +**On Windows, prefer `bash.exe`**, but only a real MSYS2, Cygwin, or +MinGW/Git Bash installation, for example `D:\msys2\usr\bin\bash.exe`. + +**Never use `C:\Windows\System32\bash.exe`.** Despite its name, it is not a +POSIX bash but the launcher stub for WSL. Do not start WSL to run ordinary +commands. + +If no MSYS/Cygwin/MinGW bash is installed, **use PowerShell** as the terminal. +PowerShell is the fallback, never WSL. + +If bash reports `command not found` for `dotnet`, the cause is `PATH`, not a +missing install. MSYS2 sets `MSYS2_PATH_TYPE` to `minimal` by default, which +drops the Windows `PATH`. Setting it to `inherit` fixes it. + +## Commands + +```sh +dotnet build ParadoxSaveParser.sln # build everything +dotnet test ParadoxSaveParser.Lib.Tests # parser tests +dotnet run --project ParadoxSaveParser.WebAPI # run the backend (mind the working directory) +dotnet build ParadoxSaveParser.Benchmarks -c Release # see its README for running +``` + +A one-off C# check can be a .NET 10 file-based app in `TEMP/` +(`dotnet run -c Release TEMP/check.cs`); it doesn't need a project. + +### Checking your work + +**Always build the solution and run the tests after changing code.** Do not +report a change as finished until both pass. + +- `dotnet build ParadoxSaveParser.sln` must finish with zero errors and no + **new** warnings. Three warnings are known and come from + sqlite-net-pcl's `SQLitePCLRaw.lib.e_sqlite3` dependency (NU1903 twice, + NETSDK1206 once). Everything else is yours. +- `dotnet test ParadoxSaveParser.Lib.Tests` must pass completely. +- A WebAPI change is not verified until it has been run: start the server + from `TEMP/` and exercise the route with `curl`, including the error cases. + Stop the server afterwards. +- For a parser performance change, run the relevant benchmark before and + after, and report both numbers. + +## Version control + +This project is managed with **git**. + +**Read-only git commands are fine to use without asking.** +You can run `git status`, `git log`, `git diff`, `git show`, `git blame` and +similar inspection commands freely. + +**Never run a git command that changes the repository without the user's +explicit permission.** +This is a strict ban. It covers every command that writes to the history, the +index, or the working tree, including `add`, `commit`, `checkout`, `restore`, +`reset`, `stash`, `branch`, `merge`, `rebase`, `push`, and `rm`. + +Ask first, and wait for a clear yes before running such a command. + +## Conventions + +Match the surrounding code exactly: + +- File-scoped namespaces, 4 spaces, `_camelCase` private fields, `PascalCase` + for everything else. +- **`SaveFileMetadata` properties are lowercase on purpose.** They are the + JSON wire format that the frontend reads, and the sqlite column names too. + Do not rename them to PascalCase. Enums on the wire serialize in lowercase + through `LowerCaseEnumConverter` (`SavingResults` becomes + `saving_results`). +- `Program.cs` declares global usings, which make `File`, `Directory` and + `Path` refer to **DTLib.Filesystem**, not `System.IO`. Paths are `IOPath`. + When a `System.IO` member is needed, write it out in full, for example + `System.IO.Path.GetFileName`. +- Errors from route handlers go through `ReturnHelper.ResponseError` with an + `ErrorMessage` and the right status code. Query values come from + `RequestHelper.GetQueryValue`. Do not write responses by hand. +- **Line endings:** most `.cs` files are CRLF with a UTF-8 BOM. A new file + written with LF mixes endings once the IDE touches it. Match the files next + to it, or convert with `unix2dos -m`. + +**Comment the code you write.** Human programmers who did not write this code +will read it, and a wall of uncommented agent-written code is expensive to +review. Keep comments simple, preferably one line, placed above the code they +describe, and in plain language rather than in terms of the types. Say _why_ +the code does what it does: what is non-obvious, and what would break if it +were written the obvious way. Leave out what the code already states plainly. +A tricky constant, a workaround for someone else's bug, or an ordering that +matters always deserves a line. + +Write doc comments as XML doc comments (`///`), which the IDE shows on hover. +Plain `//` comments do not appear in tooltips. + +## Dependencies + +**Search for an existing library before writing your own implementation.** +Formats, protocols and codecs are full of details that only show up against +real data, and a maintained library has already met them. Check the BCL first: +tar (`System.Formats.Tar`), zip, JSON and vectorized span operations are all +built in. .NET 10 has no zstd, which is why ZstdSharp.Port is here. + +**Read the library's source before adding it, and judge it on that.** Download +counts say nothing about quality. Look for readable sources, comments that +explain intent, and an API that fits the way the code here is written. Code +that nobody can read cannot be maintained and may hide bugs nobody can find. +Do not depend on it. + +If nothing of good quality exists, say so and propose writing an +implementation here, rather than pulling in the least bad option. + +Do not add a second library for a job a package already referenced can do. A +thin wrapper that shapes an installed library's API to what the code needs is +better than a new dependency. + +## Things that will bite you + +- **Changing a route's request or response is a two-repository change.** The + frontend has its own copy of the protocol in `SaveParserBackendService.ts`, + `GAME_SAVE_ENTRY_NAMES` and its save archive code. Say what the frontend + needs changed; do not assume it will follow. +- **`Config.Load` rejects any config whose `Version` differs from + `ActualVersion`.** New config properties with defaults don't need a version + bump, because a missing key keeps the default. Bump the version only for + changes that old files cannot express. +- **`PathHelper.CreateTempFile` deletes the file right after opening it.** The + open stream keeps the data alive until it is disposed; do not reopen it by + path. +- **`SaveParsingOperation.StartAsync` is `async void`.** Every exception must + be caught inside it, or it will crash the process. Errors are recorded on + the metadata row with `SetMetadataError`. +- `Program.IsDebug` is always true for now, so debug logging is always on. diff --git a/ParadoxSaveParser.Lib.Tests/ParadoxSaveParser.Lib.Tests.csproj b/ParadoxSaveParser.Lib.Tests/ParadoxSaveParser.Lib.Tests.csproj index 535b177..13f0619 100644 --- a/ParadoxSaveParser.Lib.Tests/ParadoxSaveParser.Lib.Tests.csproj +++ b/ParadoxSaveParser.Lib.Tests/ParadoxSaveParser.Lib.Tests.csproj @@ -24,7 +24,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + all diff --git a/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs b/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs index 8f0ce0a..c234c75 100644 --- a/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs +++ b/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs @@ -6,11 +6,13 @@ namespace ParadoxSaveParser.WebAPI.BackgroundTasks; public class BackgroundJobManager { private readonly ILogger _parentLogger; + private readonly Config.SaveUploadConfig _uploadConfig; private long _lastJobId; - public BackgroundJobManager(ILogger logger) + public BackgroundJobManager(ILogger logger, Config.SaveUploadConfig uploadConfig) { _parentLogger = logger; + _uploadConfig = uploadConfig; } public SaveParsingOperation StartNewParsingOperation( @@ -18,7 +20,8 @@ public class BackgroundJobManager { long nextId = Interlocked.Increment(ref _lastJobId); var contextLogger = new ContextLogger($"BackgroundJob-{nextId}", _parentLogger); - var op = new SaveParsingOperation(nextId, meta, filter, contextLogger, ct); + var op = new SaveParsingOperation(nextId, meta, filter, + _uploadConfig.MaxSizePlainTextBytes, contextLogger, ct); op.StartAsync(); return op; } diff --git a/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs b/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs index 0ffc380..dc66b0c 100644 --- a/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs +++ b/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs @@ -1,6 +1,4 @@ using System.IO; -using System.IO.Compression; -using System.Linq; using System.Text.Encodings.Web; using System.Text.Json.Serialization; using ParadoxSaveParser.WebAPI.Database; @@ -14,6 +12,7 @@ public class SaveParsingOperation private readonly SaveFileMetadata _meta; private readonly ISaveDataFilter _filter; + private readonly long _maxGameDataBytes; private readonly ContextLogger _logger; private readonly CancellationToken _ct; @@ -26,11 +25,12 @@ public class SaveParsingOperation }; public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISaveDataFilter filter, - ContextLogger logger, CancellationToken ct) + long maxGameDataBytes, ContextLogger logger, CancellationToken ct) { OperationId = operationId; _meta = meta; _filter = filter; + _maxGameDataBytes = maxGameDataBytes; _logger = logger; _ct = ct; } @@ -76,17 +76,10 @@ public class SaveParsingOperation Dictionary parsedData; await using (var gamestateStream = PathHelper.CreateTempFile()) { - // TODO: ckeck if it is zip archive or plain text like in Search.cs - using (var zipArchive = ZipFile.Open(_meta.GetSaveFilePath().Str, ZipArchiveMode.Read)) - { - var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate"); - if (zipEntry is null) - throw new Exception("Invalid save format: no 'gamestate' file found"); + await SaveArchive.ExtractGameData(_meta.GetSaveFilePath(), _meta.game, gamestateStream, + _maxGameDataBytes, _ct); + gamestateStream.Seek(0, SeekOrigin.Begin); - await zipEntry.Open().CopyToAsync(gamestateStream, _ct); - gamestateStream.Seek(0, SeekOrigin.Begin); - } - await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.Parsing); var parser = new SaveParserEU4(gamestateStream, _filter.SearchExpression); parsedData = parser.Parse(); diff --git a/ParadoxSaveParser.WebAPI/Config.cs b/ParadoxSaveParser.WebAPI/Config.cs index 3520554..f000b0f 100644 --- a/ParadoxSaveParser.WebAPI/Config.cs +++ b/ParadoxSaveParser.WebAPI/Config.cs @@ -10,6 +10,15 @@ public class Config public string BaseUrl { get; set; } = "http://127.0.0.1:5226/"; public int UploadLifetimeDays { get; set; } = 30; public string DbPath { get; set; } = "database.sqlite"; + public SaveUploadConfig SaveUpload { get; set; } = new(); + + public class SaveUploadConfig + { + /// max size of game data entry after decompression + public long MaxSizePlainTextBytes { get; set; } = 200 * 1024 * 1024; + /// max size of uploaded .tar.zst + public long MaxSizeCompressedBytes { get; set; } = 60 * 1024 * 1024; + } public static JsonSerializerOptions JsonOptions = new() { diff --git a/ParadoxSaveParser.WebAPI/Database/DatabaseConnector.cs b/ParadoxSaveParser.WebAPI/Database/DatabaseConnector.cs index 94643f4..608eb69 100644 --- a/ParadoxSaveParser.WebAPI/Database/DatabaseConnector.cs +++ b/ParadoxSaveParser.WebAPI/Database/DatabaseConnector.cs @@ -28,14 +28,17 @@ public class DatabaseConnector .FirstOrDefault(); } - public async Task CreateMetadata(Game game) + public async Task CreateMetadata(Game game, + OriginalFileType originalFileType, string originalFileName) { var meta = new SaveFileMetadata { id = Guid.NewGuid().ToString(), game = game, status = SaveFileProcessingStatus.Initialized, - uploadDateTime = DateTime.Now + uploadDateTime = DateTime.Now, + originalFileType = originalFileType, + originalFileName = originalFileName, }; await _db.InsertAsync(meta); return meta; diff --git a/ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs b/ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs index 0a6a635..782741e 100644 --- a/ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs +++ b/ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs @@ -12,6 +12,14 @@ public enum SaveFileProcessingStatus Error, } +public enum OriginalFileType +{ + Text, + Zip, +} + +public class LowerCaseEnumConverter() : JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) where T : struct, Enum; + public class SaveFileMetadata { [SQLite.PrimaryKey] @@ -22,19 +30,26 @@ public class SaveFileMetadata [SQLite.NotNull] public Game game { get; init; } - [JsonConverter(typeof(JsonStringEnumConverter))] + [JsonConverter(typeof(LowerCaseEnumConverter))] [SQLite.NotNull] public SaveFileProcessingStatus status { get; set; } - + [SQLite.NotNull] public DateTime uploadDateTime { get; set; } - + + [JsonConverter(typeof(LowerCaseEnumConverter))] + [SQLite.NotNull] + public OriginalFileType originalFileType { get; init; } + + [SQLite.NotNull] + public string originalFileName { get; init; } = null!; + // if error occured during parsing, it's message is saved here // status stays the same as when error occured public string? errorMessage { get; set; } - - - public IOPath GetSaveFilePath() => Path.Concat(PathHelper.SAVES_DIR, id + ".eu4"); + + + public IOPath GetSaveFilePath() => Path.Concat(PathHelper.SAVES_DIR, id + ".tar.zst"); public IOPath GetParsedDataPath() => Path.Concat(PathHelper.PARSED_DIR, id + ".parsed.json"); @@ -42,4 +57,4 @@ public class SaveFileMetadata { return File.Exists(GetSaveFilePath()) && File.Exists(GetParsedDataPath()); } -} \ No newline at end of file +} diff --git a/ParadoxSaveParser.WebAPI/Game.cs b/ParadoxSaveParser.WebAPI/Game.cs index 20578b6..86e36cc 100644 --- a/ParadoxSaveParser.WebAPI/Game.cs +++ b/ParadoxSaveParser.WebAPI/Game.cs @@ -3,5 +3,16 @@ public enum Game { Unknown, - EU4 -} \ No newline at end of file + EU4, + Stellaris, +} + +public static class GameSaveEntryNames +{ + /// name of the entry in uploaded .tar.zst which contains game data + public static readonly IReadOnlyDictionary Map = new Dictionary + { + { Game.EU4, "gamestate" }, + { Game.Stellaris, "gamestate" }, + }; +} diff --git a/ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.csproj b/ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.csproj index e41ed0f..1ad15cb 100644 --- a/ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.csproj +++ b/ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.csproj @@ -9,11 +9,12 @@ - + - + + diff --git a/ParadoxSaveParser.WebAPI/Program.cs b/ParadoxSaveParser.WebAPI/Program.cs index c9bb035..468bf53 100644 --- a/ParadoxSaveParser.WebAPI/Program.cs +++ b/ParadoxSaveParser.WebAPI/Program.cs @@ -87,7 +87,7 @@ public static class Program await DB.InitializeAsync(); await DB.DeleteInvalidData(); - var bgJobManager = new BackgroundJobManager(loggerRoot); + var bgJobManager = new BackgroundJobManager(loggerRoot, config.SaveUpload); var saveFilters = new Dictionary { { Game.EU4, new SaveDataFilterEU4() }, @@ -98,11 +98,13 @@ public static class Program router.DefaultRoute = new SimpleRouter.RouteWithMethod(HttpMethod.GET, new ServeFilesRouteHandler(PathHelper.PUBLIC_DIR)); router.MapRoute("/uploadSave", HttpMethod.POST, - new UploadSaveHandler(mainCancel.Token, bgJobManager, saveFilters)); + new UploadSaveHandler(mainCancel.Token, bgJobManager, saveFilters, config.SaveUpload)); router.MapRoute("/getSaveStatus", HttpMethod.GET, new GetSaveStatusHandler(mainCancel.Token)); router.MapRoute("/getSaveData", HttpMethod.GET, new GetSaveDataHandler(mainCancel.Token)); + router.MapRoute("/getSaveFile", HttpMethod.GET, + new GetSaveFileHandler(mainCancel.Token)); var app = new WebApp(config.BaseUrl, loggerRoot, router, mainCancel.Token); await app.Run(); diff --git a/ParadoxSaveParser.WebAPI/README.md b/ParadoxSaveParser.WebAPI/README.md index f758b40..815394b 100644 --- a/ParadoxSaveParser.WebAPI/README.md +++ b/ParadoxSaveParser.WebAPI/README.md @@ -5,30 +5,50 @@ Simple web application created using DTLib.Web. ## Important Restart web application once per day to delete outdated data and clean RAM. +## Config +`SaveUpload` section of `config.json`: +- `MaxSizeCompressedBytes` - max size of uploaded `.tar.zst` (default 60 MiB) +- `MaxSizePlainTextBytes` - max size of the game data entry after decompression (default 200 MiB) + ## Routes ### POST `/uploadSave` - **Query Params:** - `game` - short name of the game (see [../README.md](../README.md)) -- **Request Body:** `application/octet-stream` - .eu4 file + - `originalFileType` - `text` | `zip` + - `originalFileName` - name of the original save file (max 255 characters) +- **Request Body:** `application/zstd` - single zstd frame containing a tar with files of the original save. + Game data is read from the entry named in [GameSaveEntryNames](./Game.cs) (`gamestate` for EU4). + A plain text save is a tar with just that one entry. + The blob is stored byte-for-byte and can be downloaded with `/getSaveFile`. - **Response:** ```json - { "saveId": "string" } + { "id": "string" } ``` - or `{ "errorMessage": "string" }` + or `{ "errorMessage": "string" }` + (status 413 if upload or uncompressed game data exceeds the limits from config) ### GET `/getSaveStatus` - **Query Params:** - `id` - id of uploaded save file -- **Response:** [SaveFileMetadata](./SaveFileMetadata.cs) +- **Response:** [SaveFileMetadata](./Database/SaveFileMetadata.cs) ```json { "id": "string", "game": "string", - "status": "string", + "status": "initialized | uploaded | parsing | saving_results | done | error", + "uploadDateTime": "string", + "originalFileType": "text | zip", + "originalFileName": "string", "errorMessage": "string?" } ``` +### GET `/getSaveFile` +- **Query Params:** + - `id` - id of uploaded save file +- **Response:** `application/zstd` - uploaded blob, unchanged + or `{ "errorMessage": "string" }` + ### GET `/getSaveData` - **Query Params:** - `id` - id of uploaded save file diff --git a/ParadoxSaveParser.WebAPI/Routes/GetSaveFileHandler.cs b/ParadoxSaveParser.WebAPI/Routes/GetSaveFileHandler.cs new file mode 100644 index 0000000..8ba94a4 --- /dev/null +++ b/ParadoxSaveParser.WebAPI/Routes/GetSaveFileHandler.cs @@ -0,0 +1,45 @@ +using System.Net; +using ParadoxSaveParser.WebAPI.Database; +using ParadoxSaveParser.WebAPI.HttpHelpers; + +namespace ParadoxSaveParser.WebAPI.Routes; + +internal class GetSaveFileHandler : RouteHandlerBase +{ + public GetSaveFileHandler(CancellationToken cancelAllToken) + : base(cancelAllToken) + { + } + + public override async Task HandleRequest( + HttpListenerContext ctx, ContextLogger requestLogger) + { + var idOrError = RequestHelper.GetQueryValue(ctx, "id"); + if (idOrError.HasError) + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, idOrError.Error!); + string id = idOrError.Value!; + + var meta = await Program.DB.GetMetadata(id); + if(meta is null) + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, + new ErrorMessage(HttpStatusCode.NotFound, + $"Save with id {id} not found")); + // file is being uploaded (Initialized) or was deleted (Error) + if(meta.status is SaveFileProcessingStatus.Initialized or SaveFileProcessingStatus.Error) + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, + new ErrorMessage(HttpStatusCode.BadRequest, + $"Save with id {id} has status {meta.status}")); + + IOPath saveFilePath = meta.GetSaveFilePath(); + if (!File.Exists(saveFilePath)) + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, + new ErrorMessage(HttpStatusCode.InternalServerError, + $"Save with id {id} not found") + ); + + await using var saveFile = File.OpenRead(saveFilePath); + + return await ReturnHelper.ResponseStream(ctx, requestLogger, _cancelAllToken, + saveFile, contentType: "application/zstd"); + } +} diff --git a/ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs b/ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs index 047be27..382275d 100644 --- a/ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs +++ b/ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs @@ -1,4 +1,4 @@ -using System.Linq; +using System.IO; using System.Net; using ParadoxSaveParser.WebAPI.BackgroundTasks; using ParadoxSaveParser.WebAPI.Database; @@ -9,24 +9,30 @@ namespace ParadoxSaveParser.WebAPI.Routes; public class UploadSaveHandler : RouteHandlerBase { + private const int MaxOriginalFileNameLength = 255; + private readonly BackgroundJobManager _bgJobManager; private readonly Dictionary _saveFilters; + private readonly Config.SaveUploadConfig _uploadConfig; public UploadSaveHandler( - CancellationToken cancelAllToken, + CancellationToken cancelAllToken, BackgroundJobManager bgJobManager, - Dictionary saveFilters) + Dictionary saveFilters, + Config.SaveUploadConfig uploadConfig) : base(cancelAllToken) { _bgJobManager = bgJobManager; _saveFilters = saveFilters; + _uploadConfig = uploadConfig; } public override async Task HandleRequest( HttpListenerContext ctx, ContextLogger requestLogger) { - string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault(); - if (contentType != "application/octet-stream") + string? contentType = ctx.Request.ContentType; + string? mediaType = contentType?.Split(';')[0].Trim(); + if (!string.Equals(mediaType, "application/zstd", StringComparison.OrdinalIgnoreCase)) return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, new ErrorMessage(HttpStatusCode.BadRequest, $"Invalid request Content-Type: '{contentType}'")); @@ -34,18 +40,82 @@ public class UploadSaveHandler : RouteHandlerBase var gameStrOrError = RequestHelper.GetQueryValue(ctx, "game"); if (gameStrOrError.HasError) return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, gameStrOrError.Error!); - if (!Enum.TryParse(gameStrOrError.Value, ignoreCase: true, out Game game)) return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, new ErrorMessage(HttpStatusCode.BadRequest, - $"Invalid requested game: '{gameStrOrError.Value}'")); - var meta = await Program.DB.CreateMetadata(game); + $"Invalid requested game: '{gameStrOrError.Value}'")); + if (!_saveFilters.ContainsKey(game) || !GameSaveEntryNames.Map.ContainsKey(game)) + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, + new ErrorMessage(HttpStatusCode.BadRequest, + $"Game {game} is not supported")); - var saveFilePath = meta.GetSaveFilePath(); - await using (var saveFile = File.OpenWrite(saveFilePath)) + var fileTypeStrOrError = RequestHelper.GetQueryValue(ctx, "originalFileType"); + if (fileTypeStrOrError.HasError) + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, fileTypeStrOrError.Error!); + OriginalFileType? originalFileType = fileTypeStrOrError.Value!.ToLowerInvariant() switch { - await using (var remoteStream = ctx.Request.InputStream) - await remoteStream.CopyToAsync(saveFile, _cancelAllToken); + "text" => OriginalFileType.Text, + "zip" => OriginalFileType.Zip, + _ => null + }; + if (originalFileType is null) + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, + new ErrorMessage(HttpStatusCode.BadRequest, + $"Invalid originalFileType: '{fileTypeStrOrError.Value}'")); + + var fileNameOrError = RequestHelper.GetQueryValue(ctx, "originalFileName"); + if (fileNameOrError.HasError) + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, fileNameOrError.Error!); + string originalFileName = fileNameOrError.Value!; + if (originalFileName.Length > MaxOriginalFileNameLength) + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, + new ErrorMessage(HttpStatusCode.BadRequest, + $"originalFileName is longer than {MaxOriginalFileNameLength} characters")); + + // Content-Length is unknown (-1) for chunked requests, so body size is checked during copying too + if (ctx.Request.ContentLength64 > _uploadConfig.MaxSizeCompressedBytes) + return await ResponseUploadTooLarge(ctx, requestLogger); + + var meta = await Program.DB.CreateMetadata(game, originalFileType.Value, originalFileName); + try + { + bool fitsLimit; + await using (var saveFile = File.OpenWrite(meta.GetSaveFilePath())) + { + await using (var remoteStream = ctx.Request.InputStream) + fitsLimit = await CopyWithLimit(remoteStream, saveFile, _uploadConfig.MaxSizeCompressedBytes); + } + if (!fitsLimit) + { + await Program.DB.DeleteMetadata(meta, "upload is too large"); + return await ResponseUploadTooLarge(ctx, requestLogger); + } + + long gameDataSize; + try + { + gameDataSize = await SaveArchive.GetGameDataSize(meta.GetSaveFilePath(), game, _cancelAllToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + await Program.DB.DeleteMetadata(meta, "invalid archive"); + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, + new ErrorMessage(HttpStatusCode.BadRequest, + $"Invalid save archive: {ex.Message}")); + } + if (gameDataSize > _uploadConfig.MaxSizePlainTextBytes) + { + await Program.DB.DeleteMetadata(meta, "game data is too large"); + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, + new ErrorMessage(HttpStatusCode.RequestEntityTooLarge, + $"Uncompressed game data size ({gameDataSize} bytes) exceeds limit " + + $"({_uploadConfig.MaxSizePlainTextBytes} bytes)")); + } + } + catch + { + await Program.DB.DeleteMetadata(meta, "upload failed"); + throw; } await Program.DB.UpdateMetadataStatus(meta, SaveFileProcessingStatus.Uploaded); @@ -53,4 +123,27 @@ public class UploadSaveHandler : RouteHandlerBase dynamic responseData = new { meta.id }; return await ReturnHelper.ResponseJson(ctx, requestLogger, _cancelAllToken, responseData); } -} \ No newline at end of file + + private Task ResponseUploadTooLarge(HttpListenerContext ctx, ContextLogger requestLogger) + { + return ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, + new ErrorMessage(HttpStatusCode.RequestEntityTooLarge, + $"Upload size exceeds limit ({_uploadConfig.MaxSizeCompressedBytes} bytes)")); + } + + /// false if source is longer than maxBytes + private async Task CopyWithLimit(Stream source, Stream dest, long maxBytes) + { + byte[] buffer = new byte[81920]; + long totalBytes = 0; + int readBytes; + while ((readBytes = await source.ReadAsync(buffer, _cancelAllToken)) > 0) + { + totalBytes += readBytes; + if (totalBytes > maxBytes) + return false; + await dest.WriteAsync(buffer.AsMemory(0, readBytes), _cancelAllToken); + } + return true; + } +} diff --git a/ParadoxSaveParser.WebAPI/SaveArchive.cs b/ParadoxSaveParser.WebAPI/SaveArchive.cs new file mode 100644 index 0000000..4b479b3 --- /dev/null +++ b/ParadoxSaveParser.WebAPI/SaveArchive.cs @@ -0,0 +1,52 @@ +using System.Formats.Tar; +using System.IO; +using ZstdSharp; + +namespace ParadoxSaveParser.WebAPI; + +/// +/// Uploaded save is a zstd-compressed tar with files of the original save inside. +/// Game data is stored in entry with name from . +/// +public static class SaveArchive +{ + public static Task GetGameDataSize(IOPath archivePath, Game game, CancellationToken ct) + { + return WithGameDataEntry(archivePath, game, ct, + entry => Task.FromResult(entry.Length)); + } + + public static Task ExtractGameData(IOPath archivePath, Game game, Stream dest, long maxBytes, + CancellationToken ct) + { + return WithGameDataEntry(archivePath, game, ct, async entry => + { + if (entry.Length > maxBytes) + throw new InvalidDataException( + $"Game data size ({entry.Length} bytes) exceeds limit ({maxBytes} bytes)"); + if (entry.DataStream is not null) + await entry.DataStream.CopyToAsync(dest, ct); + return true; + }); + } + + private static async Task WithGameDataEntry(IOPath archivePath, Game game, CancellationToken ct, + Func> action) + { + if (!GameSaveEntryNames.Map.TryGetValue(game, out string? entryName)) + throw new ArgumentOutOfRangeException(nameof(game), game, "No game data entry name for game"); + + await using var file = File.OpenRead(archivePath); + await using var zstd = new DecompressionStream(file); + await using var tar = new TarReader(zstd); + while (await tar.GetNextEntryAsync(copyData: false, ct) is { } entry) + { + if (entry.EntryType is not (TarEntryType.RegularFile or TarEntryType.V7RegularFile)) + continue; + if (System.IO.Path.GetFileName(entry.Name) == entryName) + return await action(entry); + } + + throw new InvalidDataException($"Invalid save format: no '{entryName}' entry found"); + } +}