From 7d64ab4809b82f61bf1ca0da8a0b6bf489a6e94c Mon Sep 17 00:00:00 2001 From: Altair-sh Date: Wed, 23 Sep 2026 18:24:45 +0200 Subject: [PATCH] decompress .json.zstd fetched by getSaveData --- AGENTS.md | 15 ++++++- COMPRESSION.md | 54 ++++++++++++------------ src/config/AppConfig.ts | 6 +-- src/lib/formatBytes.ts | 4 +- src/lib/saveArchive/client.ts | 9 ++-- src/services/SaveParserBackendService.ts | 18 +++++++- 6 files changed, 67 insertions(+), 39 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 48cf595..2152644 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,7 +38,7 @@ SaveParserBackendService -> /api/proxy/saveParserBackend/* -> backend - [src/services/SaveParserBackendService.ts](src/services/SaveParserBackendService.ts) is the only place that talks to the backend. It exposes `uploadSave`, - `getSaveStatus`, `getParsedData` and `getSaveFile` — upload, poll until + `getSaveStatus`, `getSaveData` and `getSaveFile` — upload, poll until parsing finishes, then fetch the result or the stored save back. - [src/app/api/proxy/saveParserBackend/[[...path]]/route.ts](src/app/api/proxy/saveParserBackend/[[...path]]/route.ts) forwards every method to the configured backend URL. It exists only to keep @@ -113,6 +113,13 @@ Three things there are easy to break: - **Compression runs in a Web Worker** because it would otherwise block the page for seconds on a large save. +`getSaveData` is the deliberate exception: the backend stores parsed save data +as zstd compressed json, and the service decompresses that response on the main +thread instead of in the worker. Decompression is cheap and its cost does not +depend on the level — the whole 87 MB of an EU4 save comes back in 0.27 s at +level 10, and the parsed json is a fraction of that, so it is a matter of +milliseconds. A worker round trip would cost more than it saves. + [COMPRESSION.md](COMPRESSION.md) has every zstd level measured on a real save — size, time and the memory the encoder and decoder need. Consult it instead of measuring again when a device cannot cope with the current level. @@ -203,6 +210,12 @@ does not replace a full build. Do not report a change as finished until both commands pass. +### Tests and benchmarks + +**Never run a test suite or a benchmark without explicit permission.** Ask +first and wait for a clear yes, however easy the command looks. Lint and build +are the exception, and are expected after every change. + ## Version control This project is managed with **git**. diff --git a/COMPRESSION.md b/COMPRESSION.md index e983a33..0571ce7 100644 --- a/COMPRESSION.md +++ b/COMPRESSION.md @@ -5,8 +5,8 @@ Reference for `saveUpload.zstdCompressionLevel` in out to be too slow or too memory hungry on some device, pick a different level from the table below rather than measuring again. -The level only affects the uploader. Decompression on the parser backend costs -the same at every level. +The level only affects the uploader. Decompression costs the same at every +level, both on the parser backend and in the browser. ## What was measured @@ -33,34 +33,34 @@ correct output rather than merely fast output. ## The table -| level | compressed | ratio | compress | decompress | encoder heap | decoder heap | window | -| -----: | ------------: | --------: | ---------: | ---------: | -----------: | -----------: | -----: | -| 1 | 6,698,281 | 1.72x | 0.54 s | 0.28 s | 8 MB | 8 MB | 1 MB | -| 2 | 6,600,451 | 1.74x | 0.54 s | 0.29 s | 8 MB | 8 MB | 1 MB | -| 3 | 6,094,116 | 1.89x | 0.57 s | 0.28 s | 8 MB | 8 MB | 2 MB | -| 4 | 6,082,362 | 1.89x | 0.61 s | 0.28 s | 8 MB | 8 MB | 2 MB | -| 5 | 5,280,090 | 2.18x | 0.92 s | 0.28 s | 8 MB | 8 MB | 2 MB | -| 6 | 4,943,366 | 2.33x | 1.13 s | 0.27 s | 8 MB | 8 MB | 2 MB | -| 7 | 4,738,515 | 2.43x | 1.27 s | 0.27 s | 9 MB | 8 MB | 2 MB | -| 8 | 4,514,853 | 2.55x | 1.52 s | 0.28 s | 9 MB | 8 MB | 2 MB | -| 9 | 4,445,368 | 2.59x | 1.51 s | 0.26 s | 16 MB | 8 MB | 4 MB | -| 10 | 4,280,567 | 2.69x | 1.74 s | 0.27 s | 26 MB | 8 MB | 4 MB | -| **11** | **4,157,684** | **2.77x** | **2.44 s** | **0.32 s** | **26 MB** | **8 MB** | 4 MB | -| 12 | 4,155,668 | 2.77x | 2.39 s | 0.27 s | 46 MB | 8 MB | 4 MB | -| 13 | 4,257,141 | 2.70x | 3.76 s | 0.27 s | 38 MB | 8 MB | 4 MB | -| 14 | 4,108,558 | 2.80x | 5.39 s | 0.28 s | 54 MB | 8 MB | 4 MB | -| 15 | 3,968,262 | 2.90x | 8.13 s | 0.33 s | 70 MB | 8 MB | 4 MB | -| 16 | 3,862,217 | 2.98x | 11.63 s | 0.26 s | 38 MB | 8 MB | 4 MB | -| 17 | 3,733,198 | 3.08x | 12.39 s | 0.26 s | 58 MB | 10 MB | 8 MB | -| 18 | 3,705,141 | 3.10x | 15.11 s | 0.26 s | 59 MB | 10 MB | 8 MB | -| 19 | 3,449,475 | 3.33x | 27.78 s | 0.28 s | 91 MB | 10 MB | 8 MB | -| 20 | 3,344,642 | 3.44x | 28.24 s | 0.27 s | 195 MB | 34 MB | 32 MB | -| 21 | 3,285,836 | 3.50x | 33.91 s | 0.30 s | 387 MB | 66 MB | 64 MB | -| 22 | 3,245,980 | 3.54x | 49.02 s | 0.30 s | 835 MB | 130 MB | 128 MB | +| level | compressed | ratio | compress | decompress | encoder heap | decoder heap | +| -----: | ------------: | --------: | ---------: | ---------: | -----------: | -----------: | +| 1 | 6,698,281 | 1.72x | 0.54 s | 0.28 s | 8 MB | 8 MB | +| 2 | 6,600,451 | 1.74x | 0.54 s | 0.29 s | 8 MB | 8 MB | +| 3 | 6,094,116 | 1.89x | 0.57 s | 0.28 s | 8 MB | 8 MB | +| 4 | 6,082,362 | 1.89x | 0.61 s | 0.28 s | 8 MB | 8 MB | +| 5 | 5,280,090 | 2.18x | 0.92 s | 0.28 s | 8 MB | 8 MB | +| 6 | 4,943,366 | 2.33x | 1.13 s | 0.27 s | 8 MB | 8 MB | +| 7 | 4,738,515 | 2.43x | 1.27 s | 0.27 s | 9 MB | 8 MB | +| 8 | 4,514,853 | 2.55x | 1.52 s | 0.28 s | 9 MB | 8 MB | +| 9 | 4,445,368 | 2.59x | 1.51 s | 0.26 s | 16 MB | 8 MB | +| **10** | **4,280,567** | **2.69x** | **1.74 s** | **0.27 s** | **26 MB** | **8 MB** | +| 11 | 4,157,684 | 2.77x | 2.44 s | 0.32 s | 26 MB | 8 MB | +| 12 | 4,155,668 | 2.77x | 2.39 s | 0.27 s | 46 MB | 8 MB | +| 13 | 4,257,141 | 2.70x | 3.76 s | 0.27 s | 38 MB | 8 MB | +| 14 | 4,108,558 | 2.80x | 5.39 s | 0.28 s | 54 MB | 8 MB | +| 15 | 3,968,262 | 2.90x | 8.13 s | 0.33 s | 70 MB | 8 MB | +| 16 | 3,862,217 | 2.98x | 11.63 s | 0.26 s | 38 MB | 8 MB | +| 17 | 3,733,198 | 3.08x | 12.39 s | 0.26 s | 58 MB | 10 MB | +| 18 | 3,705,141 | 3.10x | 15.11 s | 0.26 s | 59 MB | 10 MB | +| 19 | 3,449,475 | 3.33x | 27.78 s | 0.28 s | 91 MB | 10 MB | +| 20 | 3,344,642 | 3.44x | 28.24 s | 0.27 s | 195 MB | 34 MB | +| 21 | 3,285,836 | 3.50x | 33.91 s | 0.30 s | 387 MB | 66 MB | +| 22 | 3,245,980 | 3.54x | 49.02 s | 0.30 s | 835 MB | 130 MB | ## Picking a level -**11 is the default**, because it is where the curve flattens. It gives good compression in reasonable time. +**10 is the default**, because it is where the curve flattens. It gives good compression in reasonable time: 11 is only 3% smaller but takes 40% longer. If there will be troubles with this level in the future, just change it in config. diff --git a/src/config/AppConfig.ts b/src/config/AppConfig.ts index 91c36c8..78c32ec 100644 --- a/src/config/AppConfig.ts +++ b/src/config/AppConfig.ts @@ -10,11 +10,11 @@ export const defaultConfig = { // Size of a save if it is uploaded as uncompressed. The browser packs // saves as a stream, so these limits are about the backend's disk // rather than about memory. - maxSizePlainTextBytes: 200 * 1024 * 1024, + maxSizePlainTextMB: 200, // Archives produced by the game, recognized by the zip magic bytes. - maxSizeCompressedBytes: 60 * 1024 * 1024, + maxSizeCompressedMB: 60, // read COMPRESSION.md - zstdCompressionLevel: 11, + zstdCompressionLevel: 10, }, } diff --git a/src/lib/formatBytes.ts b/src/lib/formatBytes.ts index 6f565bd..cdf0d94 100644 --- a/src/lib/formatBytes.ts +++ b/src/lib/formatBytes.ts @@ -1,5 +1,5 @@ -const KILOBYTE = 1024 -const MEGABYTE = 1024 * KILOBYTE +export const KILOBYTE = 1024 +export const MEGABYTE = 1024 * KILOBYTE /** * Formats a byte count for error messages and status lines, e.g. "12.3 MB". diff --git a/src/lib/saveArchive/client.ts b/src/lib/saveArchive/client.ts index 62ddb52..5c2d5cc 100644 --- a/src/lib/saveArchive/client.ts +++ b/src/lib/saveArchive/client.ts @@ -3,7 +3,7 @@ import type { SaveUploadConfig } from '@/config/AppConfig' import { GAME_SAVE_ENTRY_NAMES, GameEnum } from '@/config/games' -import { formatBytes } from '@/lib/formatBytes' +import { formatBytes, MEGABYTE } from '@/lib/formatBytes' import { isZipFile } from './gameZip' import { OriginalFileType, @@ -68,9 +68,10 @@ export async function packSaveForUpload( onStage: StageCallback ): Promise { const isZip = await isZipFile(file) - const sizeLimit = isZip - ? config.maxSizeCompressedBytes - : config.maxSizePlainTextBytes + // The limits are configured in megabytes, file sizes are in bytes. + const sizeLimit = + (isZip ? config.maxSizeCompressedMB : config.maxSizePlainTextMB) * + MEGABYTE if (file.size > sizeLimit) { const saveKind = isZip ? 'archived' : 'plain text' diff --git a/src/services/SaveParserBackendService.ts b/src/services/SaveParserBackendService.ts index 822dffa..881f9c4 100644 --- a/src/services/SaveParserBackendService.ts +++ b/src/services/SaveParserBackendService.ts @@ -3,6 +3,7 @@ import type { SaveUploadConfig } from '@/config/AppConfig' import { GameEnum } from '@/config/games' import { packSaveForUpload, StageCallback } from '@/lib/saveArchive/client' import { OriginalFileType } from '@/lib/saveArchive/messages' +import { createZstdDecompressStream } from '@/lib/saveArchive/zstdStream' export interface UploadSaveResponse { id: string @@ -86,7 +87,12 @@ export class SaveParserBackendService { return await response.json() } - // Fetches the fully parsed data for a save once its parsing is done. + /** + * Fetches the fully parsed data for a save once its parsing is done. + * + * The backend stores and serves that data as zstd compressed json, so the + * body is decompressed on the way in rather than read as text first. + */ async getSaveData(id: string): Promise> { const url = `${this.API_BASE}/getSaveData?id=${encodeURIComponent(id)}` const response = await fetch(url) @@ -97,8 +103,16 @@ export class SaveParserBackendService { (await getBodyTextOrEmpty(response)) ) } + // A body is only missing on a response that cannot carry one, which a + // successful getSaveData never is. + if (!response.body) { + throw new Error('Failed to get parsed save: response has no body') + } - return await response.json() + // Decompresses as the bytes arrive, so the compressed json is never + // held whole; only the parsed object stays in memory. + const json = response.body.pipeThrough(createZstdDecompressStream()) + return await new Response(json).json() } /**