diff --git a/.gitignore b/.gitignore index 52111e0..a344cb7 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ next-env.d.ts # ai agent settings .claude + +# scratch directory for temporary files, deleted once a task is done +/TEMP diff --git a/AGENTS.md b/AGENTS.md index c941097..9de2d86 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ Source files live in `src/`: - `app/` for routes, - `components/` for shared UI - `services/` for backend calls -- `lib/` for helpers +- `lib/` for helpers, with save compression in `lib/saveArchive/` - `config/` for app config - `types/` for .d.ts type declaration files @@ -38,8 +38,8 @@ SaveParserBackendService -> /api/proxy/saveParserBackend/* -> backend - [src/services/SaveParserBackendService.ts](src/services/SaveParserBackendService.ts) is the only place that talks to the backend. It exposes `uploadSave`, - `getSaveStatus` and `getParsedSave` — upload, poll until parsing finishes, - then fetch the result. + `getSaveStatus`, `getParsedData` and `getSaveFile` — upload, poll until + parsing finishes, then fetch the result or the stored save back. - [src/app/api/proxy/saveParserBackend/[[...path]]/route.ts](src/app/api/proxy/saveParserBackend/[[...path]]/route.ts) forwards every method to the configured backend URL. It exists only to keep requests same-origin, and is meant to be deleted once the backend handles @@ -47,25 +47,116 @@ SaveParserBackendService -> /api/proxy/saveParserBackend/* -> backend - Route paths live in [src/app/api/proxy/paths.ts](src/app/api/proxy/paths.ts) rather than as string literals, so both sides of the proxy stay in sync. +### Save files on the wire + +Saves are **not** sent as the game wrote them. The backend runs on a cheap two +core server with little disk space, so the browser re-compresses every save +before uploading it: + +``` +plain text save -> tar { "gamestate" } -> zstd -> upload +archived save -> tar { "meta", "gamestate", ... } -> zstd -> upload +``` + +Uploads are `POST /uploadSave?game=…&originalFileType=text|zip&originalFileName=…` +with `Content-Type: application/zstd`. The browser cannot stream a request +body outside Chrome, so the compressed result is sent as one blob; responses +stream everywhere, so downloads need nothing special. The tar always holds the +files the game itself would have in its archive, so the backend reads the game +state from the `gamestate` entry either way, and `getSaveStatus` returns `originalFileType` +and `originalFileName` so a download can rebuild the right shape. + +The pieces all live in [src/lib/saveArchive/](src/lib/saveArchive/): +[gameZip.ts](src/lib/saveArchive/gameZip.ts) (the game's own zip format, +written here on the browser's `CompressionStream('deflate-raw')`), +[zstdStream.ts](src/lib/saveArchive/zstdStream.ts) (compression, on +[@tootallnate/zstd-wasm](https://github.com/TooTallNate/switch-tools/tree/main/packages/zstd-wasm)), +[archive.ts](src/lib/saveArchive/archive.ts) (pack and rebuild, with the tar +from [modern-tar](https://github.com/ayuhito/modern-tar)), +[messages.ts](src/lib/saveArchive/messages.ts) (the worker protocol), and +[worker.ts](src/lib/saveArchive/worker.ts) with +[client.ts](src/lib/saveArchive/client.ts) (the worker and its main thread +side). + +**The whole pipeline is streams.** A save is never held in memory as a whole: +the file is read, unzipped, tarred and compressed as it flows, and only the +finished result becomes a `Blob`, which is browser managed storage rather than +a heap allocation. A real 11 MB EU4 save (84 MB of `gamestate`) compresses to +about 4.15 MB in two seconds. Keep it that way — an `arrayBuffer()` call on a +save is a bug. + +Three things there are easy to break: + +- **The zip headers are not decoration.** The games reject archives with NTFS + timestamps or unexpected header fields, so every field matches a real save: + deflate, zeroed DOS date, MS-DOS origin, version made by 63, version needed + 20, no extra field, no data descriptor. That last pair is why no zip library + is used: they all build the DOS date from a `Date` and clamp it to 1980, and + the streaming writers add a data descriptor and a zip64 extra field to every + entry. Writing the headers here is about a hundred lines, because deflate + itself comes from the browser. One consequence to keep: each entry is + deflated into a blob before its header is written, since the sizes and the + CRC go in front of the data — the _compressed_ bytes are buffered, never the + save. +- **The zstd wasm is a runtime asset, not part of the bundle.** + `@tootallnate/zstd-wasm` deliberately loads no files of its own, so + `zstdStream.ts` points `new URL(…, import.meta.url)` at + `node_modules/@tootallnate/zstd-wasm/dist/zstd.wasm` and Turbopack emits it + as a hashed asset the worker fetches. That path is relative to + `src/lib/saveArchive/zstdStream.ts`, so moving the file means fixing it. +- **Three zstd libraries were tried and rejected before this one**, so do not + swap it lightly. `@oneidentity/zstd-js` compresses whole buffers and leaves a + frame unterminated whenever the payload length is an exact multiple of + 131072, `zstd-codec`'s emscripten build never returns control to the event + loop, and `fzstd` is the unreadable single-file kind of code this repository + does not depend on. +- **Compression runs in a Web Worker** because it would otherwise block the + page for seconds on a large save. + +[COMPRESSION.md](COMPRESSION.md) has every zstd level measured on a real save — +size, time and the memory the encoder and decoder need. Consult it instead of +measuring again when a device cannot cope with the current level. + Configuration comes from [src/config/AppConfig.ts](src/config/AppConfig.ts) (defaults) merged with `runtimeConfig.json` at the repo root, which holds the -backend URL. That file is gitignored, so a fresh clone starts without one and -falls back to the defaults. +backend URL and the save upload limits. That file is gitignored, so a fresh +clone starts without one and falls back to the defaults. + +`AppConfig` is server side only, so values the browser needs are passed down as +props: [src/app/page.tsx](src/app/page.tsx) reads `appConfig.saveUpload` and +hands it to the upload dialog. ### Current status Much of the app is still scaffold. Do not assume a feature works because the UI for it exists: -- **The grid is not wired up.** [src/app/browse/page.tsx](src/app/browse/page.tsx) - renders two hardcoded rows and never calls the backend. Nothing connects the - upload flow to it yet. +- **The grid is not wired up.** [src/components/BrowseSaveDataGrid.tsx](src/components/BrowseSaveDataGrid.tsx) + renders two hardcoded rows and never calls the backend. `/browse?id=…` only + uses the id for the download button so far. +- **The backend does not speak the new upload format yet.** It has to accept the + zstd compressed tar described above and serve `getSaveFile`; until it does, + uploads fail against the running service. - **Auth does not work.** `AuthService` posts to `/api/auth/login` and `/api/auth/register`, which do not exist as routes. The service is slated for removal in favour of Discord OAuth. - **`hashPassword` is a no-op** that returns the password unchanged. Treat any current auth code as a placeholder, not as something to build on. +## Tools and files + +**Prefer the editor's own tools over shell commands that do the same thing.** +Reading, editing, writing and searching files have dedicated tools; reaching +for `cat`, `sed`, `grep` or a throwaway script instead does the same work while +spending more tokens, and every command may need its own approval. Use the shell +for what only the shell can do — running the build, the linter, git, a real +program. + +**Work inside the project directory.** For temporary files — scratch scripts, +intermediate output, test data — create a `TEMP/` directory in the project root +and keep everything there, then delete that directory once the work is done. +Ask before reading or writing anything outside the project directory. + ## Terminal **On Windows, prefer `bash.exe`** — but only a real MSYS2, Cygwin, or @@ -134,11 +225,6 @@ editing, or match the surrounding style exactly. Import from within `src/` using the `@/` alias, for example `import { getMsgFromError } from '@/lib/errors'`. -Write doc comments as JSDoc (`/** ... */`), which editors show on hover. -Plain `//` and `///` comments do not appear in tooltips. -In `.ts` files put only prose in the JSDoc and let the signature carry the types; -do not duplicate types in `{braces}`. - Catch blocks should not annotate the error. Let it be `unknown` and convert it with the shared helper: @@ -151,6 +237,56 @@ with the shared helper: Avoid `any`; `no-explicit-any` is enforced. Prefer `unknown` plus narrowing. +**Comment the code you write.** This codebase is read by human programmers who +did not write it, and a wall of uncommented agent-written code is expensive to +review. Keep comments simple and preferably single line, placed above the code +they describe, and write them in plain language rather than in terms of the +types. Say _why_ the code does what it does — what is non-obvious, what would +break if it were written the obvious way — and leave out what the code already +states plainly. A tricky constant, a workaround for someone else's bug, or an +ordering that matters always deserves a line. + +Write doc comments as JSDoc (`/** ... */`), which editors show on hover. +Plain `//` and `///` comments do not appear in tooltips. +In `.ts` files put only prose in the JSDoc and let the signature carry the types; +do not duplicate types in `{braces}`. + +## Dependencies + +**Search for an existing library before writing your own implementation.** +Formats, protocols and codecs are full of details that only show up against +real data, and a maintained library has already met them. This project once +carried a hand-written tar reader/writer and zip writer; both were deleted in +favour of libraries. + +**Read the library's source before adding it, and judge it on that.** Download +counts say nothing about quality. Even a popular package may turn out to be a +single ten thousand line file whose variables and functions are all named `a`, +`b` and `c`, with no comments anywhere. Such code cannot be read, cannot be +maintained, and is very likely to be carrying bugs nobody can find — do not +depend on it. Look for readable sources, comments that explain intent, and an +API that fits the way the code here is written. + +If nothing of good quality exists, say so and propose writing an implementation +here instead, rather than pulling in the least bad option. + +Do not add a second library for a job a package already in `package.json` can +do. A thin wrapper that shapes an installed library's API to what the code +needs is better than a new dependency. But if the API of the library +doesn't fit the project, the wrapper can be bigger than your own implementation +of the algorithm. + +## Checking browser support + +**Never use caniuse.com.** Its data is wrong and incomplete, missing features +that shipped five or more years ago, and a decision made from it will be wrong. + +Use MDN's [browser-compat-data](https://github.com/mdn/browser-compat-data), +which is the source behind the MDN compatibility tables, and +[webstatus.dev](https://webstatus.dev) for the baseline status of a feature. If +those disagree or say nothing, the browsers' own release notes and bug trackers +settle it. + ## Things that will bite you **Both TypeScript 6 and 7 are installed on purpose, under swapped names.** diff --git a/COMPRESSION.md b/COMPRESSION.md new file mode 100644 index 0000000..e983a33 --- /dev/null +++ b/COMPRESSION.md @@ -0,0 +1,73 @@ +# Compression levels + +Reference for `saveUpload.zstdCompressionLevel` in +[src/config/AppConfig.ts](src/config/AppConfig.ts). If compression ever turns +out to be too slow or too memory hungry on some device, pick a different level +from the table below rather than measuring again. + +The level only affects the uploader. Decompression on the parser backend costs +the same at every level. + +## What was measured + +A real Europa Universalis IV save, as the game wrote it: + +| | | +| ------------- | ----------------------------------------------------- | +| File | `Ibevar1475_01_02save.eu4`, a zip archive | +| Size on disk | **11,502,739 bytes** (11.0 MB) | +| Entries | `gamestate` (87,188,760 bytes), `meta`, `ai` | +| Input to zstd | **87,342,592 bytes** — the tar of those three entries | + +So the ratios below are against the original 11.5 MB game archive, and zstd +itself is working on the 87 MB of uncompressed save text inside it. + +Each level ran through the project's own `packSave` in a separate process. The +heap columns are the WebAssembly linear memory the encoder and the decoder grow +to, read from the instance itself — that is the memory a browser tab spends, +and it is what to look at when a device runs out. The machine was a desktop; +**a phone is several times slower**, so treat the times as a lower bound. + +Every level round tripped to the same 87,342,592 byte tar, so each row is +correct output rather than merely fast output. + +## The table + +| level | compressed | ratio | compress | decompress | encoder heap | decoder heap | window | +| -----: | ------------: | --------: | ---------: | ---------: | -----------: | -----------: | -----: | +| 1 | 6,698,281 | 1.72x | 0.54 s | 0.28 s | 8 MB | 8 MB | 1 MB | +| 2 | 6,600,451 | 1.74x | 0.54 s | 0.29 s | 8 MB | 8 MB | 1 MB | +| 3 | 6,094,116 | 1.89x | 0.57 s | 0.28 s | 8 MB | 8 MB | 2 MB | +| 4 | 6,082,362 | 1.89x | 0.61 s | 0.28 s | 8 MB | 8 MB | 2 MB | +| 5 | 5,280,090 | 2.18x | 0.92 s | 0.28 s | 8 MB | 8 MB | 2 MB | +| 6 | 4,943,366 | 2.33x | 1.13 s | 0.27 s | 8 MB | 8 MB | 2 MB | +| 7 | 4,738,515 | 2.43x | 1.27 s | 0.27 s | 9 MB | 8 MB | 2 MB | +| 8 | 4,514,853 | 2.55x | 1.52 s | 0.28 s | 9 MB | 8 MB | 2 MB | +| 9 | 4,445,368 | 2.59x | 1.51 s | 0.26 s | 16 MB | 8 MB | 4 MB | +| 10 | 4,280,567 | 2.69x | 1.74 s | 0.27 s | 26 MB | 8 MB | 4 MB | +| **11** | **4,157,684** | **2.77x** | **2.44 s** | **0.32 s** | **26 MB** | **8 MB** | 4 MB | +| 12 | 4,155,668 | 2.77x | 2.39 s | 0.27 s | 46 MB | 8 MB | 4 MB | +| 13 | 4,257,141 | 2.70x | 3.76 s | 0.27 s | 38 MB | 8 MB | 4 MB | +| 14 | 4,108,558 | 2.80x | 5.39 s | 0.28 s | 54 MB | 8 MB | 4 MB | +| 15 | 3,968,262 | 2.90x | 8.13 s | 0.33 s | 70 MB | 8 MB | 4 MB | +| 16 | 3,862,217 | 2.98x | 11.63 s | 0.26 s | 38 MB | 8 MB | 4 MB | +| 17 | 3,733,198 | 3.08x | 12.39 s | 0.26 s | 58 MB | 10 MB | 8 MB | +| 18 | 3,705,141 | 3.10x | 15.11 s | 0.26 s | 59 MB | 10 MB | 8 MB | +| 19 | 3,449,475 | 3.33x | 27.78 s | 0.28 s | 91 MB | 10 MB | 8 MB | +| 20 | 3,344,642 | 3.44x | 28.24 s | 0.27 s | 195 MB | 34 MB | 32 MB | +| 21 | 3,285,836 | 3.50x | 33.91 s | 0.30 s | 387 MB | 66 MB | 64 MB | +| 22 | 3,245,980 | 3.54x | 49.02 s | 0.30 s | 835 MB | 130 MB | 128 MB | + +## Picking a level + +**11 is the default**, because it is where the curve flattens. It gives good compression in reasonable time. + +If there will be troubles with this level in the future, just change it in config. + +## Reproducing + +There is no script in the repository for this; it was a one-off measurement. +To repeat it, stream a save through `packSave` once per level in a separate +process, and read `memory.buffer.byteLength` off the encoder and decoder +instances of [@tootallnate/zstd-wasm](https://www.npmjs.com/package/@tootallnate/zstd-wasm) +for the heap figures. diff --git a/eslint.config.mjs b/eslint.config.mjs index c0cc6ef..27732a0 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -3,7 +3,7 @@ import nextCoreWebVitals from 'eslint-config-next/core-web-vitals' import nextTypescript from 'eslint-config-next/typescript' export default defineConfig([ - globalIgnores(['.next/**', 'node_modules/**', 'next-env.d.ts']), + globalIgnores(['.next/**', 'node_modules/**', 'TEMP/**', 'next-env.d.ts']), nextCoreWebVitals, nextTypescript, ]) diff --git a/package-lock.json b/package-lock.json index 475a0d3..e9d4c5c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,9 +8,11 @@ "name": "paradox-save-parser-frontend", "version": "0.1.0", "dependencies": { + "@tootallnate/zstd-wasm": "^0.1", "ag-grid-community": "^36", "ag-grid-react": "^36", "bootstrap": "^5", + "modern-tar": "^0.8", "next": "^16", "react": "^19", "react-dom": "^19" @@ -20,7 +22,7 @@ "@types/react": "^19", "@types/react-dom": "^19", "@typescript/native": "npm:typescript@^7.0", - "cross-env": "^10.1.0", + "cross-env": "^10", "eslint": "^9", "eslint-config-next": "^16", "prettier": "^3", @@ -1402,6 +1404,12 @@ "tslib": "^2.8.0" } }, + "node_modules/@tootallnate/zstd-wasm": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@tootallnate/zstd-wasm/-/zstd-wasm-0.1.0.tgz", + "integrity": "sha512-cIJiBKLTWjIONc2KDlQg9YjBxg99eXLmseWZumZibDb+TAZxWH4btuu4mK4XpNlCAXlkuZsQjebYfIypriDgLw==", + "license": "MIT" + }, "node_modules/@tybys/wasm-util": { "version": "0.10.4", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.4.tgz", @@ -3260,9 +3268,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.434", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.434.tgz", - "integrity": "sha512-7EeFW9OLf1NN9NKQP9xdOE/XKqjxv7JDPP6AEAkHli11+Fae1OaLoNuAf13hQv1PPNtuES0pLQSPk2+fS3s8ww==", + "version": "1.5.438", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.438.tgz", + "integrity": "sha512-AN9xMU1hJiT65LkCUPL1DZm5TCbOPb2Qsm5pYwFLEXp6/qj9SdT4yMiqs7Qqstwvkn1zF7l36SRGt+s6XcJ0FA==", "dev": true, "license": "ISC" }, @@ -5097,6 +5105,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/modern-tar": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.8.5.tgz", + "integrity": "sha512-snEhs+6G5Tjd4I7tLCDOaoln2RgE0bD19RzEKgvgK2hZ5VKy3MpLhLTZ2fWpXSTg4K2cyPwp+VHATFJhxfnOeA==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -5218,9 +5235,9 @@ } }, "node_modules/node-releases": { - "version": "2.0.56", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.56.tgz", - "integrity": "sha512-x0InOIyzgdk+eyaWaRJFH5snEtiImgBgblZ2CyPrLmqqcuMQkEvcDPHbzqbD8eDsSeJbVOjn+crzyzHaM4D+/A==", + "version": "2.0.57", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.57.tgz", + "integrity": "sha512-kQK9LGGFiHtrWiNhZtA7Qbw17AQz+dmsEKODRIVTXA9+e5MS/2gZEBhYJt13GrAz5/IOZKddH/0Z3TP/Zgo+yw==", "dev": true, "license": "MIT", "engines": { @@ -5526,9 +5543,9 @@ } }, "node_modules/prettier": { - "version": "3.9.8", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.8.tgz", - "integrity": "sha512-WRFq3Wn3WId7LLROfMLdH7xaFr2jR62wU8nLO6rQUOLOxNZUviyJQs1M0iIhLexSFy+L+w0ch66wtoO2jRjG0A==", + "version": "3.9.9", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.9.tgz", + "integrity": "sha512-Z/CJHIkdujO/OtN7nXUii0Rf3VT5SRuhjBA82Xvu2XhBUgX3nhP67T0LHceBdQLex7OOFGTox+Q5Yg8Jk2Qivg==", "dev": true, "license": "MIT", "bin": { diff --git a/package.json b/package.json index 1f7d324..96ce5ad 100644 --- a/package.json +++ b/package.json @@ -11,19 +11,21 @@ "format": "npx prettier --write ." }, "dependencies": { - "ag-grid-community": "^36", - "ag-grid-react": "^36", - "bootstrap": "^5", "next": "^16", "react": "^19", - "react-dom": "^19" + "react-dom": "^19", + "bootstrap": "^5", + "ag-grid-community": "^36", + "ag-grid-react": "^36", + "@tootallnate/zstd-wasm": "^0.1", + "modern-tar": "^0.8" }, "devDependencies": { "@types/node": "^26", "@types/react": "^19", "@types/react-dom": "^19", "@typescript/native": "npm:typescript@^7.0", - "cross-env": "^10.1.0", + "cross-env": "^10", "eslint": "^9", "eslint-config-next": "^16", "prettier": "^3", diff --git a/src/app/browse/page.tsx b/src/app/browse/page.tsx index 78a1a05..de02e82 100644 --- a/src/app/browse/page.tsx +++ b/src/app/browse/page.tsx @@ -1,75 +1,29 @@ -'use client' - -import { AgGridReact } from 'ag-grid-react' -import { - AllCommunityModule, - themeQuartz, - colorSchemeDark, -} from 'ag-grid-community' -import { useSyncExternalStore } from 'react' - -const DARK_SCHEME_QUERY = '(prefers-color-scheme: dark)' - -// add the vertical lines between cells -const BASE_THEME = themeQuartz.withParams({ - columnBorder: true, - headerColumnBorder: true, - headerCellBackgroundTransitionDuration: 0, -}) -// Built once, so the grid keeps seeing the same theme object between renders. -const LIGHT_THEME = BASE_THEME -const DARK_THEME = BASE_THEME.withPart(colorSchemeDark) - -// Notifies React whenever the browser's light/dark color scheme changes. -function subscribeToColorScheme(onChange: () => void) { - const mediaQuery = window.matchMedia(DARK_SCHEME_QUERY) - mediaQuery.addEventListener('change', onChange) - return () => mediaQuery.removeEventListener('change', onChange) -} - -function isDarkSchemePreferred() { - return window.matchMedia(DARK_SCHEME_QUERY).matches -} - -// The server has no color scheme, so it always prerenders the light theme. -function isDarkSchemePreferredOnServer() { - return false -} +import BrowseSaveDataGrid from '@/components/BrowseSaveDataGrid' +import ConditionalView from '@/components/ConditionalView' +import SaveDownloadButton from '@/components/SaveDownloadButton' // TODO: create tabs with general info, tables, plots +// TODO: feed the grid with the parsed save the id points at + +interface BrowseSaveDataPageProps { + searchParams: Promise<{ id?: string }> +} // Page that shows a save file's parsed data in a grid. -export default function BrowseSaveDataPage() { - // Read the color scheme straight from the browser instead of mirroring it - // into state, which would need an extra render to catch up. - const prefersDark = useSyncExternalStore( - subscribeToColorScheme, - isDarkSchemePreferred, - isDarkSchemePreferredOnServer - ) - const agTheme = prefersDark ? DARK_THEME : LIGHT_THEME +export default async function BrowseSaveDataPage( + props: BrowseSaveDataPageProps +) { + const { id } = await props.searchParams return ( -
- +
+ +
+ +
+ + +
) } diff --git a/src/app/page.tsx b/src/app/page.tsx index 091f406..2261daf 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -1,11 +1,14 @@ import SaveFileUploadingDialog from '@/components/SaveFileUploadingDialog' +import { appConfig } from '@/lib/configLoader' export default function HomePage() { return (

Welcome to Paradox Save Parser!

- + {/* The config is read on the server, so the limits are handed to + the dialog instead of being fetched by it. */} +
) } diff --git a/src/components/BrowseSaveDataGrid.tsx b/src/components/BrowseSaveDataGrid.tsx new file mode 100644 index 0000000..5a99a9d --- /dev/null +++ b/src/components/BrowseSaveDataGrid.tsx @@ -0,0 +1,72 @@ +'use client' + +import { AgGridReact } from 'ag-grid-react' +import { + AllCommunityModule, + themeQuartz, + colorSchemeDark, +} from 'ag-grid-community' +import { useSyncExternalStore } from 'react' + +const DARK_SCHEME_QUERY = '(prefers-color-scheme: dark)' + +// add the vertical lines between cells +// Built once, so the grid keeps seeing the same theme object between renders. +const LIGHT_THEME = themeQuartz.withParams({ + columnBorder: true, + headerColumnBorder: true, + headerCellBackgroundTransitionDuration: 0, +}) +const DARK_THEME = LIGHT_THEME.withPart(colorSchemeDark) + +// Notifies React whenever the browser's light/dark color scheme changes. +function subscribeToColorScheme(onChange: () => void) { + const mediaQuery = window.matchMedia(DARK_SCHEME_QUERY) + mediaQuery.addEventListener('change', onChange) + return () => mediaQuery.removeEventListener('change', onChange) +} + +function isDarkSchemePreferred() { + return window.matchMedia(DARK_SCHEME_QUERY).matches +} + +// The server has no color scheme, so it always prerenders the light theme. +function isDarkSchemePreferredOnServer() { + return false +} + +// Grid showing a save file's parsed data. +export default function BrowseSaveDataGrid() { + // Read the color scheme straight from the browser instead of mirroring it + // into state, which would need an extra render to catch up. + const prefersDark = useSyncExternalStore( + subscribeToColorScheme, + isDarkSchemePreferred, + isDarkSchemePreferredOnServer + ) + const agTheme = prefersDark ? DARK_THEME : LIGHT_THEME + + return ( +
+ +
+ ) +} diff --git a/src/components/ConditionalView.tsx b/src/components/ConditionalView.tsx new file mode 100644 index 0000000..8e50db0 --- /dev/null +++ b/src/components/ConditionalView.tsx @@ -0,0 +1,13 @@ +interface ConditionalViewProps { + cond: unknown + children: React.ReactNode +} + +// Shows its children only when `cond` is truthy, and nothing otherwise. +// Unlike `cond && `, a falsy value here never leaks into the page: JSX +// renders 0 and '' as text, so an empty string or a count of zero would show up. +export default function ConditionalView(props: ConditionalViewProps) { + if (!props.cond) return null + + return <>{props.children} +} diff --git a/src/components/SaveDownloadButton.tsx b/src/components/SaveDownloadButton.tsx new file mode 100644 index 0000000..683bca7 --- /dev/null +++ b/src/components/SaveDownloadButton.tsx @@ -0,0 +1,73 @@ +'use client' + +import { useState } from 'react' +import ConditionalView from './ConditionalView' +import FormError from './FormError' +import { downloadBlobAsFile } from '@/lib/downloadFile' +import { getMsgFromError } from '@/lib/errors' +import { rebuildOriginalSave } from '@/lib/saveArchive/client' +import { SAVE_PROGRESS_STAGE_LABELS } from '@/lib/saveArchive/messages' +import { + saveParserBackendService, + SaveStatusResponse, +} from '@/services/SaveParserBackendService' + +interface SaveDownloadButtonProps { + saveId: string + // Status of the save, when the caller already has it. It is fetched on + // demand otherwise, since the download needs the shape the save was + // uploaded in and its file name. + status?: SaveStatusResponse | null +} + +// Downloads a stored save and rebuilds the file the game originally wrote. +export default function SaveDownloadButton(props: SaveDownloadButtonProps) { + const [error, setError] = useState(null) + const [progress, setProgress] = useState(null) + + const handleDownload = async () => { + setError(null) + try { + const status = + props.status ?? + (await saveParserBackendService.getSaveStatus(props.saveId)) + + setProgress(SAVE_PROGRESS_STAGE_LABELS.downloading) + const archive = await saveParserBackendService.getSaveFile( + props.saveId + ) + + const saveFile = await rebuildOriginalSave( + archive, + status.originalFileType, + (stage) => setProgress(SAVE_PROGRESS_STAGE_LABELS[stage]) + ) + downloadBlobAsFile(saveFile, status.originalFileName) + } catch (err) { + const errMsg = getMsgFromError(err) + console.error('Error while downloading save:', errMsg) + setError(errMsg) + } finally { + setProgress(null) + } + } + + return ( +
+ + + + + +
{progress}
+ +
+ ) +} diff --git a/src/components/SaveFileUploadingDialog.tsx b/src/components/SaveFileUploadingDialog.tsx index 1456027..dca0da1 100644 --- a/src/components/SaveFileUploadingDialog.tsx +++ b/src/components/SaveFileUploadingDialog.tsx @@ -2,30 +2,34 @@ import { saveParserBackendService, - SaveStatusResponse, UploadSaveResponse, } from '@/services/SaveParserBackendService' +import Link from 'next/link' import { useState } from 'react' -import FormError from './FormError' +import ConditionalView from './ConditionalView' +import type { SaveUploadConfig } from '@/config/AppConfig' +import { GameEnum } from '@/config/games' import { getMsgFromError } from '@/lib/errors' +import { SAVE_PROGRESS_STAGE_LABELS } from '@/lib/saveArchive/messages' -export enum GameEnum { - EU4 = 'eu4', - Stellaris = 'stellaris', +interface SaveFileUploadingDialogProps { + // Comes from the server, where runtimeConfig.json is read. + saveUploadConfig: SaveUploadConfig } -//TODO: move to separate page -//TODO: add redirect to ?id={id} to see status of long parsing save +const IN_BROWSER = '[In Browser]' +const ON_SERVER = '[On Server]' // Dialog for picking a game and save file, uploading it, and polling the parse status until done. -export default function SaveFileUploadingDialog() { - const [error, setError] = useState(null) +export default function SaveFileUploadingDialog( + props: SaveFileUploadingDialogProps +) { const [selectedFile, setSelectedFile] = useState(null) const [game, setGame] = useState(GameEnum.EU4) const [uploadDisabled, setUploadDisabled] = useState(false) - const [saveStatus, setSaveStatus] = useState( - null - ) + // This is what user will see + const [statusMessage, setStatusMessage] = useState(null) + const [uploadResult, setUploadResult] = useState( null ) @@ -42,10 +46,10 @@ export default function SaveFileUploadingDialog() { } } - // Uploads the selected save file, then repeatedly polls its status until parsing is done or an error occurs. + // Compresses and uploads the selected save file, then repeatedly polls its status until parsing is done or an error occurs. const handleUpload = async (e: React.FormEvent) => { e.preventDefault() - setError(null) + setStatusMessage(null) setUploadDisabled(true) try { if (!selectedFile) return @@ -53,10 +57,17 @@ export default function SaveFileUploadingDialog() { console.log('Uploading file:', selectedFile.name) const uploadResponse = await saveParserBackendService.uploadSave( game, - selectedFile + selectedFile, + props.saveUploadConfig, + (stage) => + setStatusMessage( + `${IN_BROWSER} ${SAVE_PROGRESS_STAGE_LABELS[stage]}` + ) ) console.log('Upload response:', uploadResponse) - setUploadResult(uploadResponse) + setStatusMessage( + `${ON_SERVER} Waiting for the save to be parsed...` + ) const pollInterval = 1000 // ms // Poll the backend every second for the parsing status until it reports "done". @@ -67,23 +78,25 @@ export default function SaveFileUploadingDialog() { uploadResponse.id ) console.log('Save status:', statusResponse.status) - setSaveStatus(statusResponse) + setStatusMessage(`${ON_SERVER} ${statusResponse.status}`) if (statusResponse.status.toLowerCase() === 'done') { clearInterval(intervalId) + // Only a parsed save is worth showing or browsing. + setUploadResult(uploadResponse) setUploadDisabled(false) } } catch (err) { const errMsg = getMsgFromError(err) console.error('Error while polling status:', errMsg) clearInterval(intervalId) - setError(errMsg) + setStatusMessage(`${ON_SERVER} ${errMsg}`) setUploadDisabled(false) } }, pollInterval) } catch (err) { const errMsg = getMsgFromError(err) console.error(errMsg) - setError(errMsg) + setStatusMessage(`${IN_BROWSER} ${errMsg}`) setUploadDisabled(false) } } @@ -97,8 +110,6 @@ export default function SaveFileUploadingDialog() { return (
- -
) diff --git a/src/config/AppConfig.ts b/src/config/AppConfig.ts index 94f6b31..91c36c8 100644 --- a/src/config/AppConfig.ts +++ b/src/config/AppConfig.ts @@ -5,9 +5,19 @@ export const defaultConfig = { saveParserBackend: { url: 'http://127.0.0.1:5226', }, - // TODO: implement auth api - auth: {}, + }, + saveUpload: { + // Size of a save if it is uploaded as uncompressed. The browser packs + // saves as a stream, so these limits are about the backend's disk + // rather than about memory. + maxSizePlainTextBytes: 200 * 1024 * 1024, + // Archives produced by the game, recognized by the zip magic bytes. + maxSizeCompressedBytes: 60 * 1024 * 1024, + // read COMPRESSION.md + zstdCompressionLevel: 11, }, } export type AppConfig = typeof defaultConfig + +export type SaveUploadConfig = AppConfig['saveUpload'] diff --git a/src/config/games.ts b/src/config/games.ts new file mode 100644 index 0000000..9b187c5 --- /dev/null +++ b/src/config/games.ts @@ -0,0 +1,20 @@ +// Games the parser backend can handle. The values go into the upload request. +//TODO: add more games +export enum GameEnum { + EU4 = 'eu4', + Stellaris = 'stellaris', +} + +/** + * Name the game state gets inside a save archive, per game. + * + * A save produced by the game is either a zip archive holding this file (plus + * companion files like `meta`), or the very same content as plain text. When a + * plain text save is uploaded, this is the name it is stored under, so the + * backend always reads the 'gamestate' or whatever is set for the specific game. + * The parser backend keeps the same map on its side. + */ +export const GAME_SAVE_ENTRY_NAMES: Record = { + [GameEnum.EU4]: 'gamestate', + [GameEnum.Stellaris]: 'gamestate', +} diff --git a/src/lib/downloadFile.ts b/src/lib/downloadFile.ts new file mode 100644 index 0000000..b26dd26 --- /dev/null +++ b/src/lib/downloadFile.ts @@ -0,0 +1,17 @@ +// Revoking the object url right after the click can cancel the download in +// some browsers, so it is kept alive for a while first. +const URL_LIFETIME_MS = 20_000 + +/** + * Makes the browser save a blob to disk under the given name. + */ +export function downloadBlobAsFile(blob: Blob, fileName: string) { + const url = URL.createObjectURL(blob) + + const link = document.createElement('a') + link.href = url + link.download = fileName + link.click() + + setTimeout(() => URL.revokeObjectURL(url), URL_LIFETIME_MS) +} diff --git a/src/lib/formatBytes.ts b/src/lib/formatBytes.ts new file mode 100644 index 0000000..6f565bd --- /dev/null +++ b/src/lib/formatBytes.ts @@ -0,0 +1,11 @@ +const KILOBYTE = 1024 +const MEGABYTE = 1024 * KILOBYTE + +/** + * Formats a byte count for error messages and status lines, e.g. "12.3 MB". + */ +export function formatBytes(bytes: number): string { + if (bytes >= MEGABYTE) return `${(bytes / MEGABYTE).toFixed(1)} MB` + if (bytes >= KILOBYTE) return `${(bytes / KILOBYTE).toFixed(1)} KB` + return `${bytes} B` +} diff --git a/src/lib/saveArchive/archive.ts b/src/lib/saveArchive/archive.ts new file mode 100644 index 0000000..528131e --- /dev/null +++ b/src/lib/saveArchive/archive.ts @@ -0,0 +1,138 @@ +// Converting between the save files a game reads and the archive the parser +// backend stores. The tar always holds the files the game itself would have in +// its archive, so the backend reads the game state from the same entry either +// way, and a download rebuilds whichever shape was uploaded: +// +// plain text save -> tar { "gamestate" } -> zstd +// archived save -> tar { "meta", "gamestate", ... } -> zstd +// +// Everything runs as a stream: a save is never held in memory as a whole. + +import { createTarDecoder, createTarPacker } from 'modern-tar' +import { createGameZipBuilder, readGameZip } from './gameZip' +import { OriginalFileType, SaveProgressStage } from './messages' +import { + createZstdCompressStream, + createZstdDecompressStream, +} from './zstdStream' + +type StageCallback = (stage: SaveProgressStage) => void + +// A fixed timestamp keeps uploads of the same save byte for byte identical. +const TAR_ENTRY_MTIME = new Date(0) + +export interface PackSaveOptions { + /** Whether the file is an archive produced by the game. */ + isZip: boolean + /** Name of the single entry used for a plain text save. */ + entryName: string + compressionLevel: number +} + +// Collects a stream into a blob, whose storage the browser manages. The blob +// takes its type from the response's Content-Type. +function collectBlob( + stream: ReadableStream, + type: string +): Promise { + return new Response(stream, { headers: { 'Content-Type': type } }).blob() +} + +// Writes the files of the save into the tar, in the order the game stored +// them. Runs alongside the compression, which consumes the tar as it is built. +async function packEntries( + file: Blob, + options: PackSaveOptions, + controller: ReturnType['controller'] +) { + try { + if (options.isZip) { + await readGameZip(file, (entry) => + // Returns writable stream. + // Zip decoder decodes file entry stream and pipes it to the tar entry stream. + controller.add({ + name: entry.name, + size: entry.size, + mtime: TAR_ENTRY_MTIME, + }) + ) + } else { + // Writes text file to tar entry + await file.stream().pipeTo( + controller.add({ + name: options.entryName, + size: file.size, + mtime: TAR_ENTRY_MTIME, + }) + ) + } + } finally { + controller.finalize() + } +} + +/** + * Compresses a save file the user picked into the archive that is uploaded. + */ +export async function packSave( + file: Blob, + options: PackSaveOptions, + onStage: StageCallback +): Promise { + onStage('compressing') + + const { readable, controller } = createTarPacker() + // The tar is written while it is being compressed, so neither side of the + // pipe ever holds the whole save. + const packed = packEntries(file, options, controller) + const compressed = collectBlob( + readable.pipeThrough( + createZstdCompressStream(options.compressionLevel) + ), + 'application/zstd' + ) + + const [, archive] = await Promise.all([packed, compressed]) + return archive +} + +/** + * Turns a stored archive back into a file the game can load. + */ +export async function rebuildSave( + archive: Blob, + originalFileType: OriginalFileType, + onStage: StageCallback +): Promise { + onStage('decompressing') + + const entries = archive + .stream() + .pipeThrough(createZstdDecompressStream()) + .pipeThrough(createTarDecoder()) + .getReader() + + const zipBuilder = + originalFileType === 'zip' ? createGameZipBuilder() : null + let saveFile: Blob | null = null + + for (;;) { + const { done, value: entry } = await entries.read() + if (done) break + + if (zipBuilder) { + onStage('rebuilding') + await zipBuilder.add(entry.header.name, entry.body) + } else if (saveFile) { + // A plain text save is a single file, so anything after it would + // be silently dropped. + await entry.body.cancel() + } else { + saveFile = await collectBlob(entry.body, 'application/octet-stream') + } + } + + if (zipBuilder) return await zipBuilder.finish() + if (!saveFile) throw new Error('Stored save archive is empty') + return saveFile +} diff --git a/src/lib/saveArchive/client.ts b/src/lib/saveArchive/client.ts new file mode 100644 index 0000000..62ddb52 --- /dev/null +++ b/src/lib/saveArchive/client.ts @@ -0,0 +1,113 @@ +// Main thread side of the save archive worker: checks the file, starts the +// worker and reports what it is doing. + +import type { SaveUploadConfig } from '@/config/AppConfig' +import { GAME_SAVE_ENTRY_NAMES, GameEnum } from '@/config/games' +import { formatBytes } from '@/lib/formatBytes' +import { isZipFile } from './gameZip' +import { + OriginalFileType, + SaveArchiveRequest, + SaveArchiveResponse, + SaveProgressStage, +} from './messages' + +export type StageCallback = (stage: SaveProgressStage) => void + +export interface PackedSave { + data: Blob + originalFileType: OriginalFileType + originalFileName: string +} + +// Runs a single request in a fresh worker, which is terminated once it answers. +function runInWorker( + request: SaveArchiveRequest, + onStage: StageCallback +): Promise { + return new Promise((resolve, reject) => { + const worker = new Worker(new URL('./worker.ts', import.meta.url), { + type: 'module', + }) + + worker.onmessage = (event: MessageEvent) => { + const message = event.data + if (message.kind === 'progress') { + onStage(message.stage) + return + } + + worker.terminate() + if (message.kind === 'error') { + reject(new Error(message.message)) + } else { + resolve(message.data) + } + } + + worker.onerror = (event) => { + worker.terminate() + reject(new Error(event.message || 'Save file compression failed')) + } + + worker.postMessage(request) + }) +} + +/** + * Compresses a save file the user picked into the archive that is uploaded. + * + * Plain text and archived saves are both accepted, are told apart by the zip + * magic bytes rather than by the file extension, and are checked against their + * own size limit. + */ +export async function packSaveForUpload( + game: GameEnum, + file: File, + config: SaveUploadConfig, + onStage: StageCallback +): Promise { + const isZip = await isZipFile(file) + const sizeLimit = isZip + ? config.maxSizeCompressedBytes + : config.maxSizePlainTextBytes + + if (file.size > sizeLimit) { + const saveKind = isZip ? 'archived' : 'plain text' + throw new Error( + `Save file is ${formatBytes(file.size)}, but the limit for ` + + `${saveKind} saves is ${formatBytes(sizeLimit)}` + ) + } + + const data = await runInWorker( + { + kind: 'pack', + file, + isZip, + entryName: GAME_SAVE_ENTRY_NAMES[game], + compressionLevel: config.zstdCompressionLevel, + }, + onStage + ) + + return { + data, + originalFileType: isZip ? 'zip' : 'text', + originalFileName: file.name, + } +} + +/** + * Rebuilds the save file as the game wrote it, out of a stored archive. + */ +export async function rebuildOriginalSave( + archive: Blob, + originalFileType: OriginalFileType, + onStage: StageCallback +): Promise { + return await runInWorker( + { kind: 'rebuild', data: archive, originalFileType }, + onStage + ) +} diff --git a/src/lib/saveArchive/gameZip.ts b/src/lib/saveArchive/gameZip.ts new file mode 100644 index 0000000..a4bea85 --- /dev/null +++ b/src/lib/saveArchive/gameZip.ts @@ -0,0 +1,313 @@ +// Reading and writing the zip archives the Paradox games themselves use. +// +// This is written here rather than taken from a library because the games are +// picky about the headers and no browser zip library can produce them: they +// all derive the DOS date from a Date and clamp it to 1980, while the games +// write zeros, and the streaming writers emit a data descriptor plus a Zip64 +// extra field for every entry, which the games reject. The heavy part of zip, +// deflate, is in the browser itself as CompressionStream('deflate-raw'), so +// what is left is the header layout below. +// +// The field values follow a known good reference, which produces archives the +// games accept: http://pastebin.com/nyX1B4rH +// A real EU4 save agrees with it: MS-DOS as the system of origin, encoding +// software 6.3, extract version 2.0, no data descriptor, no extra field and a +// zeroed DOS date. 7-Zip's NTFS timestamps are a known way to produce a save +// that will not load: https://stellaris.paradoxwikis.com/Save-game_editing + +const LOCAL_HEADER_SIGNATURE = 0x04034b50 +const CENTRAL_HEADER_SIGNATURE = 0x02014b50 +const END_OF_CENTRAL_DIR_SIGNATURE = 0x06054b50 + +const LOCAL_HEADER_SIZE = 30 +const CENTRAL_HEADER_SIZE = 46 +const END_OF_CENTRAL_DIR_SIZE = 22 + +const METHOD_STORED = 0 +const METHOD_DEFLATE = 8 + +// "version needed to extract" 2.0, which is what deflate requires. +const EXTRACT_VERSION = 20 +// "version made by": MS-DOS (0) in the high byte, 6.3 (63) in the low byte. +const CREATE_VERSION = (0 << 8) | 63 + +// Anything at or above this needs the zip64 format, which no save comes near. +const MAX_ENTRY_SIZE = 0xffffffff + +// First bytes of every zip archive. +const ZIP_MAGIC = [0x50, 0x4b, 0x03, 0x04] + +export interface GameZipEntry { + name: string + size: number +} + +// The standard CRC-32 lookup table. +const CRC_TABLE = new Uint32Array(256).map((_, i) => { + let value = i + for (let bit = 0; bit < 8; bit++) { + value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1 + } + return value >>> 0 +}) + +// Continues a CRC-32 over one more chunk. Start with 0; the result is the +// checksum of everything so far, so an entry is summed as it streams past. +function updateCrc(crc: number, data: Uint8Array): number { + let value = ~crc + for (const byte of data) { + value = CRC_TABLE[(value ^ byte) & 0xff] ^ (value >>> 8) + } + return ~value >>> 0 +} + +/** + * Tells whether the file is a zip archive, by its magic bytes rather than by + * its extension: the games use the same extension for both an archived and a + * plain text save. + */ +export async function isZipFile(file: Blob): Promise { + const header = new Uint8Array( + await file.slice(0, ZIP_MAGIC.length).arrayBuffer() + ) + return ZIP_MAGIC.every((byte, i) => header[i] === byte) +} + +// Little endian reader over a slice of the archive. +function reader(data: Uint8Array) { + const view = new DataView(data.buffer, data.byteOffset, data.byteLength) + return { + u16: (offset: number) => view.getUint16(offset, true), + u32: (offset: number) => view.getUint32(offset, true), + } +} + +interface CentralEntry { + name: string + method: number + compressedSize: number + size: number + localHeaderOffset: number +} + +// Reads the central directory, which lists every entry with its size and +// position. Only the last few hundred bytes of the archive are touched. +async function readCentralDirectory(zip: Blob): Promise { + const end = new Uint8Array( + await zip.slice(zip.size - END_OF_CENTRAL_DIR_SIZE).arrayBuffer() + ) + const endFields = reader(end) + if (endFields.u32(0) !== END_OF_CENTRAL_DIR_SIGNATURE) { + throw new Error('Save archive has no end of central directory') + } + + const entryCount = endFields.u16(10) + const directoryOffset = endFields.u32(16) + const directory = new Uint8Array( + await zip.slice(directoryOffset, zip.size).arrayBuffer() + ) + const fields = reader(directory) + + const entries: CentralEntry[] = [] + let offset = 0 + for (let i = 0; i < entryCount; i++) { + if (fields.u32(offset) !== CENTRAL_HEADER_SIGNATURE) { + throw new Error('Save archive has a corrupted entry header') + } + + const nameLength = fields.u16(offset + 28) + const extraLength = fields.u16(offset + 30) + const commentLength = fields.u16(offset + 32) + const nameStart = offset + CENTRAL_HEADER_SIZE + entries.push({ + name: new TextDecoder().decode( + directory.subarray(nameStart, nameStart + nameLength) + ), + method: fields.u16(offset + 10), + compressedSize: fields.u32(offset + 20), + size: fields.u32(offset + 24), + localHeaderOffset: fields.u32(offset + 42), + }) + offset += CENTRAL_HEADER_SIZE + nameLength + extraLength + commentLength + } + return entries +} + +/** + * Reads every file of a save archive in the order they are stored, asking the + * caller where each one should go and writing it straight into that stream. + * + * Sizes come from the central directory, so they are known before an entry + * starts streaming, and the entry data itself is never held in memory. + */ +export async function readGameZip( + zip: Blob, + openEntry: (entry: GameZipEntry) => WritableStream +): Promise { + for (const entry of await readCentralDirectory(zip)) { + // The local header repeats the name and may carry a different extra + // field, so the data can only be located through it. + const header = new Uint8Array( + await zip + .slice( + entry.localHeaderOffset, + entry.localHeaderOffset + LOCAL_HEADER_SIZE + ) + .arrayBuffer() + ) + const fields = reader(header) + if (fields.u32(0) !== LOCAL_HEADER_SIGNATURE) { + throw new Error(`Save archive entry "${entry.name}" is corrupted`) + } + + const dataStart = + entry.localHeaderOffset + + LOCAL_HEADER_SIZE + + fields.u16(26) + + fields.u16(28) + const data = zip.slice(dataStart, dataStart + entry.compressedSize) + + if (entry.method === METHOD_STORED) { + await data.stream().pipeTo(openEntry(entry)) + } else if (entry.method === METHOD_DEFLATE) { + await data + .stream() + .pipeThrough(new DecompressionStream('deflate-raw')) + .pipeTo(openEntry(entry)) + } else { + throw new Error( + `Save archive entry "${entry.name}" uses compression ` + + `method ${entry.method}, which the games do not produce` + ) + } + } +} + +// Writes a local file header. The date and time fields stay zero, which is +// what the games write and what a Date cannot express. +function localHeader(entry: WrittenEntry): Uint8Array { + const header = new Uint8Array(LOCAL_HEADER_SIZE + entry.nameBytes.length) + const view = new DataView(header.buffer) + view.setUint32(0, LOCAL_HEADER_SIGNATURE, true) + view.setUint16(4, EXTRACT_VERSION, true) + view.setUint16(8, METHOD_DEFLATE, true) + view.setUint32(14, entry.crc, true) + view.setUint32(18, entry.compressedSize, true) + view.setUint32(22, entry.size, true) + view.setUint16(26, entry.nameBytes.length, true) + header.set(entry.nameBytes, LOCAL_HEADER_SIZE) + return header +} + +// Writes the central directory record for one entry. +function centralHeader(entry: WrittenEntry): Uint8Array { + const header = new Uint8Array(CENTRAL_HEADER_SIZE + entry.nameBytes.length) + const view = new DataView(header.buffer) + view.setUint32(0, CENTRAL_HEADER_SIGNATURE, true) + view.setUint16(4, CREATE_VERSION, true) + view.setUint16(6, EXTRACT_VERSION, true) + view.setUint16(10, METHOD_DEFLATE, true) + view.setUint32(16, entry.crc, true) + view.setUint32(20, entry.compressedSize, true) + view.setUint32(24, entry.size, true) + view.setUint16(28, entry.nameBytes.length, true) + view.setUint32(42, entry.localHeaderOffset, true) + header.set(entry.nameBytes, CENTRAL_HEADER_SIZE) + return header +} + +// Writes the end of central directory record that closes the archive. +function endOfCentralDirectory( + entries: WrittenEntry[], + directoryOffset: number, + directorySize: number +): Uint8Array { + const record = new Uint8Array(END_OF_CENTRAL_DIR_SIZE) + const view = new DataView(record.buffer) + view.setUint32(0, END_OF_CENTRAL_DIR_SIGNATURE, true) + view.setUint16(8, entries.length, true) + view.setUint16(10, entries.length, true) + view.setUint32(12, directorySize, true) + view.setUint32(16, directoryOffset, true) + return record +} + +interface WrittenEntry { + nameBytes: Uint8Array + crc: number + size: number + compressedSize: number + localHeaderOffset: number +} + +/** + * Builds a save archive the games accept, one streamed file at a time. + */ +export function createGameZipBuilder() { + const entries: WrittenEntry[] = [] + // Headers are plain bytes and entry data is a blob, so assembling the + // archive at the end copies nothing through memory. + const parts: BlobPart[] = [] + let offset = 0 + + return { + async add(name: string, body: ReadableStream) { + const nameBytes = new TextEncoder().encode(name) + let crc = 0 + let size = 0 + + // Sums the entry as it goes past. The cast only drops the shared + // buffer case from the type; a save never arrives in one. + const summed = body.pipeThrough( + new TransformStream>({ + transform(chunk, controller) { + crc = updateCrc(crc, chunk) + size += chunk.length + controller.enqueue(chunk as Uint8Array) + }, + }) + ) + + // The sizes and the checksum belong in the local header, which is + // written before the data, so the entry is deflated first. Only + // the compressed bytes are kept, and in a blob rather than in the + // javascript heap. + const compressed = await new Response( + summed.pipeThrough(new CompressionStream('deflate-raw')) + ).blob() + + if (size > MAX_ENTRY_SIZE) { + throw new Error(`Save file "${name}" is too large for a zip`) + } + + const entry: WrittenEntry = { + nameBytes, + crc, + size, + compressedSize: compressed.size, + localHeaderOffset: offset, + } + entries.push(entry) + + const header = localHeader(entry) + parts.push(header, compressed) + offset += header.length + compressed.size + }, + + async finish(): Promise { + const directory = entries.map(centralHeader) + const directorySize = directory.reduce( + (total, header) => total + header.length, + 0 + ) + return new Blob( + [ + ...parts, + ...directory, + endOfCentralDirectory(entries, offset, directorySize), + ], + { type: 'application/zip' } + ) + }, + } +} diff --git a/src/lib/saveArchive/messages.ts b/src/lib/saveArchive/messages.ts new file mode 100644 index 0000000..031aa3c --- /dev/null +++ b/src/lib/saveArchive/messages.ts @@ -0,0 +1,44 @@ +// Types shared between the save archive worker and the code that drives it. +// This module deliberately imports nothing, so importing it never pulls the +// compression code into a bundle. + +/** Shape the user originally uploaded, and the shape a download rebuilds. */ +export type OriginalFileType = 'text' | 'zip' + +/** + * Steps worth showing to the user, since each of them takes seconds on a big + * save. The worker reports the ones it runs itself; uploading and downloading + * are reported by the service around it. + */ +export type SaveProgressStage = + 'compressing' | 'uploading' | 'downloading' | 'decompressing' | 'rebuilding' + +export type SaveArchiveRequest = + | { + kind: 'pack' + file: File + isZip: boolean + entryName: string + compressionLevel: number + } + | { + kind: 'rebuild' + data: Blob + originalFileType: OriginalFileType + } + +export type SaveArchiveResponse = + // Blobs are handles to storage the browser manages, so passing one between + // threads copies nothing. + | { kind: 'progress'; stage: SaveProgressStage } + | { kind: 'done'; data: Blob } + | { kind: 'error'; message: string } + +/** Text shown while a stage is running. */ +export const SAVE_PROGRESS_STAGE_LABELS: Record = { + compressing: 'Compressing save file...', + uploading: 'Uploading save file...', + downloading: 'Downloading save file...', + decompressing: 'Decompressing save file...', + rebuilding: 'Rebuilding save file...', +} diff --git a/src/lib/saveArchive/worker.ts b/src/lib/saveArchive/worker.ts new file mode 100644 index 0000000..921b818 --- /dev/null +++ b/src/lib/saveArchive/worker.ts @@ -0,0 +1,66 @@ +// Worker that packs and rebuilds save archives. +// +// Compressing a save is seconds of solid cpu work, so on the main thread it +// would freeze the page long enough for the browser to offer to kill it. +// Keeping the compression code here also keeps it out of every other bundle, +// including the 534 kB zstd wasm. + +import { getMsgFromError } from '@/lib/errors' +import { packSave, rebuildSave } from './archive' +import { + SaveArchiveRequest, + SaveArchiveResponse, + SaveProgressStage, +} from './messages' + +// The part of the worker global scope this file uses. It is declared here +// because the project compiles against the dom types, where `self` is a window +// whose postMessage takes a target origin; the worker types live in a lib that +// cannot be loaded alongside them. Naming the message types also checks both +// ends of the protocol, which DedicatedWorkerGlobalScope would leave as `any`. +interface SaveArchiveWorkerScope { + postMessage(message: SaveArchiveResponse, transfer?: Transferable[]): void + addEventListener( + type: 'message', + listener: (event: MessageEvent) => void + ): void +} + +const workerScope = self as unknown as SaveArchiveWorkerScope + +function reportStage(stage: SaveProgressStage) { + workerScope.postMessage({ kind: 'progress', stage }) +} + +// Runs one request and answers with its result or with the error it failed on. +async function handleRequest(request: SaveArchiveRequest) { + try { + const result = + request.kind === 'pack' + ? await packSave( + request.file, + { + isZip: request.isZip, + entryName: request.entryName, + compressionLevel: request.compressionLevel, + }, + reportStage + ) + : await rebuildSave( + request.data, + request.originalFileType, + reportStage + ) + + workerScope.postMessage({ kind: 'done', data: result }) + } catch (err) { + workerScope.postMessage({ + kind: 'error', + message: getMsgFromError(err), + }) + } +} + +workerScope.addEventListener('message', (event) => { + void handleRequest(event.data) +}) diff --git a/src/lib/saveArchive/zstdStream.ts b/src/lib/saveArchive/zstdStream.ts new file mode 100644 index 0000000..efdf550 --- /dev/null +++ b/src/lib/saveArchive/zstdStream.ts @@ -0,0 +1,58 @@ +// Zstandard compression and decompression as stream transforms. +// +// Saves are re-compressed in the browser because the parser backend runs on a +// cheap two core server with little disk space. zstd was picked over lzma2 and +// brotli because its decompression speed does not depend on the level the data +// was compressed with, so the level can be raised without costing the server +// anything, and because its wasm build is by far the smallest of the three. +// +// @tootallnate/zstd-wasm does the work: it is upstream zstd 1.5.7 built with +// wasi-sdk and driven through ZSTD_compressStream2, so it compresses a stream +// as it arrives rather than a buffer at a time. A save of any size comes out +// as one zstd frame, and memory stays at the library's own input and output +// buffers. + +import { + ZstdCompressStream, + ZstdDecompressStream, +} from '@tootallnate/zstd-wasm' + +// The wasm file ships inside the package. Turbopack rewrites this into the URL +// it emits the asset at, so the 534 kB binary is fetched once at runtime and +// cached by the browser instead of being inlined into the worker bundle. +const wasmUrl = new URL( + '../../../node_modules/@tootallnate/zstd-wasm/dist/zstd.wasm', + import.meta.url +) + +// Compiling takes a moment, so it happens once and both directions share it. +let wasmModule: Promise | null = null + +function getWasm(): Promise { + // compileStreaming would need the server to send Content-Type: + // application/wasm, which is not worth depending on. + wasmModule ??= fetch(wasmUrl) + .then((res) => res.arrayBuffer()) + .then((bytes) => WebAssembly.compile(bytes)) + return wasmModule +} + +/** + * Compresses a stream into a single zstd frame. The level goes from 1 to 22. + */ +export function createZstdCompressStream( + compressionLevel: number +): TransformStream { + return new ZstdCompressStream(getWasm(), compressionLevel) +} + +/** + * Decompresses a zstd stream. Frames that follow one another are read straight + * through, so an archive packed as several frames still opens. + */ +export function createZstdDecompressStream(): TransformStream< + Uint8Array, + Uint8Array +> { + return new ZstdDecompressStream(getWasm()) +} diff --git a/src/services/AuthService.ts b/src/services/AuthService.ts index 10f6e6a..b33d6dc 100644 --- a/src/services/AuthService.ts +++ b/src/services/AuthService.ts @@ -2,14 +2,12 @@ // already uses Discord, so email/password login is not worth building. The // /api/auth/login and /api/auth/register routes below are never implemented — // delete them along with this service. -//TODO: controllable rate limits class AuthService { private API_BASE = '/api/auth' // Hashes the password before sending it to the backend private hashPassword(password: string) { - // TODO: password hashing return password } diff --git a/src/services/SaveParserBackendService.ts b/src/services/SaveParserBackendService.ts index 2149da0..822dffa 100644 --- a/src/services/SaveParserBackendService.ts +++ b/src/services/SaveParserBackendService.ts @@ -1,4 +1,8 @@ import { PROXY_PATHS } from '@/app/api/proxy/paths' +import type { SaveUploadConfig } from '@/config/AppConfig' +import { GameEnum } from '@/config/games' +import { packSaveForUpload, StageCallback } from '@/lib/saveArchive/client' +import { OriginalFileType } from '@/lib/saveArchive/messages' export interface UploadSaveResponse { id: string @@ -9,6 +13,10 @@ export interface SaveStatusResponse { game: string status: string uploadDateTime: string + // Shape the save had when it was uploaded, needed to rebuild a file the + // game can load again. + originalFileType: OriginalFileType + originalFileName: string } // Tries to read the response body as string, returning an empty string if that fails. @@ -23,16 +31,34 @@ async function getBodyTextOrEmpty(res: Response): Promise { export class SaveParserBackendService { private API_BASE = PROXY_PATHS.saveParserBackend - //TODO: save size limit 30 mb - // Uploads a save file for the given game and returns its assigned id. - async uploadSave(game: string, file: File): Promise { - const url = `${this.API_BASE}/uploadSave?game=${encodeURIComponent(game)}` + /** + * Re-compresses a save file and uploads it, returning its assigned id. + * + * Saves are not uploaded as the game wrote them. Plain text and archived + * saves are both packed into a tar inside a zstd frame, which is a lot + * smaller than the game's own zip and cheap for the backend to unpack. + * The shape the file originally had travels along, so it can be rebuilt + * when someone downloads the save later. + */ + async uploadSave( + game: GameEnum, + file: File, + config: SaveUploadConfig, + onStage: StageCallback + ): Promise { + const save = await packSaveForUpload(game, file, config, onStage) + onStage('uploading') + + const url = + `${this.API_BASE}/uploadSave?game=${encodeURIComponent(game)}` + + `&originalFileType=${encodeURIComponent(save.originalFileType)}` + + `&originalFileName=${encodeURIComponent(save.originalFileName)}` const response = await fetch(url, { method: 'POST', headers: { - 'Content-Type': 'application/octet-stream', + 'Content-Type': 'application/zstd', }, - body: file, + body: save.data, }) if (!response.ok) { @@ -61,8 +87,8 @@ export class SaveParserBackendService { } // Fetches the fully parsed data for a save once its parsing is done. - async getParsedSave(id: string): Promise> { - const url = `${this.API_BASE}/getParsedSave?id=${encodeURIComponent(id)}` + async getSaveData(id: string): Promise> { + const url = `${this.API_BASE}/getSaveData?id=${encodeURIComponent(id)}` const response = await fetch(url) if (!response.ok) { @@ -74,6 +100,24 @@ export class SaveParserBackendService { return await response.json() } + + /** + * Fetches the stored save archive, exactly as it was uploaded. Turning it + * back into a file the game can load is done in the browser. + */ + async getSaveFile(id: string): Promise { + const url = `${this.API_BASE}/getSaveFile?id=${encodeURIComponent(id)}` + const response = await fetch(url) + + if (!response.ok) { + throw new Error( + `Failed to get save file: ${response.statusText} ` + + (await getBodyTextOrEmpty(response)) + ) + } + + return await response.blob() + } } export const saveParserBackendService = new SaveParserBackendService() diff --git a/tsconfig.json b/tsconfig.json index 83e3baa..2ace016 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -29,5 +29,5 @@ ".next/types/**/*.ts", ".next/dev/types/**/*.ts" ], - "exclude": ["node_modules"] + "exclude": ["node_modules", "TEMP"] }