implemented save file compression in browser

2026-09-23 17:26:40 +02:00
parent 54ddc11b44
commit a022dd00dc
25 changed files with 1331 additions and 142 deletions
@@ -1,75 +1,29 @@
-'use client'
-
-import { AgGridReact } from 'ag-grid-react'
-import {
-    AllCommunityModule,
-    themeQuartz,
-    colorSchemeDark,
-} from 'ag-grid-community'
-import { useSyncExternalStore } from 'react'
-
-const DARK_SCHEME_QUERY = '(prefers-color-scheme: dark)'
-
-// add the vertical lines between cells
-const BASE_THEME = themeQuartz.withParams({
-    columnBorder: true,
-    headerColumnBorder: true,
-    headerCellBackgroundTransitionDuration: 0,
-})
-// Built once, so the grid keeps seeing the same theme object between renders.
-const LIGHT_THEME = BASE_THEME
-const DARK_THEME = BASE_THEME.withPart(colorSchemeDark)
-
-// Notifies React whenever the browser's light/dark color scheme changes.
-function subscribeToColorScheme(onChange: () => void) {
-    const mediaQuery = window.matchMedia(DARK_SCHEME_QUERY)
-    mediaQuery.addEventListener('change', onChange)
-    return () => mediaQuery.removeEventListener('change', onChange)
-}
-
-function isDarkSchemePreferred() {
-    return window.matchMedia(DARK_SCHEME_QUERY).matches
-}
-
-// The server has no color scheme, so it always prerenders the light theme.
-function isDarkSchemePreferredOnServer() {
-    return false
-}
+import BrowseSaveDataGrid from '@/components/BrowseSaveDataGrid'
+import ConditionalView from '@/components/ConditionalView'
+import SaveDownloadButton from '@/components/SaveDownloadButton'

 // TODO: create tabs with general info, tables, plots
+// TODO: feed the grid with the parsed save the id points at
+
+interface BrowseSaveDataPageProps {
+    searchParams: Promise<{ id?: string }>
+}

 // Page that shows a save file's parsed data in a grid.
-export default function BrowseSaveDataPage() {
-    // Read the color scheme straight from the browser instead of mirroring it
-    // into state, which would need an extra render to catch up.
-    const prefersDark = useSyncExternalStore(
-        subscribeToColorScheme,
-        isDarkSchemePreferred,
-        isDarkSchemePreferredOnServer
-    )
-    const agTheme = prefersDark ? DARK_THEME : LIGHT_THEME
+export default async function BrowseSaveDataPage(
+    props: BrowseSaveDataPageProps
+) {
+    const { id } = await props.searchParams

    return (
-        <div style={{ height: '500px' }}>
-            <AgGridReact
-                theme={agTheme}
-                modules={[AllCommunityModule]}
-                // ddisable all animations
-                animateRows={false}
-                suppressColumnMoveAnimation={true}
-                rowData={[
-                    { id: 1, col1: 'Hello', col2: 'AG Grid' },
-                    { id: 2, col1: 'Dark Mode', col2: 'Support' },
-                ]}
-                columnDefs={[
-                    {
-                        field: 'col1',
-                        headerName: 'Column 1',
-                        filter: 'agTextColumnFilter',
-                    },
-                    { field: 'col2', headerName: 'Column 2' },
-                ]}
-            />
+        <div>
+            <ConditionalView cond={id}>
+                <div className="container mb-3">
+                    <SaveDownloadButton saveId={id ?? ''} />
+                </div>
+            </ConditionalView>
+
+            <BrowseSaveDataGrid />
        </div>
    )
 }
@@ -1,11 +1,14 @@
 import SaveFileUploadingDialog from '@/components/SaveFileUploadingDialog'
+import { appConfig } from '@/lib/configLoader'

 export default function HomePage() {
    return (
        <div className="container mt-5 text-center">
            <h1 className="mb-4">Welcome to Paradox Save Parser!</h1>

-            <SaveFileUploadingDialog />
+            {/* The config is read on the server, so the limits are handed to
+                the dialog instead of being fetched by it. */}
+            <SaveFileUploadingDialog saveUploadConfig={appConfig.saveUpload} />
        </div>
    )
 }
@@ -0,0 +1,72 @@
+'use client'
+
+import { AgGridReact } from 'ag-grid-react'
+import {
+    AllCommunityModule,
+    themeQuartz,
+    colorSchemeDark,
+} from 'ag-grid-community'
+import { useSyncExternalStore } from 'react'
+
+const DARK_SCHEME_QUERY = '(prefers-color-scheme: dark)'
+
+// add the vertical lines between cells
+// Built once, so the grid keeps seeing the same theme object between renders.
+const LIGHT_THEME = themeQuartz.withParams({
+    columnBorder: true,
+    headerColumnBorder: true,
+    headerCellBackgroundTransitionDuration: 0,
+})
+const DARK_THEME = LIGHT_THEME.withPart(colorSchemeDark)
+
+// Notifies React whenever the browser's light/dark color scheme changes.
+function subscribeToColorScheme(onChange: () => void) {
+    const mediaQuery = window.matchMedia(DARK_SCHEME_QUERY)
+    mediaQuery.addEventListener('change', onChange)
+    return () => mediaQuery.removeEventListener('change', onChange)
+}
+
+function isDarkSchemePreferred() {
+    return window.matchMedia(DARK_SCHEME_QUERY).matches
+}
+
+// The server has no color scheme, so it always prerenders the light theme.
+function isDarkSchemePreferredOnServer() {
+    return false
+}
+
+// Grid showing a save file's parsed data.
+export default function BrowseSaveDataGrid() {
+    // Read the color scheme straight from the browser instead of mirroring it
+    // into state, which would need an extra render to catch up.
+    const prefersDark = useSyncExternalStore(
+        subscribeToColorScheme,
+        isDarkSchemePreferred,
+        isDarkSchemePreferredOnServer
+    )
+    const agTheme = prefersDark ? DARK_THEME : LIGHT_THEME
+
+    return (
+        <div style={{ height: '500px' }}>
+            <AgGridReact
+                theme={agTheme}
+                modules={[AllCommunityModule]}
+                // ddisable all animations
+                animateRows={false}
+                suppressColumnMoveAnimation={true}
+                rowData={[
+                    { id: 1, col1: 'Hello', col2: 'AG Grid' },
+                    { id: 2, col1: 'Dark Mode', col2: 'Support' },
+                ]}
+                columnDefs={[
+                    {
+                        field: 'col1',
+                        headerName: 'Column 1',
+                        filter: 'agTextColumnFilter',
+                    },
+                    { field: 'col2', headerName: 'Column 2' },
+                ]}
+            />
+        </div>
+    )
+}
@@ -0,0 +1,13 @@
+interface ConditionalViewProps {
+    cond: unknown
+    children: React.ReactNode
+}
+
+// Shows its children only when `cond` is truthy, and nothing otherwise.
+// Unlike `cond && <View/>`, a falsy value here never leaks into the page: JSX
+// renders 0 and '' as text, so an empty string or a count of zero would show up.
+export default function ConditionalView(props: ConditionalViewProps) {
+    if (!props.cond) return null
+
+    return <>{props.children}</>
+}
@@ -0,0 +1,73 @@
+'use client'
+
+import { useState } from 'react'
+import ConditionalView from './ConditionalView'
+import FormError from './FormError'
+import { downloadBlobAsFile } from '@/lib/downloadFile'
+import { getMsgFromError } from '@/lib/errors'
+import { rebuildOriginalSave } from '@/lib/saveArchive/client'
+import { SAVE_PROGRESS_STAGE_LABELS } from '@/lib/saveArchive/messages'
+import {
+    saveParserBackendService,
+    SaveStatusResponse,
+} from '@/services/SaveParserBackendService'
+
+interface SaveDownloadButtonProps {
+    saveId: string
+    // Status of the save, when the caller already has it. It is fetched on
+    // demand otherwise, since the download needs the shape the save was
+    // uploaded in and its file name.
+    status?: SaveStatusResponse | null
+}
+
+// Downloads a stored save and rebuilds the file the game originally wrote.
+export default function SaveDownloadButton(props: SaveDownloadButtonProps) {
+    const [error, setError] = useState<string | null>(null)
+    const [progress, setProgress] = useState<string | null>(null)
+
+    const handleDownload = async () => {
+        setError(null)
+        try {
+            const status =
+                props.status ??
+                (await saveParserBackendService.getSaveStatus(props.saveId))
+
+            setProgress(SAVE_PROGRESS_STAGE_LABELS.downloading)
+            const archive = await saveParserBackendService.getSaveFile(
+                props.saveId
+            )
+
+            const saveFile = await rebuildOriginalSave(
+                archive,
+                status.originalFileType,
+                (stage) => setProgress(SAVE_PROGRESS_STAGE_LABELS[stage])
+            )
+            downloadBlobAsFile(saveFile, status.originalFileName)
+        } catch (err) {
+            const errMsg = getMsgFromError(err)
+            console.error('Error while downloading save:', errMsg)
+            setError(errMsg)
+        } finally {
+            setProgress(null)
+        }
+    }
+
+    return (
+        <div>
+            <FormError message={error} />
+
+            <button
+                className="btn btn-outline-primary"
+                type="button"
+                onClick={handleDownload}
+                disabled={progress !== null}
+            >
+                Download original save
+            </button>
+
+            <ConditionalView cond={progress}>
+                <div className="mt-2">{progress}</div>
+            </ConditionalView>
+        </div>
+    )
+}
@@ -2,30 +2,34 @@

 import {
    saveParserBackendService,
-    SaveStatusResponse,
    UploadSaveResponse,
 } from '@/services/SaveParserBackendService'
+import Link from 'next/link'
 import { useState } from 'react'
-import FormError from './FormError'
+import ConditionalView from './ConditionalView'
+import type { SaveUploadConfig } from '@/config/AppConfig'
+import { GameEnum } from '@/config/games'
 import { getMsgFromError } from '@/lib/errors'
+import { SAVE_PROGRESS_STAGE_LABELS } from '@/lib/saveArchive/messages'

-export enum GameEnum {
-    EU4 = 'eu4',
-    Stellaris = 'stellaris',
+interface SaveFileUploadingDialogProps {
+    // Comes from the server, where runtimeConfig.json is read.
+    saveUploadConfig: SaveUploadConfig
 }

-//TODO: move to separate page
-//TODO: add redirect to ?id={id} to see status of long parsing save
+const IN_BROWSER = '[In Browser]'
+const ON_SERVER = '[On Server]'

 // Dialog for picking a game and save file, uploading it, and polling the parse status until done.
-export default function SaveFileUploadingDialog() {
-    const [error, setError] = useState<string | null>(null)
+export default function SaveFileUploadingDialog(
+    props: SaveFileUploadingDialogProps
+) {
    const [selectedFile, setSelectedFile] = useState<File | null>(null)
    const [game, setGame] = useState<GameEnum>(GameEnum.EU4)
    const [uploadDisabled, setUploadDisabled] = useState(false)
-    const [saveStatus, setSaveStatus] = useState<SaveStatusResponse | null>(
-        null
-    )
+    // This is what user will see
+    const [statusMessage, setStatusMessage] = useState<string | null>(null)
+
    const [uploadResult, setUploadResult] = useState<UploadSaveResponse | null>(
        null
    )
@@ -42,10 +46,10 @@ export default function SaveFileUploadingDialog() {
        }
    }

-    // Uploads the selected save file, then repeatedly polls its status until parsing is done or an error occurs.
+    // Compresses and uploads the selected save file, then repeatedly polls its status until parsing is done or an error occurs.
    const handleUpload = async (e: React.FormEvent) => {
        e.preventDefault()
-        setError(null)
+        setStatusMessage(null)
        setUploadDisabled(true)
        try {
            if (!selectedFile) return
@@ -53,10 +57,17 @@ export default function SaveFileUploadingDialog() {
            console.log('Uploading file:', selectedFile.name)
            const uploadResponse = await saveParserBackendService.uploadSave(
                game,
-                selectedFile
+                selectedFile,
+                props.saveUploadConfig,
+                (stage) =>
+                    setStatusMessage(
+                        `${IN_BROWSER} ${SAVE_PROGRESS_STAGE_LABELS[stage]}`
+                    )
            )
            console.log('Upload response:', uploadResponse)
-            setUploadResult(uploadResponse)
+            setStatusMessage(
+                `${ON_SERVER} Waiting for the save to be parsed...`
+            )

            const pollInterval = 1000 // ms
            // Poll the backend every second for the parsing status until it reports "done".
@@ -67,23 +78,25 @@ export default function SaveFileUploadingDialog() {
                            uploadResponse.id
                        )
                    console.log('Save status:', statusResponse.status)
-                    setSaveStatus(statusResponse)
+                    setStatusMessage(`${ON_SERVER} ${statusResponse.status}`)
                    if (statusResponse.status.toLowerCase() === 'done') {
                        clearInterval(intervalId)
+                        // Only a parsed save is worth showing or browsing.
+                        setUploadResult(uploadResponse)
                        setUploadDisabled(false)
                    }
                } catch (err) {
                    const errMsg = getMsgFromError(err)
                    console.error('Error while polling status:', errMsg)
                    clearInterval(intervalId)
-                    setError(errMsg)
+                    setStatusMessage(`${ON_SERVER} ${errMsg}`)
                    setUploadDisabled(false)
                }
            }, pollInterval)
        } catch (err) {
            const errMsg = getMsgFromError(err)
            console.error(errMsg)
-            setError(errMsg)
+            setStatusMessage(`${IN_BROWSER} ${errMsg}`)
            setUploadDisabled(false)
        }
    }
@@ -97,8 +110,6 @@ export default function SaveFileUploadingDialog() {

    return (
        <div className="container mt-5" style={{ maxWidth: '500px' }}>
-            <FormError message={error} />
-
            <form onSubmit={handleUpload} className="text-start">
                <label htmlFor="gameSelect" className="form-label">
                    Select Game
@@ -134,21 +145,21 @@ export default function SaveFileUploadingDialog() {
            </form>

            <div className="container mt-5 text-center">
-                {/* TODO: extract this to a component 
-                <ConditionalView cond={uploadResult}> <LabeledValue val={}/> </ConditionalView> */}
-                {uploadResult && (
-                    <div className="mb-3">
-                        Save Id: <strong>{uploadResult.id}</strong>
-                    </div>
-                )}
+                <ConditionalView cond={statusMessage}>
+                    <div className="mb-3">{statusMessage}</div>
+                </ConditionalView>

-                {saveStatus && (
+                <ConditionalView cond={uploadResult}>
                    <div className="mb-3">
-                        Status: <strong>{saveStatus.status}</strong>
+                        Save Id: <strong>{uploadResult?.id}</strong>
                    </div>
-                )}
-
-                {/* TODO: add link to <ConditionalView cond={done}> /browse?id=${saveStatus.id} */}
+                    <Link
+                        href={`/browse?id=${uploadResult?.id}`}
+                        className="btn btn-primary"
+                    >
+                        Browse parsed save
+                    </Link>
+                </ConditionalView>
            </div>
        </div>
    )
@@ -5,9 +5,19 @@ export const defaultConfig = {
        saveParserBackend: {
            url: 'http://127.0.0.1:5226',
        },
-        // TODO: implement auth api
-        auth: {},
+    },
+    saveUpload: {
+        // Size of a save if it is uploaded as uncompressed. The browser packs
+        // saves as a stream, so these limits are about the backend's disk
+        // rather than about memory.
+        maxSizePlainTextBytes: 200 * 1024 * 1024,
+        // Archives produced by the game, recognized by the zip magic bytes.
+        maxSizeCompressedBytes: 60 * 1024 * 1024,
+        // read COMPRESSION.md
+        zstdCompressionLevel: 11,
    },
 }

 export type AppConfig = typeof defaultConfig
+
+export type SaveUploadConfig = AppConfig['saveUpload']
@@ -0,0 +1,20 @@
+// Games the parser backend can handle. The values go into the upload request.
+//TODO: add more games
+export enum GameEnum {
+    EU4 = 'eu4',
+    Stellaris = 'stellaris',
+}
+
+/**
+ * Name the game state gets inside a save archive, per game.
+ *
+ * A save produced by the game is either a zip archive holding this file (plus
+ * companion files like `meta`), or the very same content as plain text. When a
+ * plain text save is uploaded, this is the name it is stored under, so the
+ * backend always reads the 'gamestate' or whatever is set for the specific game.
+ * The parser backend keeps the same map on its side.
+ */
+export const GAME_SAVE_ENTRY_NAMES: Record<GameEnum, string> = {
+    [GameEnum.EU4]: 'gamestate',
+    [GameEnum.Stellaris]: 'gamestate',
+}
@@ -0,0 +1,17 @@
+// Revoking the object url right after the click can cancel the download in
+// some browsers, so it is kept alive for a while first.
+const URL_LIFETIME_MS = 20_000
+
+/**
+ * Makes the browser save a blob to disk under the given name.
+ */
+export function downloadBlobAsFile(blob: Blob, fileName: string) {
+    const url = URL.createObjectURL(blob)
+
+    const link = document.createElement('a')
+    link.href = url
+    link.download = fileName
+    link.click()
+
+    setTimeout(() => URL.revokeObjectURL(url), URL_LIFETIME_MS)
+}
@@ -0,0 +1,11 @@
+const KILOBYTE = 1024
+const MEGABYTE = 1024 * KILOBYTE
+
+/**
+ * Formats a byte count for error messages and status lines, e.g. "12.3 MB".
+ */
+export function formatBytes(bytes: number): string {
+    if (bytes >= MEGABYTE) return `${(bytes / MEGABYTE).toFixed(1)} MB`
+    if (bytes >= KILOBYTE) return `${(bytes / KILOBYTE).toFixed(1)} KB`
+    return `${bytes} B`
+}
@@ -0,0 +1,138 @@
+// Converting between the save files a game reads and the archive the parser
+// backend stores. The tar always holds the files the game itself would have in
+// its archive, so the backend reads the game state from the same entry either
+// way, and a download rebuilds whichever shape was uploaded:
+//
+//     plain text save  ->  tar { "gamestate" }                  -> zstd
+//     archived save    ->  tar { "meta", "gamestate", ... }     -> zstd
+//
+// Everything runs as a stream: a save is never held in memory as a whole.
+
+import { createTarDecoder, createTarPacker } from 'modern-tar'
+import { createGameZipBuilder, readGameZip } from './gameZip'
+import { OriginalFileType, SaveProgressStage } from './messages'
+import {
+    createZstdCompressStream,
+    createZstdDecompressStream,
+} from './zstdStream'
+
+type StageCallback = (stage: SaveProgressStage) => void
+
+// A fixed timestamp keeps uploads of the same save byte for byte identical.
+const TAR_ENTRY_MTIME = new Date(0)
+
+export interface PackSaveOptions {
+    /** Whether the file is an archive produced by the game. */
+    isZip: boolean
+    /** Name of the single entry used for a plain text save. */
+    entryName: string
+    compressionLevel: number
+}
+
+// Collects a stream into a blob, whose storage the browser manages. The blob
+// takes its type from the response's Content-Type.
+function collectBlob(
+    stream: ReadableStream<Uint8Array>,
+    type: string
+): Promise<Blob> {
+    return new Response(stream, { headers: { 'Content-Type': type } }).blob()
+}
+
+// Writes the files of the save into the tar, in the order the game stored
+// them. Runs alongside the compression, which consumes the tar as it is built.
+async function packEntries(
+    file: Blob,
+    options: PackSaveOptions,
+    controller: ReturnType<typeof createTarPacker>['controller']
+) {
+    try {
+        if (options.isZip) {
+            await readGameZip(file, (entry) =>
+                // Returns writable stream.
+                // Zip decoder decodes file entry stream and pipes it to the tar entry stream.
+                controller.add({
+                    name: entry.name,
+                    size: entry.size,
+                    mtime: TAR_ENTRY_MTIME,
+                })
+            )
+        } else {
+            // Writes text file to tar entry
+            await file.stream().pipeTo(
+                controller.add({
+                    name: options.entryName,
+                    size: file.size,
+                    mtime: TAR_ENTRY_MTIME,
+                })
+            )
+        }
+    } finally {
+        controller.finalize()
+    }
+}
+
+/**
+ * Compresses a save file the user picked into the archive that is uploaded.
+ */
+export async function packSave(
+    file: Blob,
+    options: PackSaveOptions,
+    onStage: StageCallback
+): Promise<Blob> {
+    onStage('compressing')
+
+    const { readable, controller } = createTarPacker()
+    // The tar is written while it is being compressed, so neither side of the
+    // pipe ever holds the whole save.
+    const packed = packEntries(file, options, controller)
+    const compressed = collectBlob(
+        readable.pipeThrough(
+            createZstdCompressStream(options.compressionLevel)
+        ),
+        'application/zstd'
+    )
+
+    const [, archive] = await Promise.all([packed, compressed])
+    return archive
+}
+
+/**
+ * Turns a stored archive back into a file the game can load.
+ */
+export async function rebuildSave(
+    archive: Blob,
+    originalFileType: OriginalFileType,
+    onStage: StageCallback
+): Promise<Blob> {
+    onStage('decompressing')
+
+    const entries = archive
+        .stream()
+        .pipeThrough(createZstdDecompressStream())
+        .pipeThrough(createTarDecoder())
+        .getReader()
+
+    const zipBuilder =
+        originalFileType === 'zip' ? createGameZipBuilder() : null
+    let saveFile: Blob | null = null
+
+    for (;;) {
+        const { done, value: entry } = await entries.read()
+        if (done) break
+
+        if (zipBuilder) {
+            onStage('rebuilding')
+            await zipBuilder.add(entry.header.name, entry.body)
+        } else if (saveFile) {
+            // A plain text save is a single file, so anything after it would
+            // be silently dropped.
+            await entry.body.cancel()
+        } else {
+            saveFile = await collectBlob(entry.body, 'application/octet-stream')
+        }
+    }
+
+    if (zipBuilder) return await zipBuilder.finish()
+    if (!saveFile) throw new Error('Stored save archive is empty')
+    return saveFile
+}
@@ -0,0 +1,113 @@
+// Main thread side of the save archive worker: checks the file, starts the
+// worker and reports what it is doing.
+
+import type { SaveUploadConfig } from '@/config/AppConfig'
+import { GAME_SAVE_ENTRY_NAMES, GameEnum } from '@/config/games'
+import { formatBytes } from '@/lib/formatBytes'
+import { isZipFile } from './gameZip'
+import {
+    OriginalFileType,
+    SaveArchiveRequest,
+    SaveArchiveResponse,
+    SaveProgressStage,
+} from './messages'
+
+export type StageCallback = (stage: SaveProgressStage) => void
+
+export interface PackedSave {
+    data: Blob
+    originalFileType: OriginalFileType
+    originalFileName: string
+}
+
+// Runs a single request in a fresh worker, which is terminated once it answers.
+function runInWorker(
+    request: SaveArchiveRequest,
+    onStage: StageCallback
+): Promise<Blob> {
+    return new Promise((resolve, reject) => {
+        const worker = new Worker(new URL('./worker.ts', import.meta.url), {
+            type: 'module',
+        })
+
+        worker.onmessage = (event: MessageEvent<SaveArchiveResponse>) => {
+            const message = event.data
+            if (message.kind === 'progress') {
+                onStage(message.stage)
+                return
+            }
+
+            worker.terminate()
+            if (message.kind === 'error') {
+                reject(new Error(message.message))
+            } else {
+                resolve(message.data)
+            }
+        }
+
+        worker.onerror = (event) => {
+            worker.terminate()
+            reject(new Error(event.message || 'Save file compression failed'))
+        }
+
+        worker.postMessage(request)
+    })
+}
+
+/**
+ * Compresses a save file the user picked into the archive that is uploaded.
+ *
+ * Plain text and archived saves are both accepted, are told apart by the zip
+ * magic bytes rather than by the file extension, and are checked against their
+ * own size limit.
+ */
+export async function packSaveForUpload(
+    game: GameEnum,
+    file: File,
+    config: SaveUploadConfig,
+    onStage: StageCallback
+): Promise<PackedSave> {
+    const isZip = await isZipFile(file)
+    const sizeLimit = isZip
+        ? config.maxSizeCompressedBytes
+        : config.maxSizePlainTextBytes
+
+    if (file.size > sizeLimit) {
+        const saveKind = isZip ? 'archived' : 'plain text'
+        throw new Error(
+            `Save file is ${formatBytes(file.size)}, but the limit for ` +
+                `${saveKind} saves is ${formatBytes(sizeLimit)}`
+        )
+    }
+
+    const data = await runInWorker(
+        {
+            kind: 'pack',
+            file,
+            isZip,
+            entryName: GAME_SAVE_ENTRY_NAMES[game],
+            compressionLevel: config.zstdCompressionLevel,
+        },
+        onStage
+    )
+
+    return {
+        data,
+        originalFileType: isZip ? 'zip' : 'text',
+        originalFileName: file.name,
+    }
+}
+
+/**
+ * Rebuilds the save file as the game wrote it, out of a stored archive.
+ */
+export async function rebuildOriginalSave(
+    archive: Blob,
+    originalFileType: OriginalFileType,
+    onStage: StageCallback
+): Promise<Blob> {
+    return await runInWorker(
+        { kind: 'rebuild', data: archive, originalFileType },
+        onStage
+    )
+}
@@ -0,0 +1,313 @@
+// Reading and writing the zip archives the Paradox games themselves use.
+//
+// This is written here rather than taken from a library because the games are
+// picky about the headers and no browser zip library can produce them: they
+// all derive the DOS date from a Date and clamp it to 1980, while the games
+// write zeros, and the streaming writers emit a data descriptor plus a Zip64
+// extra field for every entry, which the games reject. The heavy part of zip,
+// deflate, is in the browser itself as CompressionStream('deflate-raw'), so
+// what is left is the header layout below.
+//
+// The field values follow a known good reference, which produces archives the
+// games accept: http://pastebin.com/nyX1B4rH
+// A real EU4 save agrees with it: MS-DOS as the system of origin, encoding
+// software 6.3, extract version 2.0, no data descriptor, no extra field and a
+// zeroed DOS date. 7-Zip's NTFS timestamps are a known way to produce a save
+// that will not load: https://stellaris.paradoxwikis.com/Save-game_editing
+
+const LOCAL_HEADER_SIGNATURE = 0x04034b50
+const CENTRAL_HEADER_SIGNATURE = 0x02014b50
+const END_OF_CENTRAL_DIR_SIGNATURE = 0x06054b50
+
+const LOCAL_HEADER_SIZE = 30
+const CENTRAL_HEADER_SIZE = 46
+const END_OF_CENTRAL_DIR_SIZE = 22
+
+const METHOD_STORED = 0
+const METHOD_DEFLATE = 8
+
+// "version needed to extract" 2.0, which is what deflate requires.
+const EXTRACT_VERSION = 20
+// "version made by": MS-DOS (0) in the high byte, 6.3 (63) in the low byte.
+const CREATE_VERSION = (0 << 8) | 63
+
+// Anything at or above this needs the zip64 format, which no save comes near.
+const MAX_ENTRY_SIZE = 0xffffffff
+
+// First bytes of every zip archive.
+const ZIP_MAGIC = [0x50, 0x4b, 0x03, 0x04]
+
+export interface GameZipEntry {
+    name: string
+    size: number
+}
+
+// The standard CRC-32 lookup table.
+const CRC_TABLE = new Uint32Array(256).map((_, i) => {
+    let value = i
+    for (let bit = 0; bit < 8; bit++) {
+        value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1
+    }
+    return value >>> 0
+})
+
+// Continues a CRC-32 over one more chunk. Start with 0; the result is the
+// checksum of everything so far, so an entry is summed as it streams past.
+function updateCrc(crc: number, data: Uint8Array): number {
+    let value = ~crc
+    for (const byte of data) {
+        value = CRC_TABLE[(value ^ byte) & 0xff] ^ (value >>> 8)
+    }
+    return ~value >>> 0
+}
+
+/**
+ * Tells whether the file is a zip archive, by its magic bytes rather than by
+ * its extension: the games use the same extension for both an archived and a
+ * plain text save.
+ */
+export async function isZipFile(file: Blob): Promise<boolean> {
+    const header = new Uint8Array(
+        await file.slice(0, ZIP_MAGIC.length).arrayBuffer()
+    )
+    return ZIP_MAGIC.every((byte, i) => header[i] === byte)
+}
+
+// Little endian reader over a slice of the archive.
+function reader(data: Uint8Array) {
+    const view = new DataView(data.buffer, data.byteOffset, data.byteLength)
+    return {
+        u16: (offset: number) => view.getUint16(offset, true),
+        u32: (offset: number) => view.getUint32(offset, true),
+    }
+}
+
+interface CentralEntry {
+    name: string
+    method: number
+    compressedSize: number
+    size: number
+    localHeaderOffset: number
+}
+
+// Reads the central directory, which lists every entry with its size and
+// position. Only the last few hundred bytes of the archive are touched.
+async function readCentralDirectory(zip: Blob): Promise<CentralEntry[]> {
+    const end = new Uint8Array(
+        await zip.slice(zip.size - END_OF_CENTRAL_DIR_SIZE).arrayBuffer()
+    )
+    const endFields = reader(end)
+    if (endFields.u32(0) !== END_OF_CENTRAL_DIR_SIGNATURE) {
+        throw new Error('Save archive has no end of central directory')
+    }
+
+    const entryCount = endFields.u16(10)
+    const directoryOffset = endFields.u32(16)
+    const directory = new Uint8Array(
+        await zip.slice(directoryOffset, zip.size).arrayBuffer()
+    )
+    const fields = reader(directory)
+
+    const entries: CentralEntry[] = []
+    let offset = 0
+    for (let i = 0; i < entryCount; i++) {
+        if (fields.u32(offset) !== CENTRAL_HEADER_SIGNATURE) {
+            throw new Error('Save archive has a corrupted entry header')
+        }
+
+        const nameLength = fields.u16(offset + 28)
+        const extraLength = fields.u16(offset + 30)
+        const commentLength = fields.u16(offset + 32)
+        const nameStart = offset + CENTRAL_HEADER_SIZE
+        entries.push({
+            name: new TextDecoder().decode(
+                directory.subarray(nameStart, nameStart + nameLength)
+            ),
+            method: fields.u16(offset + 10),
+            compressedSize: fields.u32(offset + 20),
+            size: fields.u32(offset + 24),
+            localHeaderOffset: fields.u32(offset + 42),
+        })
+        offset += CENTRAL_HEADER_SIZE + nameLength + extraLength + commentLength
+    }
+    return entries
+}
+
+/**
+ * Reads every file of a save archive in the order they are stored, asking the
+ * caller where each one should go and writing it straight into that stream.
+ *
+ * Sizes come from the central directory, so they are known before an entry
+ * starts streaming, and the entry data itself is never held in memory.
+ */
+export async function readGameZip(
+    zip: Blob,
+    openEntry: (entry: GameZipEntry) => WritableStream<Uint8Array>
+): Promise<void> {
+    for (const entry of await readCentralDirectory(zip)) {
+        // The local header repeats the name and may carry a different extra
+        // field, so the data can only be located through it.
+        const header = new Uint8Array(
+            await zip
+                .slice(
+                    entry.localHeaderOffset,
+                    entry.localHeaderOffset + LOCAL_HEADER_SIZE
+                )
+                .arrayBuffer()
+        )
+        const fields = reader(header)
+        if (fields.u32(0) !== LOCAL_HEADER_SIGNATURE) {
+            throw new Error(`Save archive entry "${entry.name}" is corrupted`)
+        }
+
+        const dataStart =
+            entry.localHeaderOffset +
+            LOCAL_HEADER_SIZE +
+            fields.u16(26) +
+            fields.u16(28)
+        const data = zip.slice(dataStart, dataStart + entry.compressedSize)
+
+        if (entry.method === METHOD_STORED) {
+            await data.stream().pipeTo(openEntry(entry))
+        } else if (entry.method === METHOD_DEFLATE) {
+            await data
+                .stream()
+                .pipeThrough(new DecompressionStream('deflate-raw'))
+                .pipeTo(openEntry(entry))
+        } else {
+            throw new Error(
+                `Save archive entry "${entry.name}" uses compression ` +
+                    `method ${entry.method}, which the games do not produce`
+            )
+        }
+    }
+}
+
+// Writes a local file header. The date and time fields stay zero, which is
+// what the games write and what a Date cannot express.
+function localHeader(entry: WrittenEntry): Uint8Array<ArrayBuffer> {
+    const header = new Uint8Array(LOCAL_HEADER_SIZE + entry.nameBytes.length)
+    const view = new DataView(header.buffer)
+    view.setUint32(0, LOCAL_HEADER_SIGNATURE, true)
+    view.setUint16(4, EXTRACT_VERSION, true)
+    view.setUint16(8, METHOD_DEFLATE, true)
+    view.setUint32(14, entry.crc, true)
+    view.setUint32(18, entry.compressedSize, true)
+    view.setUint32(22, entry.size, true)
+    view.setUint16(26, entry.nameBytes.length, true)
+    header.set(entry.nameBytes, LOCAL_HEADER_SIZE)
+    return header
+}
+
+// Writes the central directory record for one entry.
+function centralHeader(entry: WrittenEntry): Uint8Array<ArrayBuffer> {
+    const header = new Uint8Array(CENTRAL_HEADER_SIZE + entry.nameBytes.length)
+    const view = new DataView(header.buffer)
+    view.setUint32(0, CENTRAL_HEADER_SIGNATURE, true)
+    view.setUint16(4, CREATE_VERSION, true)
+    view.setUint16(6, EXTRACT_VERSION, true)
+    view.setUint16(10, METHOD_DEFLATE, true)
+    view.setUint32(16, entry.crc, true)
+    view.setUint32(20, entry.compressedSize, true)
+    view.setUint32(24, entry.size, true)
+    view.setUint16(28, entry.nameBytes.length, true)
+    view.setUint32(42, entry.localHeaderOffset, true)
+    header.set(entry.nameBytes, CENTRAL_HEADER_SIZE)
+    return header
+}
+
+// Writes the end of central directory record that closes the archive.
+function endOfCentralDirectory(
+    entries: WrittenEntry[],
+    directoryOffset: number,
+    directorySize: number
+): Uint8Array<ArrayBuffer> {
+    const record = new Uint8Array(END_OF_CENTRAL_DIR_SIZE)
+    const view = new DataView(record.buffer)
+    view.setUint32(0, END_OF_CENTRAL_DIR_SIGNATURE, true)
+    view.setUint16(8, entries.length, true)
+    view.setUint16(10, entries.length, true)
+    view.setUint32(12, directorySize, true)
+    view.setUint32(16, directoryOffset, true)
+    return record
+}
+
+interface WrittenEntry {
+    nameBytes: Uint8Array
+    crc: number
+    size: number
+    compressedSize: number
+    localHeaderOffset: number
+}
+
+/**
+ * Builds a save archive the games accept, one streamed file at a time.
+ */
+export function createGameZipBuilder() {
+    const entries: WrittenEntry[] = []
+    // Headers are plain bytes and entry data is a blob, so assembling the
+    // archive at the end copies nothing through memory.
+    const parts: BlobPart[] = []
+    let offset = 0
+
+    return {
+        async add(name: string, body: ReadableStream<Uint8Array>) {
+            const nameBytes = new TextEncoder().encode(name)
+            let crc = 0
+            let size = 0
+
+            // Sums the entry as it goes past. The cast only drops the shared
+            // buffer case from the type; a save never arrives in one.
+            const summed = body.pipeThrough(
+                new TransformStream<Uint8Array, Uint8Array<ArrayBuffer>>({
+                    transform(chunk, controller) {
+                        crc = updateCrc(crc, chunk)
+                        size += chunk.length
+                        controller.enqueue(chunk as Uint8Array<ArrayBuffer>)
+                    },
+                })
+            )
+
+            // The sizes and the checksum belong in the local header, which is
+            // written before the data, so the entry is deflated first. Only
+            // the compressed bytes are kept, and in a blob rather than in the
+            // javascript heap.
+            const compressed = await new Response(
+                summed.pipeThrough(new CompressionStream('deflate-raw'))
+            ).blob()
+
+            if (size > MAX_ENTRY_SIZE) {
+                throw new Error(`Save file "${name}" is too large for a zip`)
+            }
+
+            const entry: WrittenEntry = {
+                nameBytes,
+                crc,
+                size,
+                compressedSize: compressed.size,
+                localHeaderOffset: offset,
+            }
+            entries.push(entry)
+
+            const header = localHeader(entry)
+            parts.push(header, compressed)
+            offset += header.length + compressed.size
+        },
+
+        async finish(): Promise<Blob> {
+            const directory = entries.map(centralHeader)
+            const directorySize = directory.reduce(
+                (total, header) => total + header.length,
+                0
+            )
+            return new Blob(
+                [
+                    ...parts,
+                    ...directory,
+                    endOfCentralDirectory(entries, offset, directorySize),
+                ],
+                { type: 'application/zip' }
+            )
+        },
+    }
+}
@@ -0,0 +1,44 @@
+// Types shared between the save archive worker and the code that drives it.
+// This module deliberately imports nothing, so importing it never pulls the
+// compression code into a bundle.
+
+/** Shape the user originally uploaded, and the shape a download rebuilds. */
+export type OriginalFileType = 'text' | 'zip'
+
+/**
+ * Steps worth showing to the user, since each of them takes seconds on a big
+ * save. The worker reports the ones it runs itself; uploading and downloading
+ * are reported by the service around it.
+ */
+export type SaveProgressStage =
+    'compressing' | 'uploading' | 'downloading' | 'decompressing' | 'rebuilding'
+
+export type SaveArchiveRequest =
+    | {
+          kind: 'pack'
+          file: File
+          isZip: boolean
+          entryName: string
+          compressionLevel: number
+      }
+    | {
+          kind: 'rebuild'
+          data: Blob
+          originalFileType: OriginalFileType
+      }
+
+export type SaveArchiveResponse =
+    // Blobs are handles to storage the browser manages, so passing one between
+    // threads copies nothing.
+    | { kind: 'progress'; stage: SaveProgressStage }
+    | { kind: 'done'; data: Blob }
+    | { kind: 'error'; message: string }
+
+/** Text shown while a stage is running. */
+export const SAVE_PROGRESS_STAGE_LABELS: Record<SaveProgressStage, string> = {
+    compressing: 'Compressing save file...',
+    uploading: 'Uploading save file...',
+    downloading: 'Downloading save file...',
+    decompressing: 'Decompressing save file...',
+    rebuilding: 'Rebuilding save file...',
+}
@@ -0,0 +1,66 @@
+// Worker that packs and rebuilds save archives.
+//
+// Compressing a save is seconds of solid cpu work, so on the main thread it
+// would freeze the page long enough for the browser to offer to kill it.
+// Keeping the compression code here also keeps it out of every other bundle,
+// including the 534 kB zstd wasm.
+
+import { getMsgFromError } from '@/lib/errors'
+import { packSave, rebuildSave } from './archive'
+import {
+    SaveArchiveRequest,
+    SaveArchiveResponse,
+    SaveProgressStage,
+} from './messages'
+
+// The part of the worker global scope this file uses. It is declared here
+// because the project compiles against the dom types, where `self` is a window
+// whose postMessage takes a target origin; the worker types live in a lib that
+// cannot be loaded alongside them. Naming the message types also checks both
+// ends of the protocol, which DedicatedWorkerGlobalScope would leave as `any`.
+interface SaveArchiveWorkerScope {
+    postMessage(message: SaveArchiveResponse, transfer?: Transferable[]): void
+    addEventListener(
+        type: 'message',
+        listener: (event: MessageEvent<SaveArchiveRequest>) => void
+    ): void
+}
+
+const workerScope = self as unknown as SaveArchiveWorkerScope
+
+function reportStage(stage: SaveProgressStage) {
+    workerScope.postMessage({ kind: 'progress', stage })
+}
+
+// Runs one request and answers with its result or with the error it failed on.
+async function handleRequest(request: SaveArchiveRequest) {
+    try {
+        const result =
+            request.kind === 'pack'
+                ? await packSave(
+                      request.file,
+                      {
+                          isZip: request.isZip,
+                          entryName: request.entryName,
+                          compressionLevel: request.compressionLevel,
+                      },
+                      reportStage
+                  )
+                : await rebuildSave(
+                      request.data,
+                      request.originalFileType,
+                      reportStage
+                  )
+
+        workerScope.postMessage({ kind: 'done', data: result })
+    } catch (err) {
+        workerScope.postMessage({
+            kind: 'error',
+            message: getMsgFromError(err),
+        })
+    }
+}
+
+workerScope.addEventListener('message', (event) => {
+    void handleRequest(event.data)
+})
@@ -0,0 +1,58 @@
+// Zstandard compression and decompression as stream transforms.
+//
+// Saves are re-compressed in the browser because the parser backend runs on a
+// cheap two core server with little disk space. zstd was picked over lzma2 and
+// brotli because its decompression speed does not depend on the level the data
+// was compressed with, so the level can be raised without costing the server
+// anything, and because its wasm build is by far the smallest of the three.
+//
+// @tootallnate/zstd-wasm does the work: it is upstream zstd 1.5.7 built with
+// wasi-sdk and driven through ZSTD_compressStream2, so it compresses a stream
+// as it arrives rather than a buffer at a time. A save of any size comes out
+// as one zstd frame, and memory stays at the library's own input and output
+// buffers.
+
+import {
+    ZstdCompressStream,
+    ZstdDecompressStream,
+} from '@tootallnate/zstd-wasm'
+
+// The wasm file ships inside the package. Turbopack rewrites this into the URL
+// it emits the asset at, so the 534 kB binary is fetched once at runtime and
+// cached by the browser instead of being inlined into the worker bundle.
+const wasmUrl = new URL(
+    '../../../node_modules/@tootallnate/zstd-wasm/dist/zstd.wasm',
+    import.meta.url
+)
+
+// Compiling takes a moment, so it happens once and both directions share it.
+let wasmModule: Promise<WebAssembly.Module> | null = null
+
+function getWasm(): Promise<WebAssembly.Module> {
+    // compileStreaming would need the server to send Content-Type:
+    // application/wasm, which is not worth depending on.
+    wasmModule ??= fetch(wasmUrl)
+        .then((res) => res.arrayBuffer())
+        .then((bytes) => WebAssembly.compile(bytes))
+    return wasmModule
+}
+
+/**
+ * Compresses a stream into a single zstd frame. The level goes from 1 to 22.
+ */
+export function createZstdCompressStream(
+    compressionLevel: number
+): TransformStream<Uint8Array, Uint8Array> {
+    return new ZstdCompressStream(getWasm(), compressionLevel)
+}
+
+/**
+ * Decompresses a zstd stream. Frames that follow one another are read straight
+ * through, so an archive packed as several frames still opens.
+ */
+export function createZstdDecompressStream(): TransformStream<
+    Uint8Array,
+    Uint8Array
+> {
+    return new ZstdDecompressStream(getWasm())
+}
@@ -2,14 +2,12 @@
 // already uses Discord, so email/password login is not worth building. The
 // /api/auth/login and /api/auth/register routes below are never implemented —
 // delete them along with this service.
-//TODO: controllable rate limits

 class AuthService {
    private API_BASE = '/api/auth'

    // Hashes the password before sending it to the backend
    private hashPassword(password: string) {
-        // TODO: password hashing
        return password
    }

@@ -1,4 +1,8 @@
 import { PROXY_PATHS } from '@/app/api/proxy/paths'
+import type { SaveUploadConfig } from '@/config/AppConfig'
+import { GameEnum } from '@/config/games'
+import { packSaveForUpload, StageCallback } from '@/lib/saveArchive/client'
+import { OriginalFileType } from '@/lib/saveArchive/messages'

 export interface UploadSaveResponse {
    id: string
@@ -9,6 +13,10 @@ export interface SaveStatusResponse {
    game: string
    status: string
    uploadDateTime: string
+    // Shape the save had when it was uploaded, needed to rebuild a file the
+    // game can load again.
+    originalFileType: OriginalFileType
+    originalFileName: string
 }

 // Tries to read  the response body as string, returning an empty string if that fails.
@@ -23,16 +31,34 @@ async function getBodyTextOrEmpty(res: Response): Promise<string> {
 export class SaveParserBackendService {
    private API_BASE = PROXY_PATHS.saveParserBackend

-    //TODO: save size limit 30 mb
-    // Uploads a save file for the given game and returns its assigned id.
-    async uploadSave(game: string, file: File): Promise<UploadSaveResponse> {
-        const url = `${this.API_BASE}/uploadSave?game=${encodeURIComponent(game)}`
+    /**
+     * Re-compresses a save file and uploads it, returning its assigned id.
+     *
+     * Saves are not uploaded as the game wrote them. Plain text and archived
+     * saves are both packed into a tar inside a zstd frame, which is a lot
+     * smaller than the game's own zip and cheap for the backend to unpack.
+     * The shape the file originally had travels along, so it can be rebuilt
+     * when someone downloads the save later.
+     */
+    async uploadSave(
+        game: GameEnum,
+        file: File,
+        config: SaveUploadConfig,
+        onStage: StageCallback
+    ): Promise<UploadSaveResponse> {
+        const save = await packSaveForUpload(game, file, config, onStage)
+        onStage('uploading')
+
+        const url =
+            `${this.API_BASE}/uploadSave?game=${encodeURIComponent(game)}` +
+            `&originalFileType=${encodeURIComponent(save.originalFileType)}` +
+            `&originalFileName=${encodeURIComponent(save.originalFileName)}`
        const response = await fetch(url, {
            method: 'POST',
            headers: {
-                'Content-Type': 'application/octet-stream',
+                'Content-Type': 'application/zstd',
            },
-            body: file,
+            body: save.data,
        })

        if (!response.ok) {
@@ -61,8 +87,8 @@ export class SaveParserBackendService {
    }

    // Fetches the fully parsed data for a save once its parsing is done.
-    async getParsedSave(id: string): Promise<Record<string, unknown>> {
-        const url = `${this.API_BASE}/getParsedSave?id=${encodeURIComponent(id)}`
+    async getSaveData(id: string): Promise<Record<string, unknown>> {
+        const url = `${this.API_BASE}/getSaveData?id=${encodeURIComponent(id)}`
        const response = await fetch(url)

        if (!response.ok) {
@@ -74,6 +100,24 @@ export class SaveParserBackendService {

        return await response.json()
    }
+
+    /**
+     * Fetches the stored save archive, exactly as it was uploaded. Turning it
+     * back into a file the game can load is done in the browser.
+     */
+    async getSaveFile(id: string): Promise<Blob> {
+        const url = `${this.API_BASE}/getSaveFile?id=${encodeURIComponent(id)}`
+        const response = await fetch(url)
+
+        if (!response.ok) {
+            throw new Error(
+                `Failed to get save file: ${response.statusText} ` +
+                    (await getBodyTextOrEmpty(response))
+            )
+        }
+
+        return await response.blob()
+    }
 }

 export const saveParserBackendService = new SaveParserBackendService()