// Zstandard compression and decompression as stream transforms. // // Saves are re-compressed in the browser because the parser backend runs on a // cheap two core server with little disk space. zstd was picked over lzma2 and // brotli because its decompression speed does not depend on the level the data // was compressed with, so the level can be raised without costing the server // anything, and because its wasm build is by far the smallest of the three. // // @tootallnate/zstd-wasm does the work: it is upstream zstd 1.5.7 built with // wasi-sdk and driven through ZSTD_compressStream2, so it compresses a stream // as it arrives rather than a buffer at a time. A save of any size comes out // as one zstd frame, and memory stays at the library's own input and output // buffers. import { ZstdCompressStream, ZstdDecompressStream, } from '@tootallnate/zstd-wasm' // The wasm file ships inside the package. Turbopack rewrites this into the URL // it emits the asset at, so the 534 kB binary is fetched once at runtime and // cached by the browser instead of being inlined into the worker bundle. const wasmUrl = new URL( '../../../node_modules/@tootallnate/zstd-wasm/dist/zstd.wasm', import.meta.url ) // Compiling takes a moment, so it happens once and both directions share it. let wasmModule: Promise | null = null function getWasm(): Promise { // compileStreaming would need the server to send Content-Type: // application/wasm, which is not worth depending on. wasmModule ??= fetch(wasmUrl) .then((res) => res.arrayBuffer()) .then((bytes) => WebAssembly.compile(bytes)) return wasmModule } /** * Compresses a stream into a single zstd frame. The level goes from 1 to 22. */ export function createZstdCompressStream( compressionLevel: number ): TransformStream { return new ZstdCompressStream(getWasm(), compressionLevel) } /** * Decompresses a zstd stream. Frames that follow one another are read straight * through, so an archive packed as several frames still opens. */ export function createZstdDecompressStream(): TransformStream< Uint8Array, Uint8Array > { return new ZstdDecompressStream(getWasm()) }