Altair-sh
314 lines
11 KiB
TypeScript
314 lines
11 KiB
TypeScript
// Reading and writing the zip archives the Paradox games themselves use.
|
|
//
|
|
// This is written here rather than taken from a library because the games are
|
|
// picky about the headers and no browser zip library can produce them: they
|
|
// all derive the DOS date from a Date and clamp it to 1980, while the games
|
|
// write zeros, and the streaming writers emit a data descriptor plus a Zip64
|
|
// extra field for every entry, which the games reject. The heavy part of zip,
|
|
// deflate, is in the browser itself as CompressionStream('deflate-raw'), so
|
|
// what is left is the header layout below.
|
|
//
|
|
// The field values follow a known good reference, which produces archives the
|
|
// games accept: http://pastebin.com/nyX1B4rH
|
|
// A real EU4 save agrees with it: MS-DOS as the system of origin, encoding
|
|
// software 6.3, extract version 2.0, no data descriptor, no extra field and a
|
|
// zeroed DOS date. 7-Zip's NTFS timestamps are a known way to produce a save
|
|
// that will not load: https://stellaris.paradoxwikis.com/Save-game_editing
|
|
|
|
const LOCAL_HEADER_SIGNATURE = 0x04034b50
|
|
const CENTRAL_HEADER_SIGNATURE = 0x02014b50
|
|
const END_OF_CENTRAL_DIR_SIGNATURE = 0x06054b50
|
|
|
|
const LOCAL_HEADER_SIZE = 30
|
|
const CENTRAL_HEADER_SIZE = 46
|
|
const END_OF_CENTRAL_DIR_SIZE = 22
|
|
|
|
const METHOD_STORED = 0
|
|
const METHOD_DEFLATE = 8
|
|
|
|
// "version needed to extract" 2.0, which is what deflate requires.
|
|
const EXTRACT_VERSION = 20
|
|
// "version made by": MS-DOS (0) in the high byte, 6.3 (63) in the low byte.
|
|
const CREATE_VERSION = (0 << 8) | 63
|
|
|
|
// Anything at or above this needs the zip64 format, which no save comes near.
|
|
const MAX_ENTRY_SIZE = 0xffffffff
|
|
|
|
// First bytes of every zip archive.
|
|
const ZIP_MAGIC = [0x50, 0x4b, 0x03, 0x04]
|
|
|
|
export interface GameZipEntry {
|
|
name: string
|
|
size: number
|
|
}
|
|
|
|
// The standard CRC-32 lookup table.
|
|
const CRC_TABLE = new Uint32Array(256).map((_, i) => {
|
|
let value = i
|
|
for (let bit = 0; bit < 8; bit++) {
|
|
value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1
|
|
}
|
|
return value >>> 0
|
|
})
|
|
|
|
// Continues a CRC-32 over one more chunk. Start with 0; the result is the
|
|
// checksum of everything so far, so an entry is summed as it streams past.
|
|
function updateCrc(crc: number, data: Uint8Array): number {
|
|
let value = ~crc
|
|
for (const byte of data) {
|
|
value = CRC_TABLE[(value ^ byte) & 0xff] ^ (value >>> 8)
|
|
}
|
|
return ~value >>> 0
|
|
}
|
|
|
|
/**
|
|
* Tells whether the file is a zip archive, by its magic bytes rather than by
|
|
* its extension: the games use the same extension for both an archived and a
|
|
* plain text save.
|
|
*/
|
|
export async function isZipFile(file: Blob): Promise<boolean> {
|
|
const header = new Uint8Array(
|
|
await file.slice(0, ZIP_MAGIC.length).arrayBuffer()
|
|
)
|
|
return ZIP_MAGIC.every((byte, i) => header[i] === byte)
|
|
}
|
|
|
|
// Little endian reader over a slice of the archive.
|
|
function reader(data: Uint8Array) {
|
|
const view = new DataView(data.buffer, data.byteOffset, data.byteLength)
|
|
return {
|
|
u16: (offset: number) => view.getUint16(offset, true),
|
|
u32: (offset: number) => view.getUint32(offset, true),
|
|
}
|
|
}
|
|
|
|
interface CentralEntry {
|
|
name: string
|
|
method: number
|
|
compressedSize: number
|
|
size: number
|
|
localHeaderOffset: number
|
|
}
|
|
|
|
// Reads the central directory, which lists every entry with its size and
|
|
// position. Only the last few hundred bytes of the archive are touched.
|
|
async function readCentralDirectory(zip: Blob): Promise<CentralEntry[]> {
|
|
const end = new Uint8Array(
|
|
await zip.slice(zip.size - END_OF_CENTRAL_DIR_SIZE).arrayBuffer()
|
|
)
|
|
const endFields = reader(end)
|
|
if (endFields.u32(0) !== END_OF_CENTRAL_DIR_SIGNATURE) {
|
|
throw new Error('Save archive has no end of central directory')
|
|
}
|
|
|
|
const entryCount = endFields.u16(10)
|
|
const directoryOffset = endFields.u32(16)
|
|
const directory = new Uint8Array(
|
|
await zip.slice(directoryOffset, zip.size).arrayBuffer()
|
|
)
|
|
const fields = reader(directory)
|
|
|
|
const entries: CentralEntry[] = []
|
|
let offset = 0
|
|
for (let i = 0; i < entryCount; i++) {
|
|
if (fields.u32(offset) !== CENTRAL_HEADER_SIGNATURE) {
|
|
throw new Error('Save archive has a corrupted entry header')
|
|
}
|
|
|
|
const nameLength = fields.u16(offset + 28)
|
|
const extraLength = fields.u16(offset + 30)
|
|
const commentLength = fields.u16(offset + 32)
|
|
const nameStart = offset + CENTRAL_HEADER_SIZE
|
|
entries.push({
|
|
name: new TextDecoder().decode(
|
|
directory.subarray(nameStart, nameStart + nameLength)
|
|
),
|
|
method: fields.u16(offset + 10),
|
|
compressedSize: fields.u32(offset + 20),
|
|
size: fields.u32(offset + 24),
|
|
localHeaderOffset: fields.u32(offset + 42),
|
|
})
|
|
offset += CENTRAL_HEADER_SIZE + nameLength + extraLength + commentLength
|
|
}
|
|
return entries
|
|
}
|
|
|
|
/**
|
|
* Reads every file of a save archive in the order they are stored, asking the
|
|
* caller where each one should go and writing it straight into that stream.
|
|
*
|
|
* Sizes come from the central directory, so they are known before an entry
|
|
* starts streaming, and the entry data itself is never held in memory.
|
|
*/
|
|
export async function readGameZip(
|
|
zip: Blob,
|
|
openEntry: (entry: GameZipEntry) => WritableStream<Uint8Array>
|
|
): Promise<void> {
|
|
for (const entry of await readCentralDirectory(zip)) {
|
|
// The local header repeats the name and may carry a different extra
|
|
// field, so the data can only be located through it.
|
|
const header = new Uint8Array(
|
|
await zip
|
|
.slice(
|
|
entry.localHeaderOffset,
|
|
entry.localHeaderOffset + LOCAL_HEADER_SIZE
|
|
)
|
|
.arrayBuffer()
|
|
)
|
|
const fields = reader(header)
|
|
if (fields.u32(0) !== LOCAL_HEADER_SIGNATURE) {
|
|
throw new Error(`Save archive entry "${entry.name}" is corrupted`)
|
|
}
|
|
|
|
const dataStart =
|
|
entry.localHeaderOffset +
|
|
LOCAL_HEADER_SIZE +
|
|
fields.u16(26) +
|
|
fields.u16(28)
|
|
const data = zip.slice(dataStart, dataStart + entry.compressedSize)
|
|
|
|
if (entry.method === METHOD_STORED) {
|
|
await data.stream().pipeTo(openEntry(entry))
|
|
} else if (entry.method === METHOD_DEFLATE) {
|
|
await data
|
|
.stream()
|
|
.pipeThrough(new DecompressionStream('deflate-raw'))
|
|
.pipeTo(openEntry(entry))
|
|
} else {
|
|
throw new Error(
|
|
`Save archive entry "${entry.name}" uses compression ` +
|
|
`method ${entry.method}, which the games do not produce`
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Writes a local file header. The date and time fields stay zero, which is
|
|
// what the games write and what a Date cannot express.
|
|
function localHeader(entry: WrittenEntry): Uint8Array<ArrayBuffer> {
|
|
const header = new Uint8Array(LOCAL_HEADER_SIZE + entry.nameBytes.length)
|
|
const view = new DataView(header.buffer)
|
|
view.setUint32(0, LOCAL_HEADER_SIGNATURE, true)
|
|
view.setUint16(4, EXTRACT_VERSION, true)
|
|
view.setUint16(8, METHOD_DEFLATE, true)
|
|
view.setUint32(14, entry.crc, true)
|
|
view.setUint32(18, entry.compressedSize, true)
|
|
view.setUint32(22, entry.size, true)
|
|
view.setUint16(26, entry.nameBytes.length, true)
|
|
header.set(entry.nameBytes, LOCAL_HEADER_SIZE)
|
|
return header
|
|
}
|
|
|
|
// Writes the central directory record for one entry.
|
|
function centralHeader(entry: WrittenEntry): Uint8Array<ArrayBuffer> {
|
|
const header = new Uint8Array(CENTRAL_HEADER_SIZE + entry.nameBytes.length)
|
|
const view = new DataView(header.buffer)
|
|
view.setUint32(0, CENTRAL_HEADER_SIGNATURE, true)
|
|
view.setUint16(4, CREATE_VERSION, true)
|
|
view.setUint16(6, EXTRACT_VERSION, true)
|
|
view.setUint16(10, METHOD_DEFLATE, true)
|
|
view.setUint32(16, entry.crc, true)
|
|
view.setUint32(20, entry.compressedSize, true)
|
|
view.setUint32(24, entry.size, true)
|
|
view.setUint16(28, entry.nameBytes.length, true)
|
|
view.setUint32(42, entry.localHeaderOffset, true)
|
|
header.set(entry.nameBytes, CENTRAL_HEADER_SIZE)
|
|
return header
|
|
}
|
|
|
|
// Writes the end of central directory record that closes the archive.
|
|
function endOfCentralDirectory(
|
|
entries: WrittenEntry[],
|
|
directoryOffset: number,
|
|
directorySize: number
|
|
): Uint8Array<ArrayBuffer> {
|
|
const record = new Uint8Array(END_OF_CENTRAL_DIR_SIZE)
|
|
const view = new DataView(record.buffer)
|
|
view.setUint32(0, END_OF_CENTRAL_DIR_SIGNATURE, true)
|
|
view.setUint16(8, entries.length, true)
|
|
view.setUint16(10, entries.length, true)
|
|
view.setUint32(12, directorySize, true)
|
|
view.setUint32(16, directoryOffset, true)
|
|
return record
|
|
}
|
|
|
|
interface WrittenEntry {
|
|
nameBytes: Uint8Array
|
|
crc: number
|
|
size: number
|
|
compressedSize: number
|
|
localHeaderOffset: number
|
|
}
|
|
|
|
/**
|
|
* Builds a save archive the games accept, one streamed file at a time.
|
|
*/
|
|
export function createGameZipBuilder() {
|
|
const entries: WrittenEntry[] = []
|
|
// Headers are plain bytes and entry data is a blob, so assembling the
|
|
// archive at the end copies nothing through memory.
|
|
const parts: BlobPart[] = []
|
|
let offset = 0
|
|
|
|
return {
|
|
async add(name: string, body: ReadableStream<Uint8Array>) {
|
|
const nameBytes = new TextEncoder().encode(name)
|
|
let crc = 0
|
|
let size = 0
|
|
|
|
// Sums the entry as it goes past. The cast only drops the shared
|
|
// buffer case from the type; a save never arrives in one.
|
|
const summed = body.pipeThrough(
|
|
new TransformStream<Uint8Array, Uint8Array<ArrayBuffer>>({
|
|
transform(chunk, controller) {
|
|
crc = updateCrc(crc, chunk)
|
|
size += chunk.length
|
|
controller.enqueue(chunk as Uint8Array<ArrayBuffer>)
|
|
},
|
|
})
|
|
)
|
|
|
|
// The sizes and the checksum belong in the local header, which is
|
|
// written before the data, so the entry is deflated first. Only
|
|
// the compressed bytes are kept, and in a blob rather than in the
|
|
// javascript heap.
|
|
const compressed = await new Response(
|
|
summed.pipeThrough(new CompressionStream('deflate-raw'))
|
|
).blob()
|
|
|
|
if (size > MAX_ENTRY_SIZE) {
|
|
throw new Error(`Save file "${name}" is too large for a zip`)
|
|
}
|
|
|
|
const entry: WrittenEntry = {
|
|
nameBytes,
|
|
crc,
|
|
size,
|
|
compressedSize: compressed.size,
|
|
localHeaderOffset: offset,
|
|
}
|
|
entries.push(entry)
|
|
|
|
const header = localHeader(entry)
|
|
parts.push(header, compressed)
|
|
offset += header.length + compressed.size
|
|
},
|
|
|
|
async finish(): Promise<Blob> {
|
|
const directory = entries.map(centralHeader)
|
|
const directorySize = directory.reduce(
|
|
(total, header) => total + header.length,
|
|
0
|
|
)
|
|
return new Blob(
|
|
[
|
|
...parts,
|
|
...directory,
|
|
endOfCentralDirectory(entries, offset, directorySize),
|
|
],
|
|
{ type: 'application/zip' }
|
|
)
|
|
},
|
|
}
|
|
}
|