Timerix/ParadoxSaveParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parser rewrite

Browse Source
This commit is contained in:
Timerix
2026-09-15 00:14:26 +02:00
parent 3dde5e5acf
commit 1d53f6930a
11 changed files with 889 additions and 667 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ParadoxSaveParser.Lib/Tokenizer.cs
+456
View File
@@ -0,0 +1,456 @@
using System.Buffers;
namespace ParadoxSaveParser.Lib;
internal enum TokenType : byte
{
// default value, so a slot that was never scanned is recognizably empty
Invalid,
// any bare or quoted value: the tokenizer does not tell numbers from strings
StringOrNumber,
Equals,
BracketOpen,
BracketClose,
EndOfFile
}
/// <summary>
/// Splits a Paradox text save into tokens, reading the stream through a reusable buffer.
/// Token text is exposed as a span into that buffer, so a token costs no allocation at all
/// unless it happens to straddle a buffer refill.
/// <para>
/// The parser can tell the tokenizer to throw away a whole <c>{...}</c> block with
/// <see cref="SkipBlock" />. Blocks rejected by the search expression are then never
/// turned into tokens or strings, which is what makes a query cheaper than a full parse.
/// </para>
/// </summary>
internal sealed class Tokenizer
{
/// <summary>Bytes that end an unquoted token.</summary>
private static readonly SearchValues<byte> TokenEnd = SearchValues.Create(" \t\r\n={}\""u8);
private static readonly SearchValues<byte> Whitespace = SearchValues.Create(" \t\r\n"u8);
/// <summary>Everything <see cref="SkipBlock" /> has to look at while counting depth.</summary>
private static readonly SearchValues<byte> BlockChars = SearchValues.Create("{}\""u8);
/// <summary>One scanned token. Reused forever, so scanning a token allocates nothing.</summary>
private sealed class Slot
{
public TokenType Type;
// position of the token's first byte in the file, for error messages
public int Line;
public short Column;
// text is either a range of the shared buffer, or, once a refill would overwrite it,
// a copy in Scratch
public int Start;
public int Length;
// grown on demand and kept between tokens, so long tokens stop reallocating after a while
public byte[] Scratch = [];
public bool InScratch;
}
private readonly Stream _stream;
private readonly byte[] _buffer;
// current token plus up to two lookahead tokens
private readonly Slot[] _slots = [new Slot(), new Slot(), new Slot()];
// index of the current token; the slots are used as a ring, so the array is never shifted
private int _current;
// how many slots after _current already hold a scanned, not yet consumed token
private int _lookahead;
// read position and amount of valid data in _buffer
private int _pos;
private int _length;
// set once the stream has no more bytes; stops Fill() from calling Read() again
private bool _eof;
// position of _pos in the file; _column is 0-based here and reported 1-based
private int _line = 1;
private int _column;
public Tokenizer(Stream stream, int bufferSize)
{
if (bufferSize < 16)
throw new ArgumentOutOfRangeException(nameof(bufferSize), bufferSize, "buffer is too small");
_stream = stream;
_buffer = new byte[bufferSize];
}
public TokenType Type => _slots[_current].Type;
public int Line => _slots[_current].Line;
public short Column => _slots[_current].Column;
/// <summary>
/// Bytes of the current token, without quotes. Valid only until the next
/// <see cref="Read" />, because the buffer underneath it gets reused.
/// </summary>
public ReadOnlySpan<byte> Text
{
get
{
var slot = _slots[_current];
// a token that survived a refill was copied out; everything else still points into the buffer
return slot.InScratch
? slot.Scratch.AsSpan(0, slot.Length)
: _buffer.AsSpan(slot.Start, slot.Length);
}
}
/// <summary>Consumes the file's magic header and checks it, before any token is scanned.</summary>
public void ReadHeader(ReadOnlySpan<byte> expected, Encoding encoding)
{
// read straight from the stream: this runs before the buffer holds anything
Span<byte> head = stackalloc byte[expected.Length];
_stream.ReadExactly(head);
if (!head.SequenceEqual(expected))
throw new Exception($"Invalid gamestate header. " +
$"Expected '{encoding.GetString(expected)}', got '{encoding.GetString(head)}'.");
}
/// <returns>false at end of file</returns>
public bool Read()
{
_current = NextSlot(_current);
// the next slot may already be filled by an earlier PeekType, then there is nothing to scan
if (_lookahead > 0)
_lookahead--;
else Scan(_slots[_current]);
return _slots[_current].Type != TokenType.EndOfFile;
}
/// <summary>
/// Type of a token that has not been consumed yet, 1 or 2 tokens ahead of the current one.
/// Only types are available: the parser never needs the text of a token it has not reached.
/// </summary>
public TokenType PeekType(int offset)
{
// scan only as far as asked, so lookahead never runs into a block SkipBlock is about to drop
while (_lookahead < offset)
{
// first slot after the ones that are already filled
int slot = _current;
for (int i = 0; i <= _lookahead; i++)
slot = NextSlot(slot);
Scan(_slots[slot]);
_lookahead++;
}
int index = _current;
for (int i = 0; i < offset; i++)
index = NextSlot(index);
return _slots[index].Type;
}
/// <summary>
/// Throws away the block opened by the current <c>{</c> token without tokenizing it:
/// raw bytes are scanned for braces until the depth returns to zero.
/// Braces inside quoted strings are text and do not change the depth.
/// Leaves the closing <c>}</c> as the current token.
/// </summary>
public void SkipBlock()
{
int depth = 1;
// tokens that lookahead already pulled out of the buffer still count towards the depth
while (depth > 0 && _lookahead > 0)
{
_current = NextSlot(_current);
_lookahead--;
switch (_slots[_current].Type)
{
case TokenType.BracketOpen:
depth++;
break;
case TokenType.BracketClose:
depth--;
break;
case TokenType.EndOfFile:
return;
}
}
bool inQuotes = false;
while (depth > 0)
{
if (_pos >= _length && !Fill())
break; // unbalanced braces: the file ended inside the block
var span = _buffer.AsSpan(_pos, _length - _pos);
// inside a string only the closing quote matters, braces there are ordinary characters
int i = inQuotes ? span.IndexOf((byte)'\"') : span.IndexOfAny(BlockChars);
if (i < 0)
{
// nothing interesting in this bufferful, drop all of it and refill
Consume(span.Length);
continue;
}
byte b = span[i];
Consume(i + 1);
if (b == (byte)'\"')
inQuotes = !inQuotes;
else if (b == (byte)'{')
depth++;
else depth--;
}
// hand the parser the closing brace it expects, without having tokenized anything inside
var current = _slots[_current];
current.Type = depth == 0 ? TokenType.BracketClose : TokenType.EndOfFile;
current.Length = 0;
current.InScratch = false;
current.Line = _line;
current.Column = (short)_column;
}
public string Describe(Encoding encoding) => Type switch
{
TokenType.StringOrNumber => Text.Length == 0 ? "NULL" : encoding.GetString(Text),
TokenType.Equals => "=",
TokenType.BracketOpen => "{",
TokenType.BracketClose => "}",
TokenType.EndOfFile => "END_OF_FILE",
_ => "INVALID_TOKEN",
};
/// <summary>Slot indices wrap around: the three slots form a ring buffer.</summary>
private int NextSlot(int i) => i + 1 == _slots.Length ? 0 : i + 1;
/// <summary>Reads the next token from the stream into <paramref name="slot" />.</summary>
private void Scan(Slot slot)
{
// loops only to skip comments, which produce no token
while (true)
{
if (!SkipWhitespace())
{
slot.Type = TokenType.EndOfFile;
slot.Length = 0;
slot.InScratch = false;
slot.Line = _line;
slot.Column = (short)_column;
return;
}
slot.Line = _line;
slot.Column = (short)(_column + 1); // columns are reported 1-based
// SkipWhitespace guarantees at least one buffered byte here
byte b = _buffer[_pos];
switch (b)
{
case (byte)'=':
ConsumeFlat(1);
SetDelimiter(slot, TokenType.Equals);
return;
case (byte)'{':
ConsumeFlat(1);
SetDelimiter(slot, TokenType.BracketOpen);
return;
case (byte)'}':
ConsumeFlat(1);
SetDelimiter(slot, TokenType.BracketClose);
return;
case (byte)'\"':
ReadQuoted(slot);
return;
default:
ReadBare(slot);
// comments are dropped, same as before: a token starting with '#' is not emitted
if (slot.Length > 0 && FirstByte(slot) == (byte)'#')
continue;
return;
}
}
}
/// <summary>Single-character tokens carry no text of their own.</summary>
private static void SetDelimiter(Slot slot, TokenType type)
{
slot.Type = type;
slot.Length = 0;
slot.InScratch = false;
}
/// <summary>Reads an unquoted token, which ends at the first delimiter byte.</summary>
private void ReadBare(Slot slot)
{
StartText(slot);
while (true)
{
var span = _buffer.AsSpan(_pos, _length - _pos);
// one vectorized search replaces a loop over the token's bytes
int end = span.IndexOfAny(TokenEnd);
if (end >= 0)
{
// the delimiter itself is left for the next Scan to classify
AppendText(slot, span[..end]);
ConsumeFlat(end);
return;
}
// token runs to the end of the buffer and continues in the next one
AppendText(slot, span);
ConsumeFlat(span.Length);
if (!Fill())
return;
}
}
/// <summary>
/// Reads a quoted token. Everything up to the next <c>"</c> is text, braces included;
/// the format has no escape sequences, so a quote always closes the string.
/// </summary>
private void ReadQuoted(Slot slot)
{
ConsumeFlat(1); // opening quote
StartText(slot);
while (true)
{
if (_pos >= _length && !Fill())
return; // unterminated string at end of file
var span = _buffer.AsSpan(_pos, _length - _pos);
int end = span.IndexOf((byte)'\"');
if (end >= 0)
{
AppendText(slot, span[..end]);
Consume(end + 1); // content plus the closing quote
return;
}
// Consume, not ConsumeFlat: a quoted value is allowed to span several lines
AppendText(slot, span);
Consume(span.Length);
}
}
/// <summary>Begins a text token at the current read position.</summary>
private void StartText(Slot slot)
{
slot.Type = TokenType.StringOrNumber;
slot.Start = _pos;
slot.Length = 0;
slot.InScratch = false;
}
/// <summary>Extends a text token by one chunk of bytes taken from the buffer.</summary>
private void AppendText(Slot slot, ReadOnlySpan<byte> chunk)
{
if (!slot.InScratch)
{
// chunks always continue where the previous one ended, so the range just grows
// still contiguous in the buffer, nothing to copy
if (slot.Length == 0)
slot.Start = _pos;
slot.Length += chunk.Length;
return;
}
// the start of this token is already out of the buffer, so the rest has to follow it
EnsureScratch(slot, slot.Length + chunk.Length);
chunk.CopyTo(slot.Scratch.AsSpan(slot.Length));
slot.Length += chunk.Length;
}
/// <summary>First byte of a token, wherever its text currently lives. Used to spot comments.</summary>
private byte FirstByte(Slot slot) => slot.InScratch ? slot.Scratch[0] : _buffer[slot.Start];
private static void EnsureScratch(Slot slot, int size)
{
if (slot.Scratch.Length >= size)
return;
// doubling keeps a token that is appended chunk by chunk from resizing on every chunk
int capacity = Math.Max(size, slot.Scratch.Length * 2);
Array.Resize(ref slot.Scratch, capacity);
}
/// <summary>Moves the read position onto the next non-whitespace byte.</summary>
/// <returns>false at end of file</returns>
private bool SkipWhitespace()
{
while (true)
{
if (_pos >= _length && !Fill())
return false;
var span = _buffer.AsSpan(_pos, _length - _pos);
int i = span.IndexOfAnyExcept(Whitespace);
if (i < 0)
{
// whitespace to the end of the buffer, keep going in the next one
Consume(span.Length);
continue;
}
if (i > 0)
Consume(i);
return true;
}
}
/// <summary>Refills the buffer from the stream.</summary>
/// <returns>false if the stream is exhausted</returns>
private bool Fill()
{
if (_eof)
return false;
// text of live tokens lives in the buffer, so it has to be copied out before overwriting
MaterializeSlots();
// a short read is fine, the next Fill picks up the rest
_length = _stream.Read(_buffer, 0, _buffer.Length);
_pos = 0;
if (_length > 0)
return true;
_eof = true;
_length = 0;
return false;
}
/// <summary>
/// Copies every token that still points into the buffer out to its own scratch array.
/// Called just before a refill, which is the only moment that text can be lost.
/// </summary>
private void MaterializeSlots()
{
foreach (var slot in _slots)
{
// delimiters and empty tokens own no text, and a copied one needs no second copy
if (slot.InScratch || slot.Length == 0 || slot.Type != TokenType.StringOrNumber)
continue;
EnsureScratch(slot, slot.Length);
_buffer.AsSpan(slot.Start, slot.Length).CopyTo(slot.Scratch);
slot.InScratch = true;
}
}
/// <summary>Consumes bytes that cannot contain a line break.</summary>
private void ConsumeFlat(int count)
{
_pos += count;
_column += count;
}
/// <summary>Consumes bytes that may contain line breaks, keeping line and column exact.</summary>
private void Consume(int count)
{
var slice = _buffer.AsSpan(_pos, count);
// the last break decides the column, and only then is counting all of them worth it
int lastBreak = slice.LastIndexOf((byte)'\n');
if (lastBreak < 0)
{
_column += count;
}
else
{
_line += slice.Count((byte)'\n');
// bytes left after the final break
_column = count - lastBreak - 1;
}
_pos += count;
}
}