diff --git a/ParadoxSaveParser.Benchmarks/ExtractionBenchmarks.cs b/ParadoxSaveParser.Benchmarks/ExtractionBenchmarks.cs index ef7c2a5..b175b84 100644 --- a/ParadoxSaveParser.Benchmarks/ExtractionBenchmarks.cs +++ b/ParadoxSaveParser.Benchmarks/ExtractionBenchmarks.cs @@ -20,13 +20,10 @@ public class ExtractionBenchmarks private byte[] _json = null!; private ISearchExpression _query = null!; - private Regex _flatInterpreted = null!; - private Regex _flatCompiled = null!; - private Regex _flatSourceGen = null!; + private Regex _technologyBlock = null!; private Regex _pathAwareCompiled = null!; private Regex _pathAwareNonBacktracking = null!; private Regex _countriesBlock = null!; - private PcreRegex _pcreFlat = null!; private PcreRegex _pcrePathAware = null!; [GlobalSetup] @@ -37,13 +34,10 @@ public class ExtractionBenchmarks _json = File.ReadAllBytes(BenchData.JsonPath); _query = SearchExpressionCompiler.Compile(BenchData.PdxQuery); - _flatInterpreted = new Regex(Extractors.FlatPattern, RegexOptions.None); - _flatCompiled = new Regex(Extractors.FlatPattern, RegexOptions.Compiled); - _flatSourceGen = Extractors.FlatSourceGen(); + _technologyBlock = new Regex(Extractors.TechnologyBlockPattern, RegexOptions.Compiled); _pathAwareCompiled = new Regex(Extractors.PathAwarePattern, RegexOptions.Compiled); _pathAwareNonBacktracking = new Regex(Extractors.PathAwarePattern, RegexOptions.NonBacktracking); _countriesBlock = new Regex(Extractors.CountriesBlockPattern, RegexOptions.Compiled); - _pcreFlat = new PcreRegex(Extractors.FlatPattern, PcreOptions.Compiled); _pcrePathAware = new PcreRegex(Extractors.PathAwarePattern, PcreOptions.Compiled); } @@ -53,26 +47,14 @@ public class ExtractionBenchmarks [Benchmark(Description = "Full parse, then select")] public long FullParse() => Extractors.FullParseThenSelect(_pdx); - [Benchmark(Description = ".NET Regex flat, interpreted")] - public long RegexFlatInterpreted() => Extractors.RegexScan(_flatInterpreted, _pdxText); - - [Benchmark(Description = ".NET Regex flat, compiled")] - public long RegexFlatCompiled() => Extractors.RegexScan(_flatCompiled, _pdxText); - - [Benchmark(Description = ".NET Regex flat, source generated")] - public long RegexFlatSourceGen() => Extractors.RegexScan(_flatSourceGen, _pdxText); - [Benchmark(Description = ".NET Regex path aware, compiled")] public long RegexPathAwareCompiled() => Extractors.RegexScan(_pathAwareCompiled, _pdxText); [Benchmark(Description = ".NET Regex path aware, NonBacktracking")] public long RegexPathAwareNonBacktracking() => Extractors.RegexScan(_pathAwareNonBacktracking, _pdxText); - [Benchmark(Description = ".NET Regex balanced block + flat")] - public long RegexBalancedTwoStage() => Extractors.RegexTwoStage(_countriesBlock, _flatCompiled, _pdxText); - - [Benchmark(Description = "PCRE.NET flat, JIT compiled")] - public long PcreFlat() => Extractors.PcreScan(_pcreFlat, _pdxText); + [Benchmark(Description = ".NET Regex balanced block + inner scan")] + public long RegexBalancedTwoStage() => Extractors.RegexTwoStage(_countriesBlock, _technologyBlock, _pdxText); [Benchmark(Description = "PCRE.NET path aware, JIT compiled")] public long PcrePathAware() => Extractors.PcreScan(_pcrePathAware, _pdxText); diff --git a/ParadoxSaveParser.Benchmarks/Extractors.cs b/ParadoxSaveParser.Benchmarks/Extractors.cs index e180733..25ee59d 100644 --- a/ParadoxSaveParser.Benchmarks/Extractors.cs +++ b/ParadoxSaveParser.Benchmarks/Extractors.cs @@ -65,10 +65,11 @@ public static partial class Extractors // ---------------------------------------------------------------- .NET Regex /// - /// Flat scan: matches technology blocks anywhere in the file. - /// Fastest regex option, but it does not verify that the match really sits under countries. + /// Matches a single technology block. On its own it would match anywhere in the file, + /// so it is only used as the second stage of , inside the + /// already extracted countries block. /// - public const string FlatPattern = + public const string TechnologyBlockPattern = @"\n\t\ttechnology=\{\n\t\t\tadm_tech=(\d+)\n\t\t\tdip_tech=(\d+)\n\t\t\tmil_tech=(\d+)\n\t\t\}"; /// @@ -109,9 +110,6 @@ public static partial class Extractors return RegexScan(innerRegex, block.Value); } - [GeneratedRegex(FlatPattern)] - public static partial Regex FlatSourceGen(); - [GeneratedRegex(PathAwarePattern)] public static partial Regex PathAwareSourceGen(); diff --git a/ParadoxSaveParser.Benchmarks/ParadoxSaveParser.Benchmarks.csproj b/ParadoxSaveParser.Benchmarks/ParadoxSaveParser.Benchmarks.csproj index 3edacc6..8f6fc8f 100644 --- a/ParadoxSaveParser.Benchmarks/ParadoxSaveParser.Benchmarks.csproj +++ b/ParadoxSaveParser.Benchmarks/ParadoxSaveParser.Benchmarks.csproj @@ -10,7 +10,7 @@ - + diff --git a/ParadoxSaveParser.Benchmarks/Program.cs b/ParadoxSaveParser.Benchmarks/Program.cs index f152946..3f7f681 100644 --- a/ParadoxSaveParser.Benchmarks/Program.cs +++ b/ParadoxSaveParser.Benchmarks/Program.cs @@ -43,16 +43,13 @@ public static class Program Time("SearchExpression", () => Extractors.SearchExpression(pdx, query)); Time("FullParse", () => Extractors.FullParseThenSelect(pdx)); - Time("Regex flat compiled", - () => Extractors.RegexScan(new Regex(Extractors.FlatPattern, RegexOptions.Compiled), text)); Time("Regex path aware compiled", () => Extractors.RegexScan(new Regex(Extractors.PathAwarePattern, RegexOptions.Compiled), text)); Time("Regex path aware nonbacktracking", () => Extractors.RegexScan(new Regex(Extractors.PathAwarePattern, RegexOptions.NonBacktracking), text)); Time("Regex balanced two stage", () => Extractors.RegexTwoStage(new Regex(Extractors.CountriesBlockPattern, RegexOptions.Compiled), - new Regex(Extractors.FlatPattern, RegexOptions.Compiled), text)); - Time("PCRE flat", () => Extractors.PcreScan(new PcreRegex(Extractors.FlatPattern, PcreOptions.Compiled), text)); + new Regex(Extractors.TechnologyBlockPattern, RegexOptions.Compiled), text)); Time("PCRE path aware", () => Extractors.PcreScan(new PcreRegex(Extractors.PathAwarePattern, PcreOptions.Compiled), text)); Time("Utf8JsonReader", () => Extractors.Utf8JsonReaderScan(json)); diff --git a/ParadoxSaveParser.Benchmarks/README.md b/ParadoxSaveParser.Benchmarks/README.md index d110d74..5ddffef 100644 --- a/ParadoxSaveParser.Benchmarks/README.md +++ b/ParadoxSaveParser.Benchmarks/README.md @@ -9,7 +9,7 @@ regex engines and `jq` on one realistic task: |---|---| | this repo | `countries.*.technology` | | jq | `.countries \| map_values(.technology)` | -| .NET `Regex` / PCRE.NET | see `Extractors.FlatPattern` / `Extractors.PathAwarePattern` | +| .NET `Regex` / PCRE.NET | see `Extractors.PathAwarePattern` / `Extractors.CountriesBlockPattern` | | `Utf8JsonReader` / `JsonDocument` | hand written navigation | ## Corpus @@ -48,64 +48,70 @@ shows immediately when an engine finds something different from the others. ## Results -AMD Ryzen 7 5700X, 16 logical cores, Windows 10 21H2, .NET 10.0.12, jq 1.8.2, -109 MB save / 92 MB JSON twin, BenchmarkDotNet `RunStrategy.Monitoring`, 5 iterations. +AMD Ryzen 7 5700X, 8 physical cores (16 logical), Windows 10 21H2, .NET 10.0.12, jq 1.8.2, +109 MB save / 92 MB JSON twin, BenchmarkDotNet `RunStrategy.Monitoring`, +3 warmups and 10 iterations for the in-process engines, 5 for jq. | engine | mean | vs this project | allocated | correct? | |---|---:|---:|---:|---| -| .NET Regex flat, source generated | 14.8 ms | 0.02x | 1.2 MB | path-blind | -| .NET Regex flat, compiled | 17.0 ms | 0.03x | 1.2 MB | path-blind | -| .NET Regex flat, interpreted | 19.3 ms | 0.03x | 1.2 MB | path-blind | -| PCRE.NET flat, JIT compiled | 97.4 ms | 0.15x | 0.9 MB | path-blind | -| .NET Regex balanced block + flat | 112.3 ms | 0.17x | 224 MB | yes | -| .NET Regex path aware, compiled | 116.2 ms | 0.18x | 1.4 MB | 1869/1870 | -| PCRE.NET path aware, JIT compiled | 122.9 ms | 0.19x | 0.9 MB | 1869/1870 | -| Utf8JsonReader over JSON twin | 155.7 ms | 0.24x | 0 B | yes | -| JsonDocument over JSON twin | 321.4 ms | 0.50x | 0 B (native) | yes | -| .NET Regex path aware, NonBacktracking | 444.5 ms | 0.69x | 32 MB | 1869/1870 | -| **SearchExpression (this project)** | **645.1 ms** | **1.00x** | **1.6 MB** | yes | -| Full parse, then select | 1831 ms | 2.84x | 1153 MB | yes | -| jq `.countries \| map_values(.technology)` | 4340 ms | 6.73x | n/a | yes | -| jq `empty` (parse the file, emit nothing) | 4227 ms | 6.55x | n/a | n/a | -| jq `--stream` | 17490 ms | 27.1x | n/a | n/a | -| jq process startup only | 4.2 ms | 0.01x | n/a | n/a | +| **SearchExpression (this project)** | **49.6 ms** | **1.00x** | **1.46 MB** | yes | +| .NET Regex balanced block + inner scan | 114.0 ms | 2.30x | 213 MB | yes | +| PCRE.NET path aware, JIT compiled | 123.4 ms | 2.49x | 0.90 MB | 1869/1870 | +| .NET Regex path aware, compiled | 135.1 ms | 2.72x | 1.34 MB | 1869/1870 | +| Utf8JsonReader over JSON twin | 155.5 ms | 3.13x | 16 KB | yes | +| JsonDocument over JSON twin | 323.2 ms | 6.51x | 72 B (native) | yes | +| .NET Regex path aware, NonBacktracking | 440.5 ms | 8.87x | 30.6 MB | 1869/1870 | +| Full parse, then select | 1204 ms | 24.3x | 962 MB | yes | +| jq `empty` (parse the file, emit nothing) | 4188 ms | 84.4x | n/a | n/a | +| jq `.countries \| map_values(.technology)` | 4339 ms | 87.4x | n/a | yes | +| jq `--stream` | 17497 ms | 352x | n/a | n/a | +| jq process startup only | 4.1 ms | 0.08x | n/a | n/a | + +Reading the save from disk instead of a preloaded `byte[]` costs the parser ~9 ms more: +58.4 ms via `FileStream` (`ParserInputBenchmarks`). ### Reading the table -* **jq is ~6.7x slower than this parser** and 97% of that time is JSON parsing, not the - query: `jq empty` on the same file costs 4227 ms of the 4340 ms. Process startup is - negligible (4 ms). jq's `--stream` mode, often recommended for large inputs, is 4x - *slower* still. jq also needs the data converted to JSON first, which this parser has to - do anyway — so end to end jq is strictly more expensive here. -* **The regexes are 5-40x faster, but they are not doing the same job.** A regex never - parses the structure; it scans bytes for a literal and validates a short window around it. - `Regex` with a literal prefix (`technology={`) is vectorized, so 109 MB is scanned at - several GB/s. That speed is real and the cost is real too: nothing verifies that the hit - is under `countries`, at the right depth, or belongs to the tag matched before it. -* **Path-aware regexes lose both the speed and the correctness.** Forcing the tag into the - pattern costs 7x (17 ms -> 116 ms) and still returns 1869 of 1870 countries: the tag class - `[A-Z0-9]{3}` silently drops EU4's `---` pseudo-country. Pairing survives here only by - luck of the file layout — all 933 tech-less country blocks happen to be grouped at the end - of the save, so the lazy gap never runs across one. A save that interleaves them would - make the regex report a technology block under the wrong country tag with no error. -* **`RegexOptions.NonBacktracking`** guarantees linear time but is 26x slower than the - compiled backtracking engine on this pattern and allocates 32 MB. -* **PCRE.NET** (the most used non-BCL regex engine in .NET, ~460k downloads) is 6x slower - than `System.Text.RegularExpressions` on the flat pattern, because .NET's vectorized - literal prefix search beats PCRE2's JIT here. On the path-aware pattern the two are equal. - There is no reason to leave the BCL engine for this workload. -* **The query is what makes this parser fast, not the parsing.** Same parser, same file: - 645 ms with `countries.*.technology`, 1831 ms and 1.1 GB allocated without a query. The - search expression prunes ~65% of the work and 99.9% of the allocations. -* **Against a JSON reader on equivalent data**, the parser is 4x slower than `Utf8JsonReader` - and 2x slower than `JsonDocument`. Those numbers exclude the pdx -> JSON conversion - (~4.7 s), so they are a ceiling for the format, not a usable alternative. +* **This parser is the fastest engine measured here.** It is 2.3x faster than the only + regex approach that enforces the path, 3.1x faster than a hand written `Utf8JsonReader` + over the equivalent JSON, 6.5x faster than `JsonDocument`, and 87x faster than jq — while + allocating 1.46 MB for a 109 MB input. +* **jq is ~87x slower and 97% of that is JSON parsing, not the query**: `jq empty` on the + same file costs 4188 ms of the 4339 ms. Process startup is negligible (4 ms). jq's + `--stream` mode, often recommended for large inputs, is 4x *slower* still. jq also cannot + read the Paradox format, so it first needs the save converted to JSON — a conversion this + parser has to perform anyway. +* **Only one regex approach here is actually correct**, and it is the expensive one: cut the + `countries={...}` block out with a balancing-group pattern (a .NET-only feature; PCRE would + need recursion), then scan inside it. It allocates 213 MB, because stage one materialises + the whole block as a string. +* **The cheaper path-aware pattern pairs a country tag with the next technology block over a + lazy gap, and silently gets it wrong**: 1869 of 1870 countries, because the tag class + `[A-Z0-9]{3}` drops EU4's `---` pseudo-country. Even that much only holds by luck of the + file layout — all 933 tech-less country blocks happen to be grouped at the end of the save, + so the lazy gap never runs across one. A save that interleaved them would report a + technology block under the wrong country tag, with no error. +* **`RegexOptions.NonBacktracking`** guarantees linear time but is 3.3x slower than the + compiled backtracking engine on this pattern and allocates 31 MB. +* **PCRE.NET** (the most used non-BCL regex engine in .NET, ~460k downloads) is within ~10% + of `System.Text.RegularExpressions` on the path-aware pattern (123 ms vs 135 ms). Nothing + here justifies leaving the BCL engine. +* **The query is what makes this parser fast.** Same parser, same file: 50 ms with + `countries.*.technology`, 1204 ms and 962 MB allocated without a query — 24x and 660x. -### Where this parser's time goes +### How the parser got here -645 ms for 109 MB is ~170 MB/s, or ~13 cycles per input byte. The lexer reads the save one -byte at a time through `Stream.ReadByte()` (`SaveParserEU4.LexTextSave`), which is a virtual -call plus bounds check per byte, and appends char by char into a `StringBuilder`. Reading -into a `byte[]` buffer and scanning it with `ReadOnlySpan.IndexOfAny` would be the -first thing to try — the regex numbers above show what the same hardware does when it scans -a span instead of a stream. +Three measurements on the same benchmark (`ParserInputBenchmarks`, reading a `FileStream`): + +| lexer | mean | allocated | +|---|---:|---:| +| `Stream.ReadByte()` per byte | 834 ms | 1.55 MB | +| copy the file into a `MemoryStream` first, still `ReadByte()` | 738 ms | 257 MB | +| 64 KB buffer + `Tokenizer` with byte-level block skipping | 58 ms | 1.46 MB | + +The first jump came from removing a virtual call per byte. The large one came from making +the query prune *work* rather than just data: `Tokenizer.SkipBlock` throws away a rejected +`{...}` block by scanning raw bytes for brace depth with `SearchValues.IndexOfAny`, +so blocks the search expression rejects are never turned into tokens or strings at all. +Retained tokens are spans into the read buffer, decoded only when their value is kept, and +numbers are parsed straight from UTF8. diff --git a/ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs b/ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs index a97c6ca..10ce9b5 100644 --- a/ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs +++ b/ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs @@ -1,4 +1,6 @@ +using System.Collections.Generic; using System.IO; +using System.Text; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -41,12 +43,64 @@ public class SearchExpressionTests [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")] public void TestSearchOnSmallData(string input, string expectedOutput) { - using var saveStream = new MemoryStream(_smallSaveData, false); - var se = SearchExpressionCompiler.Compile(input); - var parser = new SaveParserEU4(saveStream, se); + Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput)); + } + + /// + /// The same queries, but with a read buffer so small that tokens, quoted strings and + /// skipped blocks are split across refills. + /// + [TestCase("a", "a={ b={ c=0 d=1 e=2 } f=3 }")] + [TestCase("a.b", "a={ b={ c=0 d=1 e=2 } }")] + [TestCase("a.[1]", "a={ f=3 }")] + [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")] + public void TestSearchAcrossBufferRefills(string input, string expectedOutput) + { + foreach (int bufferSize in new[] { 16, 17, 23, 64 }) + Assert.That(Search(_smallSaveData, input, bufferSize), Is.EqualTo(expectedOutput), + $"buffer size {bufferSize}"); + } + + [Test] + public void BracesInsideQuotedStringAreText() + { + byte[] data = "EU4txt a={ name=\"x{y}z\" b=1 }".ToBytes(); + using var saveStream = new MemoryStream(data, false); + var parser = new SaveParserEU4(saveStream, SearchExpressionCompiler.Compile("a")); + var a = (Dictionary)parser.Parse()["a"]; + Assert.Multiple(() => + { + Assert.That(a["name"], Is.EqualTo("x{y}z")); + Assert.That(a["b"], Is.EqualTo(1L)); + }); + } + + [Test] + public void SkippedBlockIgnoresBracesInsideQuotedStrings() + { + byte[] data = "EU4txt a={ s=\"{{{\" } b=2".ToBytes(); + using var saveStream = new MemoryStream(data, false); + var parser = new SaveParserEU4(saveStream, SearchExpressionCompiler.Compile("b")); + Assert.That(parser.Parse()["b"], Is.EqualTo(2L)); + } + + [Test] + public void EncodingIsConfigurable() + { + byte[] data = Encoding.UTF8.GetBytes("EU4txt a={ name=\"Ä\" }"); + using var saveStream = new MemoryStream(data, false); + var parser = new SaveParserEU4(saveStream, SearchExpressionCompiler.Compile("a"), Encoding.UTF8); + var a = (Dictionary)parser.Parse()["a"]; + Assert.That(a["name"], Is.EqualTo("Ä")); + } + + private static string Search(byte[] saveData, string query, int bufferSize = 64 * 1024) + { + using var saveStream = new MemoryStream(saveData, false); + var se = SearchExpressionCompiler.Compile(query); + var parser = new SaveParserEU4(saveStream, se, bufferSize: bufferSize); var rootNode = parser.Parse(); string json = JsonSerializer.Serialize(rootNode, _smallSaveSerializerOptions); - string pdx = JsonToPdx(json); - Assert.That(pdx, Is.EqualTo(expectedOutput)); + return JsonToPdx(json); } } \ No newline at end of file diff --git a/ParadoxSaveParser.Lib/BufferedEnumerator.cs b/ParadoxSaveParser.Lib/BufferedEnumerator.cs deleted file mode 100644 index fdabe3c..0000000 --- a/ParadoxSaveParser.Lib/BufferedEnumerator.cs +++ /dev/null @@ -1,121 +0,0 @@ -using System.Collections; - -namespace ParadoxSaveParser.Lib; - -/// -/// Enumerator wrapper that stores N/2 items before and N/2-1 after Current item. -/// -/// -/// IEnumerator<int> Enumerator() -/// { -/// for(int i = 0; i < 6; i++) -/// yield return i; -/// } -/// -/// var en = Enumerator(); -/// var bufen = new BufferedEnumerator<int>(en, 5); -/// -/// while(bufen.MoveNext()) -/// { -/// var cur = bufen.Current; -/// for (var prev = cur.List?.First; prev != cur; prev = prev?.Next) -/// Console.Write($"{prev?.Value} "); -/// -/// Console.Write($"| {cur.Value} |"); -/// -/// for (var next = cur.Next; next is not null; next = next.Next) -/// Console.Write($" {next.Value}"); -/// Console.WriteLine(); -/// } -/// -/// Output: -/// -/// | 0 | 1 2 3 4 -/// 0 | 1 | 2 3 4 -/// 0 1 | 2 | 3 4 -/// 1 2 | 3 | 4 5 -/// 2 3 | 4 | 5 -/// 3 4 | 5 | -/// -public class BufferedEnumerator : IEnumerator.Node> -{ - public class Node - { -#nullable disable - public Node Previous; - public Node Next; - public T Value; -#nullable enable - } - - private readonly IEnumerator _enumerator; - private readonly Node[] _ringBuffer; - private Node? _currentNode; - private int _currentBufferIndex = -1; - private int _lastValueIndex = -1; - - public BufferedEnumerator(IEnumerator enumerator, int bufferSize) - { - _enumerator = enumerator; - _ringBuffer = new Node[bufferSize]; - } - - private void InitBuffer() - { - _ringBuffer[0] = new Node - { - Value = default! - }; - for (int i = 1; i < _ringBuffer.Length; i++) - { - _ringBuffer[i] = new Node - { - Previous = _ringBuffer[i - 1], - Value = default!, - }; - _ringBuffer[i - 1].Next = _ringBuffer[i]; - } - _ringBuffer[^1].Next = _ringBuffer[0]; - _ringBuffer[0].Previous = _ringBuffer[^1]; - } - - public bool MoveNext() - { - if (_currentBufferIndex == -1) - { - InitBuffer(); - - int beforeMidpoint = _ringBuffer.Length / 2 - 1; - for (int i = 0; i <= beforeMidpoint && _enumerator.MoveNext(); i++) - { - _ringBuffer[i].Value = _enumerator.Current; - } - } - - _currentBufferIndex = (_currentBufferIndex + 1) % _ringBuffer.Length; - if (_enumerator.MoveNext()) - { - int midpoint = (_currentBufferIndex + _ringBuffer.Length / 2) % _ringBuffer.Length; - _ringBuffer[midpoint].Value = _enumerator.Current; - _lastValueIndex = midpoint; - } - if(_currentBufferIndex == (_lastValueIndex + 1) % _ringBuffer.Length) - return false; - - _currentNode = _ringBuffer[_currentBufferIndex]; - return true; - } - - public void Reset() - { - throw new NotImplementedException(); - } - - public Node Current => _currentNode!; - - object IEnumerator.Current => Current; - - public void Dispose() - { - } -} \ No newline at end of file diff --git a/ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj b/ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj index 62f6358..261b6f5 100644 --- a/ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj +++ b/ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj @@ -6,8 +6,8 @@ disable true - + - + diff --git a/ParadoxSaveParser.Lib/SaveParserEU4.cs b/ParadoxSaveParser.Lib/SaveParserEU4.cs index 2c1f431..f2dd1f0 100644 --- a/ParadoxSaveParser.Lib/SaveParserEU4.cs +++ b/ParadoxSaveParser.Lib/SaveParserEU4.cs @@ -1,430 +1,261 @@ -global using System; -global using System.Collections.Generic; -global using System.IO; -global using System.Text; -using Microsoft.Extensions.ObjectPool; - -namespace ParadoxSaveParser.Lib; - -/// -/// Sequential parser that doesn't cache anything. -/// -public class SaveParserEU4 -{ - protected readonly Stream _saveFile; - private readonly BufferedEnumerator _tokens; - private readonly ObjectPool _stringBuilderPool; - private ISearchExpression? _searchExprCurrent; - - /// - /// Uncompressed stream of gamestate file which can be extracted from save archive - /// - /// - /// Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM, - /// so you should specify what exactly you want to get from save file - /// - public SaveParserEU4(Stream savefile, ISearchExpression? query) - { - _saveFile = savefile; - _searchExprCurrent = query; - const int tokenBufSize = 5; - _tokens = new BufferedEnumerator(LexTextSave(), tokenBufSize); - _stringBuilderPool = new DefaultObjectPool( - new StringBuilderPooledObjectPolicy - { - InitialCapacity = tokenBufSize * 13, - MaximumRetainedCapacity = tokenBufSize * 13, - }); - } - - protected IEnumerator LexTextSave() - { - string expectedHeader = "EU4txt"; - byte[] headBytes = new byte[expectedHeader.Length]; - _saveFile.ReadExactly(headBytes); - string headStr = Encoding.UTF8.GetString(headBytes); - if (headStr != expectedHeader) - throw new Exception($"Invalid gamestate header. Expected '{expectedHeader}', got '{headStr}'."); - - StringBuilder strb = _stringBuilderPool.Get(); - int line = 2; - int column = 0; - bool isQuoteOpen = false; - bool isStrInQuotes = false; - Token strToken = new() - { - type = TokenType.Invalid, - column = -1, - line = -1, - value = null, - }; - - bool TryCompleteStringToken() - { - if (isQuoteOpen) - return false; - - // strings in quotes may be empty - if (!isStrInQuotes && (strb.Length <= 0 || strb[0] == '#')) - return false; - - strToken = new Token - { - type = TokenType.StringOrNumber, - column = (short)(column - strb.Length), - line = line, - value = strb, - }; - strb = _stringBuilderPool.Get(); - isStrInQuotes = false; - return true; - } - - // Reading the save one byte at a time through Stream.ReadByte() costs a virtual call - // per byte, which dominated the parsing time. Bytes are pulled into this buffer - // instead, so the stream is touched once per 64 KB and the inner loop reads an array. - byte[] buffer = new byte[64 * 1024]; - int bufferLength; - while ((bufferLength = _saveFile.Read(buffer, 0, buffer.Length)) > 0) - { - for (int i = 0; i < bufferLength; i++) - { - int c = buffer[i]; - column++; - switch (c) - { - case '\"': - isQuoteOpen = !isQuoteOpen; - isStrInQuotes = true; - break; - case ' ': - case '\t': - case '\r': - if (TryCompleteStringToken()) - yield return strToken; - break; - case '\n': - if (TryCompleteStringToken()) - yield return strToken; - line++; - column = 0; - break; - case '=': - if (TryCompleteStringToken()) - yield return strToken; - yield return new Token - { - type = TokenType.Equals, - line = line, column = (short)column - }; - break; - case '{': - if (TryCompleteStringToken()) - yield return strToken; - yield return new Token - { - type = TokenType.BracketOpen, - line = line, column = (short)column - }; - break; - case '}': - if (TryCompleteStringToken()) - yield return strToken; - yield return new Token - { - type = TokenType.BracketClose, - line = line, column = (short)column - }; - break; - default: - // Skip control characters, which are invisible and causing frontend bugs. - // I dont know why there are so many of them in strings. - if (c >= 0x20) - strb.Append((char)c); - break; - } - } - } - - // end of file: the last token may still be unterminated - if (TryCompleteStringToken()) - yield return strToken; - _stringBuilderPool.Return(strb); - } - - - // doesn't move next - private object? ParseValue() - { - var tok = _tokens.Current.Value; - switch (tok.type) - { - case TokenType.StringOrNumber: - try - { - // string values can be empty - if (tok.value!.Length == 0) - return string.Empty; - if (tok.value.Equals("yes")) - return true; - if (tok.value.Equals("no")) - return false; - - string tokStr = tok.value.ToString(); - if (tokStr[0] != '-' && !char.IsDigit(tokStr[0])) - return tokStr; - if (tokStr.Contains('.') && double.TryParse(tokStr, out double d)) - return d; - if (long.TryParse(tokStr, out long l)) - return l; - return tokStr; - } - finally - { - _stringBuilderPool.Return(tok.value!); - } - case TokenType.BracketOpen: - object obj = ParseListOrDict(); - return obj; - case TokenType.BracketClose: - return null; - default: - throw new UnexpectedTokenException(tok); - } - } - - - // skips next value - /// true if skipped value, false if current token is closing bracket - private bool SkipValue() - { - var tok = _tokens.Current.Value; - switch (tok.type) - { - case TokenType.BracketOpen: - SkipObject(); - return true; - case TokenType.StringOrNumber: - _stringBuilderPool.Return(tok.value!); - return true; - case TokenType.BracketClose: - return false; - default: - throw new UnexpectedTokenException(tok); - } - } - - // skips all tokens inside curly braces block - private void SkipObject(int bracketBalance = 1) - { - while (bracketBalance != 0 && _tokens.MoveNext()) - { - var tok = _tokens.Current.Value; - if (tok.type == TokenType.BracketOpen) - bracketBalance++; - else if (tok.type == TokenType.BracketClose) - bracketBalance--; - else if (tok.type == TokenType.StringOrNumber) - { - _stringBuilderPool.Return(tok.value!); - } - } - } - - private static bool IsEmptyCollection(object value) - => value is Dictionary { Count: 0 } or List { Count: 0 }; - - // doesn't move next - private object ParseListOrDict() - { - var first = _tokens.Current.Next; - var second = _tokens.Current.Next?.Next; - if (first?.Value.type == TokenType.StringOrNumber && second?.Value.type == TokenType.Equals) - return ParseDict(); - - return ParseList(); - } - - // moves next - private List ParseList() - { - List list = new(); - for (int i = 0; ; i++) - { - if (!_tokens.MoveNext()) - throw new Exception("Unexpected end of file"); - - ISearchExpression? searchExprNext = null; - if (_searchExprCurrent != null - && !_searchExprCurrent.DoesMatch(new SearchArgs(i, string.Empty), out searchExprNext)) - { - if(!SkipValue()) - break; - continue; - } - var searchExprPrev = _searchExprCurrent; - _searchExprCurrent = searchExprNext; - object? value = ParseValue(); - _searchExprCurrent = searchExprPrev; - if (value is null) - break; - - // do dot add empty collections into list - if (IsEmptyCollection(value)) - continue; - - list.Add(value); - } - - return list; - } - - // moves next - private Dictionary ParseDict() - { - Dictionary dict = new(); - - // root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count - for (int localIndex = 0; _tokens.MoveNext(); localIndex++) - { - var tok = _tokens.Current.Value; - // end of dictionary - if (tok.type == TokenType.BracketClose) - break; - - // Saves may contain some blocks without key. - // Such blocks are skipped because idk where to put them. - // Example: `technology_group=tech_cannorian{ } - // { } { } { }` - if (tok.type == TokenType.BracketOpen) - { - SkipObject(); - continue; - } - - if (tok.type != TokenType.StringOrNumber) - throw new UnexpectedTokenException(tok); - - var keySB = tok.value!; - - // next token should be `=` or `{` - if (!_tokens.MoveNext()) - throw new UnexpectedTokenException(tok); - tok = _tokens.Current.Value; - if (tok.type == TokenType.Equals) - { - // skip `=` - if (!_tokens.MoveNext()) - throw new UnexpectedTokenException(tok); - } - // Saves may contain object definition without `=`. - // Example: `map_area_data {` instead of `map_area_data = {` - else if (tok.type != TokenType.BracketOpen) - { - throw new UnexpectedTokenException(tok); - } - - ISearchExpression? searchExprNext = null; - if (_searchExprCurrent != null - && !_searchExprCurrent.DoesMatch(new SearchArgs(localIndex, keySB), out searchExprNext)) - { - if(!SkipValue()) - throw new UnexpectedTokenException(_tokens.Current.Value); - _stringBuilderPool.Return(keySB); - continue; - } - - var searExpressionPrevious = _searchExprCurrent; - _searchExprCurrent = searchExprNext; - object? value = ParseValue(); - if (value is null) - throw new UnexpectedTokenException(_tokens.Current.Value); - _searchExprCurrent = searExpressionPrevious; - - string keyStr = keySB.ToString(); - _stringBuilderPool.Return(keySB); - - // Paradox save format has another way of defining list: - // a = 1 - // a = 2 - // It means `a = { 1 2 }` - if (dict.TryGetValue(keyStr, out var firstValue)) - { - // Do dot add empty collections into list. - // `key:{}` is okay, but i don't want to see `key:[{},{},{},{},{},{}]` - if (IsEmptyCollection(value)) - continue; - - if (firstValue is List existingList) - existingList.Add(value); - else dict[keyStr] = new List { firstValue, value }; - } - else - { - dict.Add(keyStr, value); - } - } - - return dict; - } - - public Dictionary Parse() - { - var root = ParseDict(); - return root; - } - - protected enum TokenType : byte - { - Invalid, - StringOrNumber, - Equals, - BracketOpen, - BracketClose - } - - protected struct Token - { - public required TokenType type; - public required short column; - public required int line; - public StringBuilder? value; - - public override string ToString() - { - string s; - switch (type) - { - case TokenType.Invalid: - s = "INVALID_TOKEN"; - break; - case TokenType.StringOrNumber: - if (value == null || value.Length == 0) - s = "NULL"; - else s = value.ToString(); - break; - case TokenType.Equals: - s = "="; - break; - case TokenType.BracketOpen: - s = "{"; - break; - case TokenType.BracketClose: - s = "}"; - break; - default: - throw new ArgumentOutOfRangeException(type.ToString()); - } - - return $"{line}:{column} '{s}'"; - } - } - - protected class UnexpectedTokenException : Exception - { - public UnexpectedTokenException(Token token) : - base($"Unexpected token: {token}") - { - } - } -} \ No newline at end of file +global using System; +global using System.Collections.Generic; +global using System.IO; +global using System.Text; +using System.Globalization; + +namespace ParadoxSaveParser.Lib; + +/// +/// Sequential parser that doesn't cache anything. +/// +public class SaveParserEU4 +{ + private const int DefaultBufferSize = 64 * 1024; + private static ReadOnlySpan Header => "EU4txt"u8; + + private readonly Tokenizer _tokens; + private ISearchExpression? _searchExprCurrent; + + /// + /// Encoding of the strings inside the save. Saves of different localizations use + /// different ones, so it can be changed; maps + /// every byte to one character and never fails, which makes it a safe default. + /// + public Encoding Encoding { get; } + + /// + /// Uncompressed stream of gamestate file which can be extracted from save archive + /// + /// + /// Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM, + /// so you should specify what exactly you want to get from save file + /// + /// Encoding of the strings inside the save. Latin1 by default. + /// Size of the read buffer. Mostly useful for tests. + public SaveParserEU4(Stream savefile, ISearchExpression? query, + Encoding? encoding = null, int bufferSize = DefaultBufferSize) + { + Encoding = encoding ?? Encoding.Latin1; + _searchExprCurrent = query; + _tokens = new Tokenizer(savefile, bufferSize); + } + + public Dictionary Parse() + { + _tokens.ReadHeader(Header, Encoding); + return ParseDict(); + } + + // doesn't move next + private object? ParseValue() + { + switch (_tokens.Type) + { + case TokenType.StringOrNumber: + return ParseScalar(_tokens.Text); + case TokenType.BracketOpen: + return ParseListOrDict(); + case TokenType.BracketClose: + return null; + default: + throw new UnexpectedTokenException(_tokens, Encoding); + } + } + + private object ParseScalar(ReadOnlySpan text) + { + // string values can be empty + if (text.Length == 0) + return string.Empty; + if (text.SequenceEqual("yes"u8)) + return true; + if (text.SequenceEqual("no"u8)) + return false; + + byte first = text[0]; + if (first != (byte)'-' && !char.IsAsciiDigit((char)first)) + return DecodeString(text); + if (text.Contains((byte)'.') + && double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out double d)) + return d; + if (long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out long l)) + return l; + return DecodeString(text); + } + + private string DecodeString(ReadOnlySpan text) + { + // Skip control characters, which are invisible and causing frontend bugs. + // I dont know why there are so many of them in strings. + if (!text.ContainsAnyInRange((byte)0, (byte)0x1F)) + return Encoding.GetString(text); + + Span cleaned = text.Length <= 256 ? stackalloc byte[text.Length] : new byte[text.Length]; + int length = 0; + foreach (byte b in text) + if (b >= 0x20) + cleaned[length++] = b; + return Encoding.GetString(cleaned[..length]); + } + + // skips next value + /// true if skipped value, false if current token is closing bracket + private bool SkipValue() + { + switch (_tokens.Type) + { + case TokenType.BracketOpen: + _tokens.SkipBlock(); + return true; + case TokenType.StringOrNumber: + return true; + case TokenType.BracketClose: + return false; + default: + throw new UnexpectedTokenException(_tokens, Encoding); + } + } + + private static bool IsEmptyCollection(object value) + => value is Dictionary { Count: 0 } or List { Count: 0 }; + + // doesn't move next + private object ParseListOrDict() + { + if (_tokens.PeekType(1) == TokenType.StringOrNumber && _tokens.PeekType(2) == TokenType.Equals) + return ParseDict(); + + return ParseList(); + } + + // moves next + private List ParseList() + { + List list = new(); + for (int i = 0; ; i++) + { + if (!_tokens.Read()) + throw new Exception("Unexpected end of file"); + + ISearchExpression? searchExprNext = null; + if (_searchExprCurrent != null + && !_searchExprCurrent.DoesMatch(new MatchCandidate(i), out searchExprNext)) + { + if (!SkipValue()) + break; + continue; + } + + var searchExprPrev = _searchExprCurrent; + _searchExprCurrent = searchExprNext; + object? value = ParseValue(); + _searchExprCurrent = searchExprPrev; + if (value is null) + break; + + // do dot add empty collections into list + if (IsEmptyCollection(value)) + continue; + + list.Add(value); + } + + return list; + } + + // moves next + private Dictionary ParseDict() + { + Dictionary dict = new(); + + // root is a dict without closing bracket, so this method must check for end of file + for (int localIndex = 0; _tokens.Read(); localIndex++) + { + // end of dictionary + if (_tokens.Type == TokenType.BracketClose) + break; + + // Saves may contain some blocks without key. + // Such blocks are skipped because idk where to put them. + // Example: `technology_group=tech_cannorian{ } + // { } { } { }` + if (_tokens.Type == TokenType.BracketOpen) + { + _tokens.SkipBlock(); + continue; + } + + if (_tokens.Type != TokenType.StringOrNumber) + throw new UnexpectedTokenException(_tokens, Encoding); + + // The key is matched before the value is read, so that a key rejected by the query + // never has to become a string. + ISearchExpression? searchExprNext = null; + bool matches = _searchExprCurrent == null + || _searchExprCurrent.DoesMatch( + new MatchCandidate(localIndex, _tokens.Text, Encoding), out searchExprNext); + string? keyStr = matches ? DecodeString(_tokens.Text) : null; + + // next token should be `=` or `{` + if (!_tokens.Read()) + throw new UnexpectedTokenException(_tokens, Encoding); + if (_tokens.Type == TokenType.Equals) + { + // skip `=` + if (!_tokens.Read()) + throw new UnexpectedTokenException(_tokens, Encoding); + } + // Saves may contain object definition without `=`. + // Example: `map_area_data {` instead of `map_area_data = {` + else if (_tokens.Type != TokenType.BracketOpen) + { + throw new UnexpectedTokenException(_tokens, Encoding); + } + + if (!matches) + { + if (!SkipValue()) + throw new UnexpectedTokenException(_tokens, Encoding); + continue; + } + + var searExpressionPrevious = _searchExprCurrent; + _searchExprCurrent = searchExprNext; + object? value = ParseValue(); + if (value is null) + throw new UnexpectedTokenException(_tokens, Encoding); + _searchExprCurrent = searExpressionPrevious; + + // Paradox save format has another way of defining list: + // a = 1 + // a = 2 + // It means `a = { 1 2 }` + if (dict.TryGetValue(keyStr!, out var firstValue)) + { + // Do dot add empty collections into list. + // `key:{}` is okay, but i don't want to see `key:[{},{},{},{},{},{}]` + if (IsEmptyCollection(value)) + continue; + + if (firstValue is List existingList) + existingList.Add(value); + else dict[keyStr!] = new List { firstValue, value }; + } + else + { + dict.Add(keyStr!, value); + } + } + + return dict; + } + + internal class UnexpectedTokenException : Exception + { + public UnexpectedTokenException(Tokenizer tokens, Encoding encoding) : + base($"Unexpected token: {tokens.Line}:{tokens.Column} '{tokens.Describe(encoding)}'") + { + } + } +} diff --git a/ParadoxSaveParser.Lib/SearchExpression.cs b/ParadoxSaveParser.Lib/SearchExpression.cs index 54c8ac9..ef1a55b 100644 --- a/ParadoxSaveParser.Lib/SearchExpression.cs +++ b/ParadoxSaveParser.Lib/SearchExpression.cs @@ -1,29 +1,37 @@ namespace ParadoxSaveParser.Lib; -public readonly record struct SearchArgs +/// +/// The node a search expression is tested against: its key inside the parent dictionary, +/// still in the raw bytes of the save, and its position inside the parent list. +/// Keys stay undecoded so that a key rejected by the query never becomes a string. +/// +public readonly ref struct MatchCandidate { - public readonly string KeyStr; - public readonly StringBuilder? KeySB; - public readonly int LocalIndex; + public readonly ReadOnlySpan Key; + public readonly int Index; - public SearchArgs(int localIndex, string keyStr) + /// Encoding is written in. + public readonly Encoding Encoding; + + /// A list item, which has a position but no key. + public MatchCandidate(int index) { - KeyStr = keyStr; - KeySB = null; - LocalIndex = localIndex; + Key = default; + Index = index; + Encoding = Encoding.Latin1; } - - public SearchArgs(int localIndex, StringBuilder keySb) + + public MatchCandidate(int index, ReadOnlySpan key, Encoding encoding) { - KeyStr = string.Empty; - KeySB = keySb; - LocalIndex = localIndex; + Key = key; + Index = index; + Encoding = encoding; } } public interface ISearchExpression { - bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression); + bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression); } public static class SearchExpressionCompiler @@ -108,7 +116,7 @@ public static class SearchExpressionCompiler private record AnyMatchExpression(ISearchExpression? next) : ISearchExpression { - public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression) + public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) { nextSearchExpression = next; return true; @@ -117,7 +125,7 @@ public static class SearchExpressionCompiler private record NoMatchExpression : ISearchExpression { - public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression) + public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) { nextSearchExpression = null; return false; @@ -126,10 +134,10 @@ public static class SearchExpressionCompiler private record MultipleMatchExpression(List subExprs) : ISearchExpression { - public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression) + public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) { foreach (var e in subExprs) - if (e.DoesMatch(args, out nextSearchExpression)) + if (e.DoesMatch(candidate, out nextSearchExpression)) return true; nextSearchExpression = null; @@ -139,9 +147,9 @@ public static class SearchExpressionCompiler private record IndexMatchExpression(int index, ISearchExpression? next) : ISearchExpression { - public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression) + public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) { - if (args.LocalIndex == index) + if (candidate.Index == index) { nextSearchExpression = next; return true; @@ -154,9 +162,20 @@ public static class SearchExpressionCompiler private record ExactMatchExpression(string key, ISearchExpression? next) : ISearchExpression { - public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression) + // the key is compared as bytes, so it is encoded once for whatever encoding the + // parser reads the save in + private byte[]? _keyBytes; + private Encoding? _keyEncoding; + + public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) { - if ((args.KeySB != null && args.KeySB.Equals(key)) || args.KeyStr == key) + if (!ReferenceEquals(_keyEncoding, candidate.Encoding)) + { + _keyEncoding = candidate.Encoding; + _keyBytes = candidate.Encoding.GetBytes(key); + } + + if (candidate.Key.SequenceEqual(_keyBytes)) { nextSearchExpression = next; return true; diff --git a/ParadoxSaveParser.Lib/Tokenizer.cs b/ParadoxSaveParser.Lib/Tokenizer.cs new file mode 100644 index 0000000..49d3f3e --- /dev/null +++ b/ParadoxSaveParser.Lib/Tokenizer.cs @@ -0,0 +1,456 @@ +using System.Buffers; + +namespace ParadoxSaveParser.Lib; + +internal enum TokenType : byte +{ + // default value, so a slot that was never scanned is recognizably empty + Invalid, + // any bare or quoted value: the tokenizer does not tell numbers from strings + StringOrNumber, + Equals, + BracketOpen, + BracketClose, + EndOfFile +} + +/// +/// Splits a Paradox text save into tokens, reading the stream through a reusable buffer. +/// Token text is exposed as a span into that buffer, so a token costs no allocation at all +/// unless it happens to straddle a buffer refill. +/// +/// The parser can tell the tokenizer to throw away a whole {...} block with +/// . Blocks rejected by the search expression are then never +/// turned into tokens or strings, which is what makes a query cheaper than a full parse. +/// +/// +internal sealed class Tokenizer +{ + /// Bytes that end an unquoted token. + private static readonly SearchValues TokenEnd = SearchValues.Create(" \t\r\n={}\""u8); + + private static readonly SearchValues Whitespace = SearchValues.Create(" \t\r\n"u8); + + /// Everything has to look at while counting depth. + private static readonly SearchValues BlockChars = SearchValues.Create("{}\""u8); + + /// One scanned token. Reused forever, so scanning a token allocates nothing. + private sealed class Slot + { + public TokenType Type; + // position of the token's first byte in the file, for error messages + public int Line; + public short Column; + + // text is either a range of the shared buffer, or, once a refill would overwrite it, + // a copy in Scratch + public int Start; + public int Length; + // grown on demand and kept between tokens, so long tokens stop reallocating after a while + public byte[] Scratch = []; + public bool InScratch; + } + + private readonly Stream _stream; + private readonly byte[] _buffer; + + // current token plus up to two lookahead tokens + private readonly Slot[] _slots = [new Slot(), new Slot(), new Slot()]; + // index of the current token; the slots are used as a ring, so the array is never shifted + private int _current; + // how many slots after _current already hold a scanned, not yet consumed token + private int _lookahead; + + // read position and amount of valid data in _buffer + private int _pos; + private int _length; + // set once the stream has no more bytes; stops Fill() from calling Read() again + private bool _eof; + // position of _pos in the file; _column is 0-based here and reported 1-based + private int _line = 1; + private int _column; + + public Tokenizer(Stream stream, int bufferSize) + { + if (bufferSize < 16) + throw new ArgumentOutOfRangeException(nameof(bufferSize), bufferSize, "buffer is too small"); + _stream = stream; + _buffer = new byte[bufferSize]; + } + + public TokenType Type => _slots[_current].Type; + public int Line => _slots[_current].Line; + public short Column => _slots[_current].Column; + + /// + /// Bytes of the current token, without quotes. Valid only until the next + /// , because the buffer underneath it gets reused. + /// + public ReadOnlySpan Text + { + get + { + var slot = _slots[_current]; + // a token that survived a refill was copied out; everything else still points into the buffer + return slot.InScratch + ? slot.Scratch.AsSpan(0, slot.Length) + : _buffer.AsSpan(slot.Start, slot.Length); + } + } + + /// Consumes the file's magic header and checks it, before any token is scanned. + public void ReadHeader(ReadOnlySpan expected, Encoding encoding) + { + // read straight from the stream: this runs before the buffer holds anything + Span head = stackalloc byte[expected.Length]; + _stream.ReadExactly(head); + if (!head.SequenceEqual(expected)) + throw new Exception($"Invalid gamestate header. " + + $"Expected '{encoding.GetString(expected)}', got '{encoding.GetString(head)}'."); + } + + /// false at end of file + public bool Read() + { + _current = NextSlot(_current); + // the next slot may already be filled by an earlier PeekType, then there is nothing to scan + if (_lookahead > 0) + _lookahead--; + else Scan(_slots[_current]); + return _slots[_current].Type != TokenType.EndOfFile; + } + + /// + /// Type of a token that has not been consumed yet, 1 or 2 tokens ahead of the current one. + /// Only types are available: the parser never needs the text of a token it has not reached. + /// + public TokenType PeekType(int offset) + { + // scan only as far as asked, so lookahead never runs into a block SkipBlock is about to drop + while (_lookahead < offset) + { + // first slot after the ones that are already filled + int slot = _current; + for (int i = 0; i <= _lookahead; i++) + slot = NextSlot(slot); + Scan(_slots[slot]); + _lookahead++; + } + + int index = _current; + for (int i = 0; i < offset; i++) + index = NextSlot(index); + return _slots[index].Type; + } + + /// + /// Throws away the block opened by the current { token without tokenizing it: + /// raw bytes are scanned for braces until the depth returns to zero. + /// Braces inside quoted strings are text and do not change the depth. + /// Leaves the closing } as the current token. + /// + public void SkipBlock() + { + int depth = 1; + + // tokens that lookahead already pulled out of the buffer still count towards the depth + while (depth > 0 && _lookahead > 0) + { + _current = NextSlot(_current); + _lookahead--; + switch (_slots[_current].Type) + { + case TokenType.BracketOpen: + depth++; + break; + case TokenType.BracketClose: + depth--; + break; + case TokenType.EndOfFile: + return; + } + } + + bool inQuotes = false; + while (depth > 0) + { + if (_pos >= _length && !Fill()) + break; // unbalanced braces: the file ended inside the block + + var span = _buffer.AsSpan(_pos, _length - _pos); + // inside a string only the closing quote matters, braces there are ordinary characters + int i = inQuotes ? span.IndexOf((byte)'\"') : span.IndexOfAny(BlockChars); + if (i < 0) + { + // nothing interesting in this bufferful, drop all of it and refill + Consume(span.Length); + continue; + } + + byte b = span[i]; + Consume(i + 1); + if (b == (byte)'\"') + inQuotes = !inQuotes; + else if (b == (byte)'{') + depth++; + else depth--; + } + + // hand the parser the closing brace it expects, without having tokenized anything inside + var current = _slots[_current]; + current.Type = depth == 0 ? TokenType.BracketClose : TokenType.EndOfFile; + current.Length = 0; + current.InScratch = false; + current.Line = _line; + current.Column = (short)_column; + } + + public string Describe(Encoding encoding) => Type switch + { + TokenType.StringOrNumber => Text.Length == 0 ? "NULL" : encoding.GetString(Text), + TokenType.Equals => "=", + TokenType.BracketOpen => "{", + TokenType.BracketClose => "}", + TokenType.EndOfFile => "END_OF_FILE", + _ => "INVALID_TOKEN", + }; + + /// Slot indices wrap around: the three slots form a ring buffer. + private int NextSlot(int i) => i + 1 == _slots.Length ? 0 : i + 1; + + /// Reads the next token from the stream into . + private void Scan(Slot slot) + { + // loops only to skip comments, which produce no token + while (true) + { + if (!SkipWhitespace()) + { + slot.Type = TokenType.EndOfFile; + slot.Length = 0; + slot.InScratch = false; + slot.Line = _line; + slot.Column = (short)_column; + return; + } + + slot.Line = _line; + slot.Column = (short)(_column + 1); // columns are reported 1-based + // SkipWhitespace guarantees at least one buffered byte here + byte b = _buffer[_pos]; + switch (b) + { + case (byte)'=': + ConsumeFlat(1); + SetDelimiter(slot, TokenType.Equals); + return; + case (byte)'{': + ConsumeFlat(1); + SetDelimiter(slot, TokenType.BracketOpen); + return; + case (byte)'}': + ConsumeFlat(1); + SetDelimiter(slot, TokenType.BracketClose); + return; + case (byte)'\"': + ReadQuoted(slot); + return; + default: + ReadBare(slot); + // comments are dropped, same as before: a token starting with '#' is not emitted + if (slot.Length > 0 && FirstByte(slot) == (byte)'#') + continue; + return; + } + } + } + + /// Single-character tokens carry no text of their own. + private static void SetDelimiter(Slot slot, TokenType type) + { + slot.Type = type; + slot.Length = 0; + slot.InScratch = false; + } + + /// Reads an unquoted token, which ends at the first delimiter byte. + private void ReadBare(Slot slot) + { + StartText(slot); + while (true) + { + var span = _buffer.AsSpan(_pos, _length - _pos); + // one vectorized search replaces a loop over the token's bytes + int end = span.IndexOfAny(TokenEnd); + if (end >= 0) + { + // the delimiter itself is left for the next Scan to classify + AppendText(slot, span[..end]); + ConsumeFlat(end); + return; + } + + // token runs to the end of the buffer and continues in the next one + AppendText(slot, span); + ConsumeFlat(span.Length); + if (!Fill()) + return; + } + } + + /// + /// Reads a quoted token. Everything up to the next " is text, braces included; + /// the format has no escape sequences, so a quote always closes the string. + /// + private void ReadQuoted(Slot slot) + { + ConsumeFlat(1); // opening quote + StartText(slot); + while (true) + { + if (_pos >= _length && !Fill()) + return; // unterminated string at end of file + + var span = _buffer.AsSpan(_pos, _length - _pos); + int end = span.IndexOf((byte)'\"'); + if (end >= 0) + { + AppendText(slot, span[..end]); + Consume(end + 1); // content plus the closing quote + return; + } + + // Consume, not ConsumeFlat: a quoted value is allowed to span several lines + AppendText(slot, span); + Consume(span.Length); + } + } + + /// Begins a text token at the current read position. + private void StartText(Slot slot) + { + slot.Type = TokenType.StringOrNumber; + slot.Start = _pos; + slot.Length = 0; + slot.InScratch = false; + } + + /// Extends a text token by one chunk of bytes taken from the buffer. + private void AppendText(Slot slot, ReadOnlySpan chunk) + { + if (!slot.InScratch) + { + // chunks always continue where the previous one ended, so the range just grows + // still contiguous in the buffer, nothing to copy + if (slot.Length == 0) + slot.Start = _pos; + slot.Length += chunk.Length; + return; + } + + // the start of this token is already out of the buffer, so the rest has to follow it + EnsureScratch(slot, slot.Length + chunk.Length); + chunk.CopyTo(slot.Scratch.AsSpan(slot.Length)); + slot.Length += chunk.Length; + } + + /// First byte of a token, wherever its text currently lives. Used to spot comments. + private byte FirstByte(Slot slot) => slot.InScratch ? slot.Scratch[0] : _buffer[slot.Start]; + + private static void EnsureScratch(Slot slot, int size) + { + if (slot.Scratch.Length >= size) + return; + // doubling keeps a token that is appended chunk by chunk from resizing on every chunk + int capacity = Math.Max(size, slot.Scratch.Length * 2); + Array.Resize(ref slot.Scratch, capacity); + } + + /// Moves the read position onto the next non-whitespace byte. + /// false at end of file + private bool SkipWhitespace() + { + while (true) + { + if (_pos >= _length && !Fill()) + return false; + + var span = _buffer.AsSpan(_pos, _length - _pos); + int i = span.IndexOfAnyExcept(Whitespace); + if (i < 0) + { + // whitespace to the end of the buffer, keep going in the next one + Consume(span.Length); + continue; + } + + if (i > 0) + Consume(i); + return true; + } + } + + /// Refills the buffer from the stream. + /// false if the stream is exhausted + private bool Fill() + { + if (_eof) + return false; + + // text of live tokens lives in the buffer, so it has to be copied out before overwriting + MaterializeSlots(); + // a short read is fine, the next Fill picks up the rest + _length = _stream.Read(_buffer, 0, _buffer.Length); + _pos = 0; + if (_length > 0) + return true; + + _eof = true; + _length = 0; + return false; + } + + /// + /// Copies every token that still points into the buffer out to its own scratch array. + /// Called just before a refill, which is the only moment that text can be lost. + /// + private void MaterializeSlots() + { + foreach (var slot in _slots) + { + // delimiters and empty tokens own no text, and a copied one needs no second copy + if (slot.InScratch || slot.Length == 0 || slot.Type != TokenType.StringOrNumber) + continue; + EnsureScratch(slot, slot.Length); + _buffer.AsSpan(slot.Start, slot.Length).CopyTo(slot.Scratch); + slot.InScratch = true; + } + } + + /// Consumes bytes that cannot contain a line break. + private void ConsumeFlat(int count) + { + _pos += count; + _column += count; + } + + /// Consumes bytes that may contain line breaks, keeping line and column exact. + private void Consume(int count) + { + var slice = _buffer.AsSpan(_pos, count); + // the last break decides the column, and only then is counting all of them worth it + int lastBreak = slice.LastIndexOf((byte)'\n'); + if (lastBreak < 0) + { + _column += count; + } + else + { + _line += slice.Count((byte)'\n'); + // bytes left after the final break + _column = count - lastBreak - 1; + } + + _pos += count; + } +}