Timerix/ParadoxSaveParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parser rewrite

Browse Source
This commit is contained in:
Timerix
2026-09-15 00:14:26 +02:00
parent 3dde5e5acf
commit 1d53f6930a
11 changed files with 889 additions and 667 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ParadoxSaveParser.Benchmarks/ExtractionBenchmarks.cs
+4 -22
View File
@@ -20,13 +20,10 @@ public class ExtractionBenchmarks
private byte[] _json = null!;
private ISearchExpression _query = null!;
private Regex _flatInterpreted = null!;
private Regex _flatCompiled = null!;
private Regex _flatSourceGen = null!;
private Regex _technologyBlock = null!;
private Regex _pathAwareCompiled = null!;
private Regex _pathAwareNonBacktracking = null!;
private Regex _countriesBlock = null!;
private PcreRegex _pcreFlat = null!;
private PcreRegex _pcrePathAware = null!;
[GlobalSetup]
@@ -37,13 +34,10 @@ public class ExtractionBenchmarks
_json = File.ReadAllBytes(BenchData.JsonPath);
_query = SearchExpressionCompiler.Compile(BenchData.PdxQuery);
_flatInterpreted = new Regex(Extractors.FlatPattern, RegexOptions.None);
_flatCompiled = new Regex(Extractors.FlatPattern, RegexOptions.Compiled);
_flatSourceGen = Extractors.FlatSourceGen();
_technologyBlock = new Regex(Extractors.TechnologyBlockPattern, RegexOptions.Compiled);
_pathAwareCompiled = new Regex(Extractors.PathAwarePattern, RegexOptions.Compiled);
_pathAwareNonBacktracking = new Regex(Extractors.PathAwarePattern, RegexOptions.NonBacktracking);
_countriesBlock = new Regex(Extractors.CountriesBlockPattern, RegexOptions.Compiled);
_pcreFlat = new PcreRegex(Extractors.FlatPattern, PcreOptions.Compiled);
_pcrePathAware = new PcreRegex(Extractors.PathAwarePattern, PcreOptions.Compiled);
}
@@ -53,26 +47,14 @@ public class ExtractionBenchmarks
[Benchmark(Description = "Full parse, then select")]
public long FullParse() => Extractors.FullParseThenSelect(_pdx);
[Benchmark(Description = ".NET Regex flat, interpreted")]
public long RegexFlatInterpreted() => Extractors.RegexScan(_flatInterpreted, _pdxText);
[Benchmark(Description = ".NET Regex flat, compiled")]
public long RegexFlatCompiled() => Extractors.RegexScan(_flatCompiled, _pdxText);
[Benchmark(Description = ".NET Regex flat, source generated")]
public long RegexFlatSourceGen() => Extractors.RegexScan(_flatSourceGen, _pdxText);
[Benchmark(Description = ".NET Regex path aware, compiled")]
public long RegexPathAwareCompiled() => Extractors.RegexScan(_pathAwareCompiled, _pdxText);
[Benchmark(Description = ".NET Regex path aware, NonBacktracking")]
public long RegexPathAwareNonBacktracking() => Extractors.RegexScan(_pathAwareNonBacktracking, _pdxText);
[Benchmark(Description = ".NET Regex balanced block + flat")]
public long RegexBalancedTwoStage() => Extractors.RegexTwoStage(_countriesBlock, _flatCompiled, _pdxText);
[Benchmark(Description = "PCRE.NET flat, JIT compiled")]
public long PcreFlat() => Extractors.PcreScan(_pcreFlat, _pdxText);
[Benchmark(Description = ".NET Regex balanced block + inner scan")]
public long RegexBalancedTwoStage() => Extractors.RegexTwoStage(_countriesBlock, _technologyBlock, _pdxText);
[Benchmark(Description = "PCRE.NET path aware, JIT compiled")]
public long PcrePathAware() => Extractors.PcreScan(_pcrePathAware, _pdxText);
ParadoxSaveParser.Benchmarks/Extractors.cs
+4 -6
View File
@@ -65,10 +65,11 @@ public static partial class Extractors
// ---------------------------------------------------------------- .NET Regex
/// <summary>
/// Flat scan: matches technology blocks anywhere in the file.
/// Fastest regex option, but it does not verify that the match really sits under <c>countries</c>.
/// Matches a single technology block. On its own it would match anywhere in the file,
/// so it is only used as the second stage of <see cref="RegexTwoStage" />, inside the
/// already extracted <c>countries</c> block.
/// </summary>
public const string FlatPattern =
public const string TechnologyBlockPattern =
@"\n\t\ttechnology=\{\n\t\t\tadm_tech=(\d+)\n\t\t\tdip_tech=(\d+)\n\t\t\tmil_tech=(\d+)\n\t\t\}";
/// <summary>
@@ -109,9 +110,6 @@ public static partial class Extractors
return RegexScan(innerRegex, block.Value);
}
[GeneratedRegex(FlatPattern)]
public static partial Regex FlatSourceGen();
[GeneratedRegex(PathAwarePattern)]
public static partial Regex PathAwareSourceGen();
ParadoxSaveParser.Benchmarks/ParadoxSaveParser.Benchmarks.csproj
+1 -1
View File
@@ -10,7 +10,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.15.4" />
<PackageReference Include="BenchmarkDotNet" Version="0.15.8" />
<PackageReference Include="PCRE.NET" Version="1.6.0" />
</ItemGroup>
ParadoxSaveParser.Benchmarks/Program.cs
+1 -4
View File
@@ -43,16 +43,13 @@ public static class Program
Time("SearchExpression", () => Extractors.SearchExpression(pdx, query));
Time("FullParse", () => Extractors.FullParseThenSelect(pdx));
Time("Regex flat compiled",
() => Extractors.RegexScan(new Regex(Extractors.FlatPattern, RegexOptions.Compiled), text));
Time("Regex path aware compiled",
() => Extractors.RegexScan(new Regex(Extractors.PathAwarePattern, RegexOptions.Compiled), text));
Time("Regex path aware nonbacktracking",
() => Extractors.RegexScan(new Regex(Extractors.PathAwarePattern, RegexOptions.NonBacktracking), text));
Time("Regex balanced two stage",
() => Extractors.RegexTwoStage(new Regex(Extractors.CountriesBlockPattern, RegexOptions.Compiled),
new Regex(Extractors.FlatPattern, RegexOptions.Compiled), text));
Time("PCRE flat", () => Extractors.PcreScan(new PcreRegex(Extractors.FlatPattern, PcreOptions.Compiled), text));
new Regex(Extractors.TechnologyBlockPattern, RegexOptions.Compiled), text));
Time("PCRE path aware",
() => Extractors.PcreScan(new PcreRegex(Extractors.PathAwarePattern, PcreOptions.Compiled), text));
Time("Utf8JsonReader", () => Extractors.Utf8JsonReaderScan(json));
ParadoxSaveParser.Benchmarks/README.md
+60 -54
View File
@@ -9,7 +9,7 @@ regex engines and `jq` on one realistic task:
|---|---|
| this repo | `countries.*.technology` |
| jq | `.countries \| map_values(.technology)` |
| .NET `Regex` / PCRE.NET | see `Extractors.FlatPattern` / `Extractors.PathAwarePattern` |
| .NET `Regex` / PCRE.NET | see `Extractors.PathAwarePattern` / `Extractors.CountriesBlockPattern` |
| `Utf8JsonReader` / `JsonDocument` | hand written navigation |
## Corpus
@@ -48,64 +48,70 @@ shows immediately when an engine finds something different from the others.
## Results
AMD Ryzen 7 5700X, 16 logical cores, Windows 10 21H2, .NET 10.0.12, jq 1.8.2,
109 MB save / 92 MB JSON twin, BenchmarkDotNet `RunStrategy.Monitoring`, 5 iterations.
AMD Ryzen 7 5700X, 8 physical cores (16 logical), Windows 10 21H2, .NET 10.0.12, jq 1.8.2,
109 MB save / 92 MB JSON twin, BenchmarkDotNet `RunStrategy.Monitoring`,
3 warmups and 10 iterations for the in-process engines, 5 for jq.
| engine | mean | vs this project | allocated | correct? |
|---|---:|---:|---:|---|
| .NET Regex flat, source generated | 14.8 ms | 0.02x | 1.2 MB | path-blind |
| .NET Regex flat, compiled | 17.0 ms | 0.03x | 1.2 MB | path-blind |
| .NET Regex flat, interpreted | 19.3 ms | 0.03x | 1.2 MB | path-blind |
| PCRE.NET flat, JIT compiled | 97.4 ms | 0.15x | 0.9 MB | path-blind |
| .NET Regex balanced block + flat | 112.3 ms | 0.17x | 224 MB | yes |
| .NET Regex path aware, compiled | 116.2 ms | 0.18x | 1.4 MB | 1869/1870 |
| PCRE.NET path aware, JIT compiled | 122.9 ms | 0.19x | 0.9 MB | 1869/1870 |
| Utf8JsonReader over JSON twin | 155.7 ms | 0.24x | 0 B | yes |
| JsonDocument over JSON twin | 321.4 ms | 0.50x | 0 B (native) | yes |
| .NET Regex path aware, NonBacktracking | 444.5 ms | 0.69x | 32 MB | 1869/1870 |
| **SearchExpression (this project)** | **645.1 ms** | **1.00x** | **1.6 MB** | yes |
| Full parse, then select | 1831 ms | 2.84x | 1153 MB | yes |
| jq `.countries \| map_values(.technology)` | 4340 ms | 6.73x | n/a | yes |
| jq `empty` (parse the file, emit nothing) | 4227 ms | 6.55x | n/a | n/a |
| jq `--stream` | 17490 ms | 27.1x | n/a | n/a |
| jq process startup only | 4.2 ms | 0.01x | n/a | n/a |
| **SearchExpression (this project)** | **49.6 ms** | **1.00x** | **1.46 MB** | yes |
| .NET Regex balanced block + inner scan | 114.0 ms | 2.30x | 213 MB | yes |
| PCRE.NET path aware, JIT compiled | 123.4 ms | 2.49x | 0.90 MB | 1869/1870 |
| .NET Regex path aware, compiled | 135.1 ms | 2.72x | 1.34 MB | 1869/1870 |
| Utf8JsonReader over JSON twin | 155.5 ms | 3.13x | 16 KB | yes |
| JsonDocument over JSON twin | 323.2 ms | 6.51x | 72 B (native) | yes |
| .NET Regex path aware, NonBacktracking | 440.5 ms | 8.87x | 30.6 MB | 1869/1870 |
| Full parse, then select | 1204 ms | 24.3x | 962 MB | yes |
| jq `empty` (parse the file, emit nothing) | 4188 ms | 84.4x | n/a | n/a |
| jq `.countries \| map_values(.technology)` | 4339 ms | 87.4x | n/a | yes |
| jq `--stream` | 17497 ms | 352x | n/a | n/a |
| jq process startup only | 4.1 ms | 0.08x | n/a | n/a |
Reading the save from disk instead of a preloaded `byte[]` costs the parser ~9 ms more:
58.4 ms via `FileStream` (`ParserInputBenchmarks`).
### Reading the table
* **jq is ~6.7x slower than this parser** and 97% of that time is JSON parsing, not the
query: `jq empty` on the same file costs 4227 ms of the 4340 ms. Process startup is
negligible (4 ms). jq's `--stream` mode, often recommended for large inputs, is 4x
*slower* still. jq also needs the data converted to JSON first, which this parser has to
do anyway — so end to end jq is strictly more expensive here.
* **The regexes are 5-40x faster, but they are not doing the same job.** A regex never
parses the structure; it scans bytes for a literal and validates a short window around it.
`Regex` with a literal prefix (`technology={`) is vectorized, so 109 MB is scanned at
several GB/s. That speed is real and the cost is real too: nothing verifies that the hit
is under `countries`, at the right depth, or belongs to the tag matched before it.
* **Path-aware regexes lose both the speed and the correctness.** Forcing the tag into the
pattern costs 7x (17 ms -> 116 ms) and still returns 1869 of 1870 countries: the tag class
`[A-Z0-9]{3}` silently drops EU4's `---` pseudo-country. Pairing survives here only by
luck of the file layout — all 933 tech-less country blocks happen to be grouped at the end
of the save, so the lazy gap never runs across one. A save that interleaves them would
make the regex report a technology block under the wrong country tag with no error.
* **`RegexOptions.NonBacktracking`** guarantees linear time but is 26x slower than the
compiled backtracking engine on this pattern and allocates 32 MB.
* **PCRE.NET** (the most used non-BCL regex engine in .NET, ~460k downloads) is 6x slower
than `System.Text.RegularExpressions` on the flat pattern, because .NET's vectorized
literal prefix search beats PCRE2's JIT here. On the path-aware pattern the two are equal.
There is no reason to leave the BCL engine for this workload.
* **The query is what makes this parser fast, not the parsing.** Same parser, same file:
645 ms with `countries.*.technology`, 1831 ms and 1.1 GB allocated without a query. The
search expression prunes ~65% of the work and 99.9% of the allocations.
* **Against a JSON reader on equivalent data**, the parser is 4x slower than `Utf8JsonReader`
and 2x slower than `JsonDocument`. Those numbers exclude the pdx -> JSON conversion
(~4.7 s), so they are a ceiling for the format, not a usable alternative.
* **This parser is the fastest engine measured here.** It is 2.3x faster than the only
regex approach that enforces the path, 3.1x faster than a hand written `Utf8JsonReader`
over the equivalent JSON, 6.5x faster than `JsonDocument`, and 87x faster than jq — while
allocating 1.46 MB for a 109 MB input.
* **jq is ~87x slower and 97% of that is JSON parsing, not the query**: `jq empty` on the
same file costs 4188 ms of the 4339 ms. Process startup is negligible (4 ms). jq's
`--stream` mode, often recommended for large inputs, is 4x *slower* still. jq also cannot
read the Paradox format, so it first needs the save converted to JSON — a conversion this
parser has to perform anyway.
* **Only one regex approach here is actually correct**, and it is the expensive one: cut the
`countries={...}` block out with a balancing-group pattern (a .NET-only feature; PCRE would
need recursion), then scan inside it. It allocates 213 MB, because stage one materialises
the whole block as a string.
* **The cheaper path-aware pattern pairs a country tag with the next technology block over a
lazy gap, and silently gets it wrong**: 1869 of 1870 countries, because the tag class
`[A-Z0-9]{3}` drops EU4's `---` pseudo-country. Even that much only holds by luck of the
file layout — all 933 tech-less country blocks happen to be grouped at the end of the save,
so the lazy gap never runs across one. A save that interleaved them would report a
technology block under the wrong country tag, with no error.
* **`RegexOptions.NonBacktracking`** guarantees linear time but is 3.3x slower than the
compiled backtracking engine on this pattern and allocates 31 MB.
* **PCRE.NET** (the most used non-BCL regex engine in .NET, ~460k downloads) is within ~10%
of `System.Text.RegularExpressions` on the path-aware pattern (123 ms vs 135 ms). Nothing
here justifies leaving the BCL engine.
* **The query is what makes this parser fast.** Same parser, same file: 50 ms with
`countries.*.technology`, 1204 ms and 962 MB allocated without a query — 24x and 660x.
### Where this parser's time goes
### How the parser got here
645 ms for 109 MB is ~170 MB/s, or ~13 cycles per input byte. The lexer reads the save one
byte at a time through `Stream.ReadByte()` (`SaveParserEU4.LexTextSave`), which is a virtual
call plus bounds check per byte, and appends char by char into a `StringBuilder`. Reading
into a `byte[]` buffer and scanning it with `ReadOnlySpan<byte>.IndexOfAny` would be the
first thing to try — the regex numbers above show what the same hardware does when it scans
a span instead of a stream.
Three measurements on the same benchmark (`ParserInputBenchmarks`, reading a `FileStream`):
| lexer | mean | allocated |
|---|---:|---:|
| `Stream.ReadByte()` per byte | 834 ms | 1.55 MB |
| copy the file into a `MemoryStream` first, still `ReadByte()` | 738 ms | 257 MB |
| 64 KB buffer + `Tokenizer` with byte-level block skipping | 58 ms | 1.46 MB |
The first jump came from removing a virtual call per byte. The large one came from making
the query prune *work* rather than just data: `Tokenizer.SkipBlock` throws away a rejected
`{...}` block by scanning raw bytes for brace depth with `SearchValues<byte>.IndexOfAny`,
so blocks the search expression rejects are never turned into tokens or strings at all.
Retained tokens are spans into the read buffer, decoded only when their value is kept, and
numbers are parsed straight from UTF8.