refactored SearchExpression

ParadoxSaveParser.Benchmarks/BenchData.cs

@@ -0,0 +1,74 @@
+using System;
+using System.IO;
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using ParadoxSaveParser.Lib;
+
+namespace ParadoxSaveParser.Benchmarks;
+
+/// <summary>
+///     Locates and lazily loads the benchmark corpus:
+///     a real EU4 <c>gamestate</c> file and its compact JSON twin (for jq).
+///     Override the directory with the PSP_BENCH_DATA environment variable.
+/// </summary>
+public static class BenchData
+{
+    public const string PdxQuery = "countries.*.technology";
+
+    /// <summary>jq program extracting the same data as <see cref="PdxQuery" />.</summary>
+    public const string JqQuery = ".countries | map_values(.technology)";
+
+    public static string Dir { get; } =
+        Environment.GetEnvironmentVariable("PSP_BENCH_DATA")
+        ?? Path.Combine(FindRepoRoot(), "ParadoxSaveParser.CLI", "bin", "Debug", "net10.0");
+
+    public static string PdxPath => Path.Combine(Dir, "gamestate");
+    public static string JsonPath => Path.Combine(Dir, "gamestate.min.json");
+
+    private static byte[]? _pdxBytes;
+    private static string? _pdxText;
+
+    /// <summary>Raw bytes of the save, kept in memory so benchmarks don't measure disk I/O.</summary>
+    public static byte[] PdxBytes => _pdxBytes ??= File.ReadAllBytes(PdxPath);
+
+    /// <summary>
+    ///     The save decoded as text for the regex engines.
+    /// </summary>
+    public static string PdxText => _pdxText ??= SaveParserEU4.DefaultEncoding.GetString(PdxBytes);
+
+    private static string FindRepoRoot()
+    {
+        var dir = new DirectoryInfo(AppContext.BaseDirectory);
+        while (dir != null && !File.Exists(Path.Combine(dir.FullName, "ParadoxSaveParser.sln")))
+            dir = dir.Parent;
+        return dir?.FullName ?? Directory.GetCurrentDirectory();
+    }
+
+    /// <summary>
+    ///     Produces the compact JSON twin of the save file, which is what jq is given.
+    ///     Compact (not indented) so jq isn't penalized by parsing whitespace.
+    /// </summary>
+    public static void PrepareJson()
+    {
+        if (File.Exists(JsonPath))
+        {
+            Console.WriteLine($"{JsonPath} already exists, skipping");
+            return;
+        }
+
+        Console.WriteLine($"parsing {PdxPath} ...");
+        using var input = File.OpenRead(PdxPath);
+        var root = new SaveParserEU4(input, null).Parse();
+        Console.WriteLine($"serializing to {JsonPath} ...");
+        using var output = File.Create(JsonPath);
+        JsonSerializer.Serialize(output, root, new JsonSerializerOptions
+        {
+            WriteIndented = false,
+            Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+            MaxDepth = 1024,
+            DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+        });
+        Console.WriteLine($"done: {new FileInfo(JsonPath).Length / 1024 / 1024} MB");
+    }
+}

ParadoxSaveParser.Benchmarks/ExtractionBenchmarks.cs

@@ -0,0 +1,67 @@
+using System.IO;
+using System.Text.RegularExpressions;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Engines;
+using ParadoxSaveParser.Lib;
+using PCRE;
+
+namespace ParadoxSaveParser.Benchmarks;
+
+/// <summary>
+///     In-process engines, all extracting <c>countries.*.technology</c> from the same 110 MB save.
+///     Input is preloaded into memory, so disk I/O is not part of any measurement.
+/// </summary>
+[MemoryDiagnoser]
+[SimpleJob(RunStrategy.Monitoring, launchCount: 1, warmupCount: 1, iterationCount: 5)]
+public class ExtractionBenchmarks
+{
+    private byte[] _pdx = null!;
+    private string _pdxText = null!;
+    private byte[] _json = null!;
+    private SearchExpressionCompilation _query = null!;
+
+    private Regex _technologyBlock = null!;
+    private Regex _pathAwareCompiled = null!;
+    private Regex _pathAwareNonBacktracking = null!;
+    private Regex _countriesBlock = null!;
+    private PcreRegex _pcrePathAware = null!;
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        _pdx = BenchData.PdxBytes;
+        _pdxText = BenchData.PdxText;
+        _json = File.ReadAllBytes(BenchData.JsonPath);
+        _query = new SearchExpressionCompilation(BenchData.PdxQuery);
+
+        _technologyBlock = new Regex(Extractors.TechnologyBlockPattern, RegexOptions.Compiled);
+        _pathAwareCompiled = new Regex(Extractors.PathAwarePattern, RegexOptions.Compiled);
+        _pathAwareNonBacktracking = new Regex(Extractors.PathAwarePattern, RegexOptions.NonBacktracking);
+        _countriesBlock = new Regex(Extractors.CountriesBlockPattern, RegexOptions.Compiled);
+        _pcrePathAware = new PcreRegex(Extractors.PathAwarePattern, PcreOptions.Compiled);
+    }
+
+    [Benchmark(Baseline = true, Description = "SearchExpression (this project)")]
+    public long SearchExpression() => Extractors.SearchExpression(_pdx, _query);
+
+    [Benchmark(Description = "Full parse, then select")]
+    public long FullParse() => Extractors.FullParseThenSelect(_pdx);
+
+    [Benchmark(Description = ".NET Regex path aware, compiled")]
+    public long RegexPathAwareCompiled() => Extractors.RegexScan(_pathAwareCompiled, _pdxText);
+
+    [Benchmark(Description = ".NET Regex path aware, NonBacktracking")]
+    public long RegexPathAwareNonBacktracking() => Extractors.RegexScan(_pathAwareNonBacktracking, _pdxText);
+
+    [Benchmark(Description = ".NET Regex balanced block + inner scan")]
+    public long RegexBalancedTwoStage() => Extractors.RegexTwoStage(_countriesBlock, _technologyBlock, _pdxText);
+
+    [Benchmark(Description = "PCRE.NET path aware, JIT compiled")]
+    public long PcrePathAware() => Extractors.PcreScan(_pcrePathAware, _pdxText);
+
+    [Benchmark(Description = "Utf8JsonReader over JSON twin")]
+    public long Utf8JsonReader() => Extractors.Utf8JsonReaderScan(_json);
+
+    [Benchmark(Description = "JsonDocument over JSON twin")]
+    public long JsonDocument() => Extractors.JsonDocumentScan(_json);
+}

ParadoxSaveParser.Benchmarks/Extractors.cs

@@ -0,0 +1,207 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using ParadoxSaveParser.Lib;
+using PCRE;
+
+namespace ParadoxSaveParser.Benchmarks;
+
+/// <summary>
+///     Every extractor solves the same task: pull the <c>technology</c> block
+///     (adm/dip/mil levels) of every country out of an EU4 save, i.e. the path
+///     <c>countries.*.technology</c>.
+///     They all return the same checksum so the benchmarks can be verified against each other:
+///     <c>number_of_countries * 1_000_000 + sum_of_all_tech_levels</c>.
+/// </summary>
+public static partial class Extractors
+{
+    public static long Checksum(long count, long techSum) => count * 1_000_000 + techSum;
+
+    // ---------------------------------------------------------------- this project
+
+    /// <summary>The parser of this repo: single streaming pass, guided by a compiled search expression.</summary>
+    public static long SearchExpression(byte[] pdx, SearchExpressionCompilation query)
+    {
+        using var stream = new MemoryStream(pdx, false);
+        var root = new SaveParserEU4(stream, query).Parse();
+        return ChecksumOfParsedTree(root);
+    }
+
+    /// <summary>Same query, but the parser reads from the given stream instead of a preloaded byte[].</summary>
+    public static long SearchExpressionFromStream(Stream stream, SearchExpressionCompilation query)
+    {
+        var root = new SaveParserEU4(stream, query).Parse();
+        return ChecksumOfParsedTree(root);
+    }
+
+    /// <summary>Same parser without a query: parses the whole save into memory (worst case reference point).</summary>
+    public static long FullParseThenSelect(byte[] pdx)
+    {
+        using var stream = new MemoryStream(pdx, false);
+        var root = new SaveParserEU4(stream, null).Parse();
+        return ChecksumOfParsedTree(root);
+    }
+
+    private static long ChecksumOfParsedTree(Dictionary<string, object> root)
+    {
+        long count = 0, sum = 0;
+        var countries = (Dictionary<string, object>)root["countries"];
+        foreach (var country in countries.Values)
+        {
+            if (country is not Dictionary<string, object> c
+                || !c.TryGetValue("technology", out var t)
+                || t is not Dictionary<string, object> tech)
+                continue;
+            count++;
+            foreach (var level in tech.Values)
+                sum += (long)level;
+        }
+
+        return Checksum(count, sum);
+    }
+
+    // ---------------------------------------------------------------- .NET Regex
+
+    /// <summary>
+    ///     Matches a single technology block. On its own it would match anywhere in the file,
+    ///     so it is only used as the second stage of <see cref="RegexTwoStage" />, inside the
+    ///     already extracted <c>countries</c> block.
+    /// </summary>
+    public const string TechnologyBlockPattern =
+        @"\n\t\ttechnology=\{\n\t\t\tadm_tech=(\d+)\n\t\t\tdip_tech=(\d+)\n\t\t\tmil_tech=(\d+)\n\t\t\}";
+
+    /// <summary>
+    ///     Path-aware: pairs a depth-1 country tag with the next technology block via a lazy gap.
+    ///     This is what a regex user writes when the key must be associated with the value.
+    /// </summary>
+    public const string PathAwarePattern =
+        @"\n\t([A-Z0-9]{3})=\{[\s\S]*?\n\t\ttechnology=\{\n\t\t\tadm_tech=(\d+)\n\t\t\tdip_tech=(\d+)\n\t\t\tmil_tech=(\d+)\n\t\t\}";
+
+    /// <summary>
+    ///     Balancing-group pattern isolating the whole <c>countries={...}</c> block.
+    ///     .NET-only feature; PCRE would need recursion instead.
+    /// </summary>
+    public const string CountriesBlockPattern =
+        @"\ncountries=\{(?>[^{}]+|\{(?<d>)|\}(?<-d>))*(?(d)(?!))\n\}";
+
+    public static long RegexScan(Regex regex, string text)
+    {
+        long count = 0, sum = 0;
+        foreach (Match m in regex.Matches(text))
+        {
+            count++;
+            int g = m.Groups.Count - 3;
+            sum += int.Parse(m.Groups[g].ValueSpan)
+                   + int.Parse(m.Groups[g + 1].ValueSpan)
+                   + int.Parse(m.Groups[g + 2].ValueSpan);
+        }
+
+        return Checksum(count, sum);
+    }
+
+    /// <summary>Two stages: cut out the countries block with balanced braces, then scan inside it.</summary>
+    public static long RegexTwoStage(Regex blockRegex, Regex innerRegex, string text)
+    {
+        var block = blockRegex.Match(text);
+        if (!block.Success)
+            throw new Exception("countries block not found");
+        return RegexScan(innerRegex, block.Value);
+    }
+
+    [GeneratedRegex(PathAwarePattern)]
+    public static partial Regex PathAwareSourceGen();
+
+    // ---------------------------------------------------------------- PCRE.NET
+
+    public static long PcreScan(PcreRegex regex, string text)
+    {
+        long count = 0, sum = 0;
+        foreach (var m in regex.Matches(text))
+        {
+            count++;
+            int g = m.Groups.Count - 3;
+            sum += int.Parse(m[g].Value)
+                   + int.Parse(m[g + 1].Value)
+                   + int.Parse(m[g + 2].Value);
+        }
+
+        return Checksum(count, sum);
+    }
+
+    // ---------------------------------------------------------------- System.Text.Json
+
+    private static readonly byte[] CountriesUtf8 = "countries"u8.ToArray();
+    private static readonly byte[] TechnologyUtf8 = "technology"u8.ToArray();
+
+    /// <summary>Hand written streaming reader over the JSON twin: the closest JSON analogue of the search expression.</summary>
+    public static long Utf8JsonReaderScan(byte[] json)
+    {
+        long count = 0, sum = 0;
+        var reader = new Utf8JsonReader(json, isFinalBlock: true, state: default);
+        reader.Read(); // StartObject (root)
+
+        while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName)
+        {
+            bool isCountries = reader.ValueTextEquals(CountriesUtf8);
+            reader.Read();
+            if (!isCountries)
+            {
+                reader.Skip();
+                continue;
+            }
+
+            // inside countries: property name = country tag
+            while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName)
+            {
+                reader.Read();
+                if (reader.TokenType != JsonTokenType.StartObject)
+                {
+                    reader.Skip();
+                    continue;
+                }
+
+                while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName)
+                {
+                    bool isTech = reader.ValueTextEquals(TechnologyUtf8);
+                    reader.Read();
+                    if (!isTech)
+                    {
+                        reader.Skip();
+                        continue;
+                    }
+
+                    count++;
+                    while (reader.Read() && reader.TokenType == JsonTokenType.PropertyName)
+                    {
+                        reader.Read();
+                        sum += reader.GetInt64();
+                    }
+                }
+            }
+
+            break; // countries handled, nothing else is needed
+        }
+
+        return Checksum(count, sum);
+    }
+
+    /// <summary>DOM approach: parse the whole JSON document, then navigate. The .NET equivalent of what jq does.</summary>
+    public static long JsonDocumentScan(byte[] json)
+    {
+        long count = 0, sum = 0;
+        using var doc = JsonDocument.Parse(json, new JsonDocumentOptions { MaxDepth = 1024 });
+        foreach (var country in doc.RootElement.GetProperty("countries").EnumerateObject())
+        {
+            if (country.Value.ValueKind != JsonValueKind.Object
+                || !country.Value.TryGetProperty("technology", out var tech))
+                continue;
+            count++;
+            foreach (var level in tech.EnumerateObject())
+                sum += level.Value.GetInt64();
+        }
+
+        return Checksum(count, sum);
+    }
+}

ParadoxSaveParser.Benchmarks/JqBenchmarks.cs

@@ -0,0 +1,79 @@
+using System;
+using System.Diagnostics;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Engines;
+
+namespace ParadoxSaveParser.Benchmarks;
+
+/// <summary>
+///     jq runs as an external process on the JSON twin of the save.
+///     Numbers therefore include process start and reading the file from the OS page cache,
+///     which is exactly what you pay when you shell out to jq.
+///     <see cref="JqNoop" /> measures that fixed overhead so it can be subtracted.
+/// </summary>
+[SimpleJob(RunStrategy.Monitoring, launchCount: 1, warmupCount: 1, iterationCount: 5)]
+public class JqBenchmarks
+{
+    public static string JqPath { get; set; } = "jq";
+
+    [GlobalSetup]
+    public void Setup() => Run("--version", null);
+
+    [Benchmark(Description = "jq  .countries | map_values(.technology)")]
+    public int Jq() => Run("-c", BenchData.JqQuery);
+
+    [Benchmark(Description = "jq  empty (parse only)")]
+    public int JqParseOnly() => Run("-c", "empty");
+
+    [Benchmark(Description = "jq  --stream (no DOM)")]
+    public int JqStream() => Run("-c --stream", "empty");
+
+    [Benchmark(Description = "jq  process startup only (tiny input)")]
+    public int JqNoop()
+    {
+        var psi = new ProcessStartInfo(JqPath)
+        {
+            RedirectStandardOutput = true,
+            RedirectStandardInput = true,
+            UseShellExecute = false,
+        };
+        psi.ArgumentList.Add("-c");
+        psi.ArgumentList.Add("empty");
+        using var p = Process.Start(psi)!;
+        p.StandardInput.Write("{}");
+        p.StandardInput.Close();
+        p.StandardOutput.ReadToEnd();
+        p.WaitForExit();
+        return p.ExitCode;
+    }
+
+    private static int Run(string flags, string? program)
+    {
+        var psi = new ProcessStartInfo(JqPath)
+        {
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+            UseShellExecute = false,
+        };
+        foreach (var f in flags.Split(' ', StringSplitOptions.RemoveEmptyEntries))
+            psi.ArgumentList.Add(f);
+        if (program != null)
+        {
+            psi.ArgumentList.Add(program);
+            psi.ArgumentList.Add(BenchData.JsonPath);
+        }
+
+        using var p = Process.Start(psi)!;
+        // output must be drained, otherwise jq blocks on a full pipe
+        long bytes = 0;
+        var buf = new char[1 << 16];
+        int read;
+        while ((read = p.StandardOutput.Read(buf, 0, buf.Length)) > 0)
+            bytes += read;
+        p.StandardError.ReadToEnd();
+        p.WaitForExit();
+        if (p.ExitCode != 0)
+            throw new Exception($"jq exited with {p.ExitCode}");
+        return (int)(bytes & int.MaxValue);
+    }
+}

ParadoxSaveParser.Benchmarks/Mispairing.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.RegularExpressions;
+using ParadoxSaveParser.Lib;
+
+namespace ParadoxSaveParser.Benchmarks;
+
+/// <summary>
+///     Checks whether the regex approach associates the right country tag with the right
+///     technology block. It cannot: 933 of the 2803 country blocks in the test save have no
+///     technology block at all, so a lazy "tag ... technology" pattern silently pairs a tag
+///     with a technology block belonging to a later country.
+/// </summary>
+public static class Mispairing
+{
+    public static Dictionary<string, long> FromParser(byte[] pdx)
+    {
+        var result = new Dictionary<string, long>();
+        using var stream = new MemoryStream(pdx, false);
+        var root = new SaveParserEU4(stream, new SearchExpressionCompilation(BenchData.PdxQuery)).Parse();
+        foreach (var (tag, value) in (Dictionary<string, object>)root["countries"])
+        {
+            if (value is Dictionary<string, object> c
+                && c.TryGetValue("technology", out var t)
+                && t is Dictionary<string, object> tech
+                && tech.TryGetValue("adm_tech", out var adm))
+                result[tag] = (long)adm;
+        }
+
+        return result;
+    }
+
+    public static Dictionary<string, long> FromRegex(string text, Regex pathAware)
+    {
+        var result = new Dictionary<string, long>();
+        foreach (Match m in pathAware.Matches(text))
+            result[m.Groups[1].Value] = long.Parse(m.Groups[2].ValueSpan);
+        return result;
+    }
+
+    public static void Report(byte[] pdx, string text)
+    {
+        var expected = FromParser(pdx);
+        var actual = FromRegex(text, new Regex(Extractors.PathAwarePattern, RegexOptions.Compiled));
+        int missing = 0, wrong = 0, extra = 0;
+        foreach (var (tag, adm) in expected)
+        {
+            if (!actual.TryGetValue(tag, out long got)) { missing++; Console.WriteLine($"  missing tag: {tag}"); }
+            else if (got != adm) wrong++;
+        }
+
+        foreach (var tag in actual.Keys)
+            if (!expected.ContainsKey(tag))
+                extra++;
+
+        Console.WriteLine($"path aware regex pairing: {expected.Count} countries have technology, " +
+                          $"regex reported {actual.Count}; missing={missing} wrong_value={wrong} invented={extra}");
+    }
+}

ParadoxSaveParser.Benchmarks/ParadoxSaveParser.Benchmarks.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net10.0</TargetFramework>
+        <LangVersion>latest</LangVersion>
+        <Nullable>enable</Nullable>
+        <ImplicitUsings>disable</ImplicitUsings>
+        <InvariantGlobalization>true</InvariantGlobalization>
+        <ServerGarbageCollection>true</ServerGarbageCollection>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="BenchmarkDotNet" Version="0.15.8" />
+        <PackageReference Include="PCRE.NET" Version="1.6.0" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj" />
+    </ItemGroup>
+</Project>

ParadoxSaveParser.Benchmarks/ParserInputBenchmarks.cs

@@ -0,0 +1,29 @@
+using System.IO;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Engines;
+using ParadoxSaveParser.Lib;
+
+namespace ParadoxSaveParser.Benchmarks;
+
+/// <summary>
+///     SearchExpression only, parsing straight from the save file on disk:
+///     the file is opened inside the benchmark and is never preloaded into a byte[].
+///     Run it once with and once without the internal <c>MemoryStream</c> copy in
+///     <see cref="SaveParserEU4" /> to see what that copy buys.
+/// </summary>
+[MemoryDiagnoser]
+[SimpleJob(RunStrategy.Monitoring, launchCount: 1, warmupCount: 1, iterationCount: 5)]
+public class ParserInputBenchmarks
+{
+    private SearchExpressionCompilation _query = null!;
+
+    [GlobalSetup]
+    public void Setup() => _query = new SearchExpressionCompilation(BenchData.PdxQuery);
+
+    [Benchmark(Description = "SearchExpression over FileStream")]
+    public long FromFile()
+    {
+        using var file = File.OpenRead(BenchData.PdxPath);
+        return Extractors.SearchExpressionFromStream(file, _query);
+    }
+}

ParadoxSaveParser.Benchmarks/Program.cs

@@ -0,0 +1,77 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using BenchmarkDotNet.Running;
+using PCRE;
+using ParadoxSaveParser.Lib;
+
+namespace ParadoxSaveParser.Benchmarks;
+
+public static class Program
+{
+    public static void Main(string[] args)
+    {
+        switch (args.FirstOrDefault())
+        {
+            case "prepare":
+                BenchData.PrepareJson();
+                return;
+            case "verify":
+                Verify();
+                return;
+            default:
+                BenchmarkSwitcher.FromTypes([typeof(ExtractionBenchmarks), typeof(JqBenchmarks), typeof(ParserInputBenchmarks)])
+                    .Run(args);
+                return;
+        }
+    }
+
+    /// <summary>
+    ///     Runs every extractor once and prints its checksum and wall time,
+    ///     so that differences in what each engine actually finds are visible before benchmarking.
+    /// </summary>
+    private static void Verify()
+    {
+        Console.WriteLine($"save:  {BenchData.PdxPath}  {new FileInfo(BenchData.PdxPath).Length / 1024 / 1024} MB");
+        Console.WriteLine($"json:  {BenchData.JsonPath}  {new FileInfo(BenchData.JsonPath).Length / 1024 / 1024} MB");
+
+        var pdx = BenchData.PdxBytes;
+        var text = BenchData.PdxText;
+        var json = File.ReadAllBytes(BenchData.JsonPath);
+        var query = new SearchExpressionCompilation(BenchData.PdxQuery);
+
+        Time("SearchExpression", () => Extractors.SearchExpression(pdx, query));
+        Time("FullParse", () => Extractors.FullParseThenSelect(pdx));
+        Time("Regex path aware compiled",
+            () => Extractors.RegexScan(new Regex(Extractors.PathAwarePattern, RegexOptions.Compiled), text));
+        Time("Regex path aware nonbacktracking",
+            () => Extractors.RegexScan(new Regex(Extractors.PathAwarePattern, RegexOptions.NonBacktracking), text));
+        Time("Regex balanced two stage",
+            () => Extractors.RegexTwoStage(new Regex(Extractors.CountriesBlockPattern, RegexOptions.Compiled),
+                new Regex(Extractors.TechnologyBlockPattern, RegexOptions.Compiled), text));
+        Time("PCRE path aware",
+            () => Extractors.PcreScan(new PcreRegex(Extractors.PathAwarePattern, PcreOptions.Compiled), text));
+        Time("Utf8JsonReader", () => Extractors.Utf8JsonReaderScan(json));
+        Mispairing.Report(pdx, text);
+        Time("JsonDocument", () => Extractors.JsonDocumentScan(json));
+    }
+
+    private static void Time(string name, Func<long> f)
+    {
+        var sw = System.Diagnostics.Stopwatch.StartNew();
+        long checksum;
+        try
+        {
+            checksum = f();
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"{name,-34} FAILED: {ex.Message}");
+            return;
+        }
+
+        sw.Stop();
+        Console.WriteLine($"{name,-34} countries={checksum / 1_000_000,5}  techSum={checksum % 1_000_000,7}  {sw.ElapsedMilliseconds,7} ms");
+    }
+}

ParadoxSaveParser.Benchmarks/README.md

@@ -0,0 +1,117 @@
+# ParadoxSaveParser.Benchmarks
+
+Compares the query engine of this repo (`SearchExpression` + `SaveParserEU4`) against
+regex engines and `jq` on one realistic task:
+
+> extract the `technology` block (adm/dip/mil level) of every country from an EU4 save
+
+| engine | query |
+|---|---|
+| this repo | `countries.*.technology` |
+| jq | `.countries \| map_values(.technology)` |
+| .NET `Regex` / PCRE.NET | see `Extractors.PathAwarePattern` / `Extractors.CountriesBlockPattern` |
+| `Utf8JsonReader` / `JsonDocument` | hand written navigation |
+
+## Corpus
+
+* `gamestate` — a real 109 MB EU4 1.37 save (text format, 6.0M lines, 2803 country blocks,
+  1870 of which have a `technology` block).
+* `gamestate.min.json` — 92 MB compact JSON twin, produced from the save by this project's
+  own parser. It exists because jq and the `System.Text.Json` engines cannot read the
+  Paradox format at all.
+
+The save is **not** in the repository. Point `PSP_BENCH_DATA` at the directory holding it,
+or drop it in `ParadoxSaveParser.CLI/bin/Debug/net10.0/` (the default location).
+
+## Usage
+
+```sh
+dotnet build ParadoxSaveParser.Benchmarks -c Release
+cd ParadoxSaveParser.Benchmarks/bin/Release/net10.0
+
+./ParadoxSaveParser.Benchmarks.exe prepare   # build the JSON twin (once)
+./ParadoxSaveParser.Benchmarks.exe verify    # one run of every engine + correctness check
+./ParadoxSaveParser.Benchmarks.exe --filter '*'   # full BenchmarkDotNet run
+```
+
+Every engine returns the checksum `countries * 1_000_000 + sum_of_tech_levels`, so `verify`
+shows immediately when an engine finds something different from the others.
+
+## Fairness notes
+
+* All in-process engines read from memory; file I/O is excluded. jq is an external process,
+  so its numbers include process start (measured separately by the `jq startup only`
+  benchmark) and reading the JSON from the OS page cache.
+* The regex engines work on the save text, which is 109 MB; jq works on 92 MB of JSON.
+* jq is the only engine here that is a general query language: it can do arithmetic,
+  filtering and reshaping, which neither the regexes nor `SearchExpression` can.
+
+## Results
+
+AMD Ryzen 7 5700X, 8 physical cores (16 logical), Windows 10 21H2, .NET 10.0.12, jq 1.8.2,
+109 MB save / 92 MB JSON twin, BenchmarkDotNet `RunStrategy.Monitoring`,
+3 warmups and 10 iterations for the in-process engines, 5 for jq.
+
+| engine | mean | vs this project | allocated | correct? |
+|---|---:|---:|---:|---|
+| **SearchExpression (this project)** | **49.6 ms** | **1.00x** | **1.46 MB** | yes |
+| .NET Regex balanced block + inner scan | 114.0 ms | 2.30x | 213 MB | yes |
+| PCRE.NET path aware, JIT compiled | 123.4 ms | 2.49x | 0.90 MB | 1869/1870 |
+| .NET Regex path aware, compiled | 135.1 ms | 2.72x | 1.34 MB | 1869/1870 |
+| Utf8JsonReader over JSON twin | 155.5 ms | 3.13x | 16 KB | yes |
+| JsonDocument over JSON twin | 323.2 ms | 6.51x | 72 B (native) | yes |
+| .NET Regex path aware, NonBacktracking | 440.5 ms | 8.87x | 30.6 MB | 1869/1870 |
+| Full parse, then select | 1204 ms | 24.3x | 962 MB | yes |
+| jq `empty` (parse the file, emit nothing) | 4188 ms | 84.4x | n/a | n/a |
+| jq `.countries \| map_values(.technology)` | 4339 ms | 87.4x | n/a | yes |
+| jq `--stream` | 17497 ms | 352x | n/a | n/a |
+| jq process startup only | 4.1 ms | 0.08x | n/a | n/a |
+
+Reading the save from disk instead of a preloaded `byte[]` costs the parser ~9 ms more:
+58.4 ms via `FileStream` (`ParserInputBenchmarks`).
+
+### Reading the table
+
+* **This parser is the fastest engine measured here.** It is 2.3x faster than the only
+  regex approach that enforces the path, 3.1x faster than a hand written `Utf8JsonReader`
+  over the equivalent JSON, 6.5x faster than `JsonDocument`, and 87x faster than jq — while
+  allocating 1.46 MB for a 109 MB input.
+* **jq is ~87x slower and 97% of that is JSON parsing, not the query**: `jq empty` on the
+  same file costs 4188 ms of the 4339 ms. Process startup is negligible (4 ms). jq's
+  `--stream` mode, often recommended for large inputs, is 4x *slower* still. jq also cannot
+  read the Paradox format, so it first needs the save converted to JSON — a conversion this
+  parser has to perform anyway.
+* **Only one regex approach here is actually correct**, and it is the expensive one: cut the
+  `countries={...}` block out with a balancing-group pattern (a .NET-only feature; PCRE would
+  need recursion), then scan inside it. It allocates 213 MB, because stage one materialises
+  the whole block as a string.
+* **The cheaper path-aware pattern pairs a country tag with the next technology block over a
+  lazy gap, and silently gets it wrong**: 1869 of 1870 countries, because the tag class
+  `[A-Z0-9]{3}` drops EU4's `---` pseudo-country. Even that much only holds by luck of the
+  file layout — all 933 tech-less country blocks happen to be grouped at the end of the save,
+  so the lazy gap never runs across one. A save that interleaved them would report a
+  technology block under the wrong country tag, with no error.
+* **`RegexOptions.NonBacktracking`** guarantees linear time but is 3.3x slower than the
+  compiled backtracking engine on this pattern and allocates 31 MB.
+* **PCRE.NET** (the most used non-BCL regex engine in .NET, ~460k downloads) is within ~10%
+  of `System.Text.RegularExpressions` on the path-aware pattern (123 ms vs 135 ms). Nothing
+  here justifies leaving the BCL engine.
+* **The query is what makes this parser fast.** Same parser, same file: 50 ms with
+  `countries.*.technology`, 1204 ms and 962 MB allocated without a query — 24x and 660x.
+
+### How the parser got here
+
+Three measurements on the same benchmark (`ParserInputBenchmarks`, reading a `FileStream`):
+
+| lexer | mean | allocated |
+|---|---:|---:|
+| `Stream.ReadByte()` per byte | 834 ms | 1.55 MB |
+| copy the file into a `MemoryStream` first, still `ReadByte()` | 738 ms | 257 MB |
+| 64 KB buffer + `Tokenizer` with byte-level block skipping | 58 ms | 1.46 MB |
+
+The first jump came from removing a virtual call per byte. The large one came from making
+the query prune *work* rather than just data: `Tokenizer.SkipBlock` throws away a rejected
+`{...}` block by scanning raw bytes for brace depth with `SearchValues<byte>.IndexOfAny`,
+so blocks the search expression rejects are never turned into tokens or strings at all.
+Retained tokens are spans into the read buffer, decoded only when their value is kept, and
+numbers are parsed straight from UTF8.

ParadoxSaveParser.CLI/Modes/Search.cs

@@ -33,7 +33,7 @@ internal static partial class Modes
                 ? Console.OpenStandardOutput()
                 : File.OpenWrite(outputPath.Value);
 
-            var searchExpression = SearchExpressionCompiler.Compile(searchQuery);
+            var searchExpression = new SearchExpressionCompilation(searchQuery);
 
             var parser = new SaveParserEU4(inputStream, searchExpression);
             var parsedValue = parser.Parse();

ParadoxSaveParser.CLI/Program.cs

@@ -14,17 +14,17 @@ try
     string? searchQuery = null;
 
     new LaunchArgumentParser(
-            new LaunchArgument(["-i", "--input"],
+            new LaunchArgument(["i", "input"],
                 "Set input file path",
                 s => inputPath = s,
                 "gamestate or zip file"),
 
-            new LaunchArgument(["-o", "--output"],
+            new LaunchArgument(["o", "output"],
                 "Set output file path",
                 s => outputPath = s,
                 "json file [default=stdout]"),
 
-            new LaunchArgument(["-s", "--search"],
+            new LaunchArgument(["s", "search"],
                 "Search in input file",
                 s =>
                 {

ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs

@@ -1,4 +1,7 @@
+using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Text;
 using System.Text.Encodings.Web;
 using System.Text.Json;
 using System.Text.Json.Serialization;
@@ -41,12 +44,151 @@ public class SearchExpressionTests
     [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")]
     public void TestSearchOnSmallData(string input, string expectedOutput)
     {
-        using var saveStream = new MemoryStream(_smallSaveData, false);
-        var se = SearchExpressionCompiler.Compile(input);
-        var parser = new SaveParserEU4(saveStream, se);
+        Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput));
+    }
+
+    /// <summary>
+    ///     The same queries, but with a read buffer so small that tokens, quoted strings and
+    ///     skipped blocks are split across refills.
+    /// </summary>
+    [TestCase("a", "a={ b={ c=0 d=1 e=2 } f=3 }")]
+    [TestCase("a.b", "a={ b={ c=0 d=1 e=2 } }")]
+    [TestCase("a.[1]", "a={ f=3 }")]
+    [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")]
+    public void TestSearchAcrossBufferRefills(string input, string expectedOutput)
+    {
+        foreach (int bufferSize in new[] { 16, 17, 23, 64 })
+            Assert.That(Search(_smallSaveData, input, bufferSize), Is.EqualTo(expectedOutput),
+                $"buffer size {bufferSize}");
+    }
+
+    [Test]
+    public void BracesInsideQuotedStringAreText()
+    {
+        byte[] data = "EU4txt a={ name=\"x{y}z\" b=1 }".ToBytes();
+        using var saveStream = new MemoryStream(data, false);
+        var parser = new SaveParserEU4(saveStream, Compile("a"));
+        var a = (Dictionary<string, object>)parser.Parse()["a"];
+        Assert.Multiple(() =>
+        {
+            Assert.That(a["name"], Is.EqualTo("x{y}z"));
+            Assert.That(a["b"], Is.EqualTo(1L));
+        });
+    }
+
+    [Test]
+    public void SkippedBlockIgnoresBracesInsideQuotedStrings()
+    {
+        byte[] data = "EU4txt a={ s=\"{{{\" } b=2".ToBytes();
+        using var saveStream = new MemoryStream(data, false);
+        var parser = new SaveParserEU4(saveStream, Compile("b"));
+        Assert.That(parser.Parse()["b"], Is.EqualTo(2L));
+    }
+
+    [Test]
+    public void EncodingIsConfigurable()
+    {
+        byte[] data = Encoding.UTF8.GetBytes("EU4txt a={ name=\"Ä\" }");
+        using var saveStream = new MemoryStream(data, false);
+        var parser = new SaveParserEU4(saveStream, Compile("a", Encoding.UTF8), Encoding.UTF8);
+        var a = (Dictionary<string, object>)parser.Parse()["a"];
+        Assert.That(a["name"], Is.EqualTo("Ä"));
+    }
+
+    /// <summary>
+    ///     A path written after a group applies to every alternative of that group.
+    /// </summary>
+    [TestCase("a.(b|zz).c", "a={ b={ c=0 } }")]
+    [TestCase("(a).b.(d|e)", "a={ b={ d=1 e=2 } }")]
+    [TestCase("a.(b|f).c", "a={ b={ c=0 } f=3 }")]
+    [TestCase("(a.(b|zz)|yy).(c|d)", "a={ b={ c=0 d=1 } }")]
+    public void PathAfterGroupContinuesEveryAlternative(string input, string expectedOutput)
+    {
+        Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput));
+    }
+
+    /// <summary>
+    ///     Groups with more alternatives than the other tests use: a key that matches nothing,
+    ///     a repeated key, and a literal key standing before a "*".
+    /// </summary>
+    [TestCase("a.b.(c|d|e|zz)", "a={ b={ c=0 d=1 e=2 } }")]
+    [TestCase("a.b.(c|c|c|c)", "a={ b={ c=0 } }")]
+    [TestCase("a.(b|zz|yy|xx|*)", "a={ b={ c=0 d=1 e=2 } f=3 }")]
+    public void GroupWithManyAlternatives(string input, string expectedOutput)
+    {
+        Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput));
+    }
+
+    /// <summary>
+    ///     Bytes 0x80-0x9F are punctuation in Windows-1252 and control characters in Latin1,
+    ///     and the parser strips control characters, so the two encodings are told apart here.
+    /// </summary>
+    [Test]
+    public void DefaultEncodingIsWindows1252()
+    {
+        // 93 and 94 are curly quotes, 96 is an en dash
+        byte[] data = [.. "EU4txt a={ name=\""u8, 0x93, 0x96, 0x94, .. "\" }"u8];
+        using var saveStream = new MemoryStream(data, false);
+        var parser = new SaveParserEU4(saveStream, Compile("a"));
+        var a = (Dictionary<string, object>)parser.Parse()["a"];
+        Assert.That(a["name"], Is.EqualTo("“–”"));
+    }
+
+    /// <summary>One compilation, two saves whose keys are written in different encodings.</summary>
+    [Test]
+    public void CompilationReEncodesKeysForAnotherEncoding()
+    {
+        var compilation = new SearchExpressionCompilation("ä");
+        foreach (var encoding in new[] { SaveParserEU4.DefaultEncoding, Encoding.UTF8, SaveParserEU4.DefaultEncoding })
+        {
+            byte[] data = encoding.GetBytes("EU4txt ä={ b=1 }");
+            using var saveStream = new MemoryStream(data, false);
+            // the parser re-encodes the shared compilation to its own encoding
+            var parser = new SaveParserEU4(saveStream, compilation, encoding);
+            var found = (Dictionary<string, object>)parser.Parse()["ä"];
+            Assert.That(found["b"], Is.EqualTo(1L), encoding.EncodingName);
+        }
+    }
+
+    [TestCase("a..b")]       // empty step
+    [TestCase(".b")]         // empty first step
+    [TestCase("a.")]         // trailing point
+    [TestCase("(a||b)")]     // empty alternative
+    [TestCase("(a|b")]       // unclosed group
+    [TestCase("(a|b))")]     // extra closing bracket
+    [TestCase("(a|b)c")]     // group not followed by '.'
+    [TestCase("(a|b).")]     // group followed by nothing
+    [TestCase("a.[1")]       // index step without ']'
+    [TestCase("a.[x]")]      // index that is not a number
+    [TestCase("a.[-1]")]     // negative index
+    [TestCase("a.b*c")]      // wildcard inside a key
+    public void MalformedQueryIsReported(string query)
+    {
+        Assert.Throws<SearchExpressionException>(() => Compile(query));
+    }
+
+    [Test]
+    public void EmptyQueryIsAnArgumentError()
+    {
+        // ReSharper disable once ObjectCreationAsStatement
+        Assert.Throws<ArgumentException>(() => new SearchExpressionCompilation(""));
+    }
+
+    /// <summary>Compiles right away, so that a malformed query throws here and not inside a parser.</summary>
+    private static SearchExpressionCompilation Compile(string query, Encoding? encoding = null)
+    {
+        var compilation = new SearchExpressionCompilation(query);
+        compilation.Compile(encoding ?? SaveParserEU4.DefaultEncoding);
+        return compilation;
+    }
+
+    private static string Search(byte[] saveData, string query, int bufferSize = 64 * 1024)
+    {
+        using var saveStream = new MemoryStream(saveData, false);
+        var se = Compile(query);
+        var parser = new SaveParserEU4(saveStream, se, bufferSize: bufferSize);
         var rootNode = parser.Parse();
         string json = JsonSerializer.Serialize(rootNode, _smallSaveSerializerOptions);
-        string pdx = JsonToPdx(json);
-        Assert.That(pdx, Is.EqualTo(expectedOutput));
+        return JsonToPdx(json);
     }
 }

ParadoxSaveParser.Lib/BufferedEnumerator.cs

@@ -1,121 +0,0 @@
-using System.Collections;
-
-namespace ParadoxSaveParser.Lib;
-
-/// <summary>
-/// Enumerator wrapper that stores <c>N/2</c> items before and <c>N/2-1</c> after <c>Current</c> item.
-/// </summary>
-/// <code language="cs">
-/// IEnumerator&lt;int&gt; Enumerator()
-/// {
-///     for(int i = 0; i &lt; 6; i++)
-///         yield return i;
-/// }
-///         
-/// var en = Enumerator();
-/// var bufen = new BufferedEnumerator&lt;int&gt;(en, 5);
-///         
-/// while(bufen.MoveNext())
-/// {
-///     var cur = bufen.Current;
-///     for (var prev = cur.List?.First; prev != cur; prev = prev?.Next)
-///         Console.Write($"{prev?.Value} ");
-///             
-///     Console.Write($"| {cur.Value} |");
-///             
-///     for (var next = cur.Next; next is not null; next = next.Next) 
-///         Console.Write($" {next.Value}");
-///     Console.WriteLine();
-/// }
-/// </code>
-/// Output:
-/// <code>
-/// | 0 | 1 2 3 4
-/// 0 | 1 | 2 3 4
-/// 0 1 | 2 | 3 4
-/// 1 2 | 3 | 4 5
-/// 2 3 | 4 | 5
-/// 3 4 | 5 |
-/// </code>
-public class BufferedEnumerator<T> : IEnumerator<BufferedEnumerator<T>.Node>
-{
-    public class Node
-    {
-#nullable disable
-        public Node Previous;
-        public Node Next;
-        public T Value;
-#nullable enable
-    }
-    
-    private readonly IEnumerator<T> _enumerator;
-    private readonly Node[] _ringBuffer;
-    private Node? _currentNode;
-    private int _currentBufferIndex = -1;
-    private int _lastValueIndex = -1;
-
-    public BufferedEnumerator(IEnumerator<T> enumerator, int bufferSize)
-    {
-        _enumerator = enumerator;
-        _ringBuffer = new Node[bufferSize];
-    }
-
-    private void InitBuffer()
-    {
-        _ringBuffer[0] = new Node
-        {
-            Value = default!
-        };
-        for (int i = 1; i < _ringBuffer.Length; i++)
-        {
-            _ringBuffer[i] = new Node
-            {
-                Previous = _ringBuffer[i - 1],
-                Value = default!,
-            };
-            _ringBuffer[i - 1].Next = _ringBuffer[i];
-        }
-        _ringBuffer[^1].Next = _ringBuffer[0];
-        _ringBuffer[0].Previous = _ringBuffer[^1];
-    }
-    
-    public bool MoveNext()
-    {
-        if (_currentBufferIndex == -1)
-        {
-            InitBuffer();
-            
-            int beforeMidpoint = _ringBuffer.Length / 2 - 1;
-            for (int i = 0; i <= beforeMidpoint && _enumerator.MoveNext(); i++)
-            {
-                _ringBuffer[i].Value = _enumerator.Current;
-            }
-        }
-        
-        _currentBufferIndex = (_currentBufferIndex + 1) % _ringBuffer.Length;
-        if (_enumerator.MoveNext())
-        {
-            int midpoint = (_currentBufferIndex + _ringBuffer.Length / 2) % _ringBuffer.Length;
-            _ringBuffer[midpoint].Value = _enumerator.Current;
-            _lastValueIndex = midpoint;
-        }
-        if(_currentBufferIndex == (_lastValueIndex + 1) % _ringBuffer.Length)
-            return false;
-        
-        _currentNode = _ringBuffer[_currentBufferIndex];
-        return true;
-    }
-
-    public void Reset()
-    {
-        throw new NotImplementedException();
-    }
-
-    public Node Current => _currentNode!;
-
-    object IEnumerator.Current => Current;
-
-    public void Dispose()
-    {
-    }
-}

ParadoxSaveParser.Lib/ParadoxEncodings.cs

@@ -0,0 +1,19 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>
+///     Encodings used by Paradox save files.
+///     Touching this class registers the code page provider, so no consumer has to do it.
+/// </summary>
+public static class ParadoxEncodings
+{
+    /// <summary>Encoding of EU4 save files.</summary>
+    public static Encoding Windows1252 { get; }
+
+    static ParadoxEncodings()
+    {
+        // .NET ships only ASCII, Latin1 and the Unicode encodings; code pages come from this provider
+        Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
+        // assigned here and not in an initializer, which would run before the provider is registered
+        Windows1252 = Encoding.GetEncoding(1252);
+    }
+}

ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj

@@ -8,6 +8,6 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="Microsoft.Extensions.ObjectPool" Version="10.0.12" />
+      <InternalsVisibleTo Include="ParadoxSaveParser.Lib.Tests" />
     </ItemGroup>
 </Project>

ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj.DotSettings

@@ -0,0 +1,2 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=searchexpression/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

ParadoxSaveParser.Lib/SaveParserEU4.cs

@@ -1,8 +1,8 @@
-global using System;
+global using System;
 global using System.Collections.Generic;
 global using System.IO;
 global using System.Text;
-using Microsoft.Extensions.ObjectPool;
+using System.Globalization;
 
 namespace ParadoxSaveParser.Lib;
 
@@ -11,217 +11,118 @@ namespace ParadoxSaveParser.Lib;
 /// </summary>
 public class SaveParserEU4
 {
-    protected readonly Stream _saveFile;
-    private readonly BufferedEnumerator<Token> _tokens;
-    private readonly ObjectPool<StringBuilder> _stringBuilderPool;
+    private const int DefaultBufferSize = 64 * 1024;
+    private static ReadOnlySpan<byte> Header => "EU4txt"u8;
+
+    /// <summary>
+    ///     Windows-1252 is the default encoding of the save files
+    /// </summary>
+    public static Encoding DefaultEncoding => ParadoxEncodings.Windows1252;
+
+    private readonly Tokenizer _tokens;
+    private readonly SearchExpressionCompilation? _query;
+
+    // step of the query the parser is currently at, compiled by Parse()
     private ISearchExpression? _searchExprCurrent;
 
+    /// <summary>
+    ///     Encoding of the strings inside the save. Saves of different localizations use
+    ///     different ones, so it can be changed; <see cref="DefaultEncoding" /> is what EU4 writes.
+    /// </summary>
+    public Encoding Encoding { get; }
+
     /// <param name="savefile">
     ///     Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive
     /// </param>
     /// <param name="query">
-    ///     Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM,
+    ///     Parsing whole save may take a few seconds on mid pc and takes 1GB of RAM,
     ///     so you should specify what exactly you want to get from save file
     /// </param>
-    public SaveParserEU4(Stream savefile, ISearchExpression? query)
+    /// <param name="encoding">Encoding of the strings inside the save. <see cref="DefaultEncoding"/></param>
+    /// <param name="bufferSize">Size of the read buffer. Mostly useful for tests.</param>
+    public SaveParserEU4(Stream savefile, SearchExpressionCompilation? query,
+        Encoding? encoding = null, int bufferSize = DefaultBufferSize)
     {
-        _saveFile = savefile;
-        _searchExprCurrent = query;
-        const int tokenBufSize = 5;
-        _tokens = new BufferedEnumerator<Token>(LexTextSave(), tokenBufSize);
-        _stringBuilderPool = new DefaultObjectPool<StringBuilder>(
-            new StringBuilderPooledObjectPolicy
-            {
-                InitialCapacity = tokenBufSize * 13,
-                MaximumRetainedCapacity = tokenBufSize * 13,
-            });
+        Encoding = encoding ?? DefaultEncoding;
+        _query = query;
+        _tokens = new Tokenizer(savefile, bufferSize);
     }
 
-    protected IEnumerator<Token> LexTextSave()
+    public Dictionary<string, object> Parse()
     {
-        string expectedHeader = "EU4txt";
-        byte[] headBytes = new byte[expectedHeader.Length];
-        _saveFile.ReadExactly(headBytes);
-        string headStr = Encoding.UTF8.GetString(headBytes);
-        if (headStr != expectedHeader)
-            throw new Exception($"Invalid gamestate header. Expected '{expectedHeader}', got '{headStr}'.");
-
-        StringBuilder strb = _stringBuilderPool.Get();
-        int line = 2;
-        int column = 0;
-        bool isQuoteOpen = false;
-        bool isStrInQuotes = false;
-        Token strToken = new()
-        {
-            type = TokenType.Invalid,
-            column = -1,
-            line = -1,
-            value = null,
-        };
-
-        bool TryCompleteStringToken()
-        {
-            if (isQuoteOpen)
-                return false;
-
-            // strings in quotes may be empty
-            if (!isStrInQuotes && (strb.Length <= 0 || strb[0] == '#'))
-                return false;
-
-            strToken = new Token
-            {
-                type = TokenType.StringOrNumber,
-                column = (short)(column - strb.Length),
-                line = line,
-                value = strb,
-            };
-            strb = _stringBuilderPool.Get();
-            isStrInQuotes = false;
-            return true;
+        // compiled here, so the keys of the query are always encoded like the save
+        _searchExprCurrent = _query?.Compile(Encoding);
+        _tokens.ReadHeader(Header, Encoding);
+        return ParseDict();
     }
 
-        while (_saveFile.CanRead)
-        {
-            int c = _saveFile.ReadByte();
-            column++;
-            switch (c)
-            {
-                case -1:
-                    if (TryCompleteStringToken())
-                        yield return strToken;
-                    _stringBuilderPool.Return(strb);
-                    yield break;
-                case '\"':
-                    isQuoteOpen = !isQuoteOpen;
-                    isStrInQuotes = true;
-                    break;
-                case ' ':
-                case '\t':
-                case '\r':
-                    if (TryCompleteStringToken())
-                        yield return strToken;
-                    break;
-                case '\n':
-                    if (TryCompleteStringToken())
-                        yield return strToken;
-                    line++;
-                    column = 0;
-                    break;
-                case '=':
-                    if (TryCompleteStringToken())
-                        yield return strToken;
-                    yield return new Token
-                    {
-                        type = TokenType.Equals,
-                        line = line, column = (short)column
-                    };
-                    break;
-                case '{':
-                    if (TryCompleteStringToken())
-                        yield return strToken;
-                    yield return new Token
-                    {
-                        type = TokenType.BracketOpen,
-                        line = line, column = (short)column
-                    };
-                    break;
-                case '}':
-                    if (TryCompleteStringToken())
-                        yield return strToken;
-                    yield return new Token
-                    {
-                        type = TokenType.BracketClose,
-                        line = line, column = (short)column
-                    };
-                    break;
-                default:
-                    // Skip control characters, which are invisible and causing frontend bugs.
-                    // I dont know why there are so many of them in strings.
-                    if (c >= 0x20)
-                        strb.Append((char)c);
-                    break;
-            }
-        }
-
-        _stringBuilderPool.Return(strb);
-    }
-
-
     // doesn't move next
     private object? ParseValue()
     {
-        var tok = _tokens.Current.Value;
-        switch (tok.type)
+        switch (_tokens.Type)
         {
             case TokenType.StringOrNumber:
-                try
-                {
-                    // string values can be empty
-                    if (tok.value!.Length == 0)
-                        return string.Empty;
-                    if (tok.value.Equals("yes"))
-                        return true;
-                    if (tok.value.Equals("no"))
-                        return false;
-                    
-                    string tokStr = tok.value.ToString();
-                    if (tokStr[0] != '-' && !char.IsDigit(tokStr[0]))
-                        return tokStr;
-                    if (tokStr.Contains('.') && double.TryParse(tokStr, out double d))
-                        return d;
-                    if (long.TryParse(tokStr, out long l))
-                        return l;
-                    return tokStr;
-                }
-                finally
-                {
-                    _stringBuilderPool.Return(tok.value!);
-                }
+                return ParseScalar(_tokens.Text);
             case TokenType.BracketOpen:
-                object obj = ParseListOrDict();
-                return obj;
+                return ParseListOrDict();
             case TokenType.BracketClose:
                 return null;
             default:
-                throw new UnexpectedTokenException(tok);
+                throw new UnexpectedTokenException(_tokens, Encoding);
         }
     }
 
+    private object ParseScalar(ReadOnlySpan<byte> text)
+    {
+        // string values can be empty
+        if (text.Length == 0)
+            return string.Empty;
+        if (text.SequenceEqual("yes"u8))
+            return true;
+        if (text.SequenceEqual("no"u8))
+            return false;
+
+        byte first = text[0];
+        if (first != (byte)'-' && !char.IsAsciiDigit((char)first))
+            return DecodeString(text);
+        if (text.Contains((byte)'.')
+            && double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out double d))
+            return d;
+        if (long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out long l))
+            return l;
+        return DecodeString(text);
+    }
+
+    private string DecodeString(ReadOnlySpan<byte> text)
+    {
+        // Skip control characters, which are invisible and causing frontend bugs.
+        // I dont know why there are so many of them in strings.
+        if (!text.ContainsAnyInRange((byte)0, (byte)0x1F))
+            return Encoding.GetString(text);
+
+        Span<byte> cleaned = text.Length <= 256 ? stackalloc byte[text.Length] : new byte[text.Length];
+        int length = 0;
+        foreach (byte b in text)
+            if (b >= 0x20)
+                cleaned[length++] = b;
+        return Encoding.GetString(cleaned[..length]);
+    }
 
     // skips next value
     /// <returns>true if skipped value, false if current token is closing bracket</returns>
     private bool SkipValue()
     {
-        var tok = _tokens.Current.Value;
-        switch (tok.type)
+        switch (_tokens.Type)
         {
             case TokenType.BracketOpen:
-                SkipObject();
+                _tokens.SkipBlock();
                 return true;
             case TokenType.StringOrNumber:
-                _stringBuilderPool.Return(tok.value!);
                 return true;
             case TokenType.BracketClose:
                 return false;
             default:
-                throw new UnexpectedTokenException(tok);
-        }
-    }
-
-    // skips all tokens inside curly braces block
-    private void SkipObject(int bracketBalance = 1)
-    {
-        while (bracketBalance != 0 && _tokens.MoveNext())
-        {
-            var tok = _tokens.Current.Value;
-            if (tok.type == TokenType.BracketOpen)
-                bracketBalance++;
-            else if (tok.type == TokenType.BracketClose)
-                bracketBalance--;
-            else if (tok.type == TokenType.StringOrNumber)
-            {
-                _stringBuilderPool.Return(tok.value!);
-            }
+                throw new UnexpectedTokenException(_tokens, Encoding);
         }
     }
 
@@ -231,9 +132,7 @@ public class SaveParserEU4
     // doesn't move next
     private object ParseListOrDict()
     {
-        var first = _tokens.Current.Next;
-        var second = _tokens.Current.Next?.Next;
-        if (first?.Value.type == TokenType.StringOrNumber && second?.Value.type == TokenType.Equals)
+        if (_tokens.PeekType(1) == TokenType.StringOrNumber && _tokens.PeekType(2) == TokenType.Equals)
             return ParseDict();
 
         return ParseList();
@@ -245,17 +144,18 @@ public class SaveParserEU4
         List<object> list = new();
         for (int i = 0; ; i++)
         {
-            if (!_tokens.MoveNext())
+            if (!_tokens.Read())
                 throw new Exception("Unexpected end of file");
 
             ISearchExpression? searchExprNext = null;
             if (_searchExprCurrent != null
-                && !_searchExprCurrent.DoesMatch(new SearchArgs(i, string.Empty), out searchExprNext))
+                && !_searchExprCurrent.DoesMatch(new MatchCandidate(i), out searchExprNext))
             {
                 if (!SkipValue())
                     break;
                 continue;
             }
+
             var searchExprPrev = _searchExprCurrent;
             _searchExprCurrent = searchExprNext;
             object? value = ParseValue();
@@ -278,53 +178,54 @@ public class SaveParserEU4
     {
         Dictionary<string, object> dict = new();
 
-        // root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count
-        for (int localIndex = 0; _tokens.MoveNext(); localIndex++)
+        // root is a dict without closing bracket, so this method must check for end of file
+        for (int localIndex = 0; _tokens.Read(); localIndex++)
         {
-            var tok = _tokens.Current.Value;
             // end of dictionary
-            if (tok.type == TokenType.BracketClose)
+            if (_tokens.Type == TokenType.BracketClose)
                 break;
 
             // Saves may contain some blocks without key.
             // Such blocks are skipped because idk where to put them.
             // Example: `technology_group=tech_cannorian{ }
             //          { } { } { }`
-            if (tok.type == TokenType.BracketOpen)
+            if (_tokens.Type == TokenType.BracketOpen)
             {
-                SkipObject();
+                _tokens.SkipBlock();
                 continue;
             }
 
-            if (tok.type != TokenType.StringOrNumber)
-                throw new UnexpectedTokenException(tok);
+            if (_tokens.Type != TokenType.StringOrNumber)
+                throw new UnexpectedTokenException(_tokens, Encoding);
 
-            var keySB = tok.value!;
+            // The key is matched before the value is read, so that a key rejected by the query
+            // never has to become a string.
+            ISearchExpression? searchExprNext = null;
+            bool matches = _searchExprCurrent == null
+                           || _searchExprCurrent.DoesMatch(
+                               new MatchCandidate(localIndex, _tokens.Text), out searchExprNext);
+            string? keyStr = matches ? DecodeString(_tokens.Text) : null;
 
             // next token should be `=` or `{`
-            if (!_tokens.MoveNext())
-                throw new UnexpectedTokenException(tok);
-            tok = _tokens.Current.Value;
-            if (tok.type == TokenType.Equals)
+            if (!_tokens.Read())
+                throw new UnexpectedTokenException(_tokens, Encoding);
+            if (_tokens.Type == TokenType.Equals)
             {
                 // skip `=`
-                if (!_tokens.MoveNext())
-                    throw new UnexpectedTokenException(tok);
+                if (!_tokens.Read())
+                    throw new UnexpectedTokenException(_tokens, Encoding);
             }
             // Saves may contain object definition without `=`.
             // Example: `map_area_data {` instead of `map_area_data = {`
-            else if (tok.type != TokenType.BracketOpen)
+            else if (_tokens.Type != TokenType.BracketOpen)
             {
-                throw new UnexpectedTokenException(tok);
+                throw new UnexpectedTokenException(_tokens, Encoding);
             }
 
-            ISearchExpression? searchExprNext = null;
-            if (_searchExprCurrent != null
-                && !_searchExprCurrent.DoesMatch(new SearchArgs(localIndex, keySB), out searchExprNext))
+            if (!matches)
             {
                 if (!SkipValue())
-                    throw new UnexpectedTokenException(_tokens.Current.Value);
-                _stringBuilderPool.Return(keySB);
+                    throw new UnexpectedTokenException(_tokens, Encoding);
                 continue;
             }
 
@@ -332,17 +233,14 @@ public class SaveParserEU4
             _searchExprCurrent = searchExprNext;
             object? value = ParseValue();
             if (value is null)
-                throw new UnexpectedTokenException(_tokens.Current.Value);
+                throw new UnexpectedTokenException(_tokens, Encoding);
             _searchExprCurrent = searExpressionPrevious;
 
-            string keyStr = keySB.ToString();
-            _stringBuilderPool.Return(keySB);
-            
             // Paradox save format has another way of defining list:
             // a = 1
             // a = 2
             // It means `a = { 1 2 }`
-            if (dict.TryGetValue(keyStr, out var firstValue))
+            if (dict.TryGetValue(keyStr!, out var firstValue))
             {
                 // Do dot add empty collections into list.
                 // `key:{}` is okay, but i don't want to see `key:[{},{},{},{},{},{}]`
@@ -351,73 +249,21 @@ public class SaveParserEU4
 
                 if (firstValue is List<object> existingList)
                     existingList.Add(value);
-                else dict[keyStr] = new List<object> { firstValue, value };
+                else dict[keyStr!] = new List<object> { firstValue, value };
             }
             else
             {
-                dict.Add(keyStr, value);
+                dict.Add(keyStr!, value);
             }
         }
 
         return dict;
     }
 
-    public Dictionary<string, object> Parse()
+    internal class UnexpectedTokenException : Exception
     {
-        var root = ParseDict();
-        return root;
-    }
-
-    protected enum TokenType : byte
-    {
-        Invalid,
-        StringOrNumber,
-        Equals,
-        BracketOpen,
-        BracketClose
-    }
-
-    protected struct Token
-    {
-        public required TokenType type;
-        public required short column;
-        public required int line;
-        public StringBuilder? value;
-
-        public override string ToString()
-        {
-            string s;
-            switch (type)
-            {
-                case TokenType.Invalid:
-                    s = "INVALID_TOKEN";
-                    break;
-                case TokenType.StringOrNumber:
-                    if (value == null || value.Length == 0)
-                        s = "NULL";
-                    else s = value.ToString();
-                    break;
-                case TokenType.Equals:
-                    s = "=";
-                    break;
-                case TokenType.BracketOpen:
-                    s = "{";
-                    break;
-                case TokenType.BracketClose:
-                    s = "}";
-                    break;
-                default:
-                    throw new ArgumentOutOfRangeException(type.ToString());
-            }
-
-            return $"{line}:{column} '{s}'";
-        }
-    }
-
-    protected class UnexpectedTokenException : Exception
-    {
-        public UnexpectedTokenException(Token token) :
-            base($"Unexpected token: {token}")
+        public UnexpectedTokenException(Tokenizer tokens, Encoding encoding) :
+            base($"Unexpected token: {tokens.Line}:{tokens.Column} '{tokens.Describe(encoding)}'")
         {
         }
     }

ParadoxSaveParser.Lib/SearchExpression.cs

@@ -1,169 +0,0 @@
-namespace ParadoxSaveParser.Lib;
-
-public readonly record struct SearchArgs
-{
-    public readonly string KeyStr;
-    public readonly StringBuilder? KeySB;
-    public readonly int LocalIndex;
-
-    public SearchArgs(int localIndex, string keyStr)
-    {
-        KeyStr = keyStr;
-        KeySB = null;
-        LocalIndex = localIndex;
-    }
-    
-    public SearchArgs(int localIndex, StringBuilder keySb)
-    {
-        KeyStr = string.Empty;
-        KeySB = keySb;
-        LocalIndex = localIndex;
-    }
-}
-
-public interface ISearchExpression
-{
-    bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression);
-}
-
-public static class SearchExpressionCompiler
-{
-    private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan<char> chars, int i)
-        => chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\');
-
-    public static ISearchExpression Compile(ReadOnlySpan<char> query)
-    {
-        if (query.IsEmpty)
-            throw new ArgumentNullException(nameof(query));
-
-        if (query[0] is '(')
-        {
-            var subExprs = new List<ISearchExpression>();
-            int supExprBegin = 1;
-            int bracketBalance = 1;
-            int i = supExprBegin;
-            for (; i < query.Length && bracketBalance != 0; i++)
-            {
-                if (CharEqualsAndNotEscaped('(', query, i))
-                {
-                    bracketBalance++;
-                }
-                else if (CharEqualsAndNotEscaped(')', query, i))
-                {
-                    bracketBalance--;
-                }
-                else if (bracketBalance == 1 && CharEqualsAndNotEscaped('|', query, i))
-                {
-                    var subPart = query.Slice(supExprBegin, i - supExprBegin);
-                    var subExpr = Compile(subPart);
-                    subExprs.Add(subExpr);
-                    supExprBegin = i + 1;
-                }
-            }
-
-            if (i != query.Length)
-                throw new NotImplementedException("Expressions after ')' are not supported");
-
-            if (bracketBalance > 0)
-                throw new Exception("Too many opening brackets");
-            if (bracketBalance < 0)
-                throw new Exception("Too many closing brackets");
-
-            var subPartLast = query.Slice(supExprBegin, i - 1 - supExprBegin);
-            var subExprLast = Compile(subPartLast);
-            subExprs.Add(subExprLast);
-            return new MultipleMatchExpression(subExprs);
-        }
-
-        int partBeforePointLength = 0;
-        while (partBeforePointLength < query.Length)
-        {
-            if (CharEqualsAndNotEscaped('.', query, partBeforePointLength))
-                break;
-            partBeforePointLength++;
-        }
-
-        var part = query.Slice(0, partBeforePointLength);
-        ReadOnlySpan<char> remaining = default;
-        if (partBeforePointLength < query.Length)
-            remaining = query.Slice(partBeforePointLength + 1);
-        if (part is "*") 
-            return new AnyMatchExpression(remaining.IsEmpty ? null : Compile(remaining));
-        if (part is "~")
-            return new NoMatchExpression();
-        
-        for (int j = 0; j < part.Length; j++)
-            if (CharEqualsAndNotEscaped('*', part, j))
-                throw new NotImplementedException("pattern matching other than '*' is not implemented yet");
-
-        if (part[0] is '[')
-        {
-            part = part.Slice(1, part.Length - 2);
-            return new IndexMatchExpression(int.Parse(part), remaining.IsEmpty ? null : Compile(remaining));
-        }
-
-        return new ExactMatchExpression(part.ToString(), remaining.IsEmpty ? null : Compile(remaining));
-    }
-
-
-    private record AnyMatchExpression(ISearchExpression? next) : ISearchExpression
-    {
-        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
-        {
-            nextSearchExpression = next;
-            return true;
-        }
-    }
-
-    private record NoMatchExpression : ISearchExpression
-    {
-        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
-        {
-            nextSearchExpression = null;
-            return false;
-        }
-    }
-
-    private record MultipleMatchExpression(List<ISearchExpression> subExprs) : ISearchExpression
-    {
-        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
-        {
-            foreach (var e in subExprs)
-                if (e.DoesMatch(args, out nextSearchExpression))
-                    return true;
-
-            nextSearchExpression = null;
-            return false;
-        }
-    }
-
-    private record IndexMatchExpression(int index, ISearchExpression? next) : ISearchExpression
-    {
-        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
-        {
-            if (args.LocalIndex == index)
-            {
-                nextSearchExpression = next;
-                return true;
-            }
-
-            nextSearchExpression = null;
-            return false;
-        }
-    }
-
-    private record ExactMatchExpression(string key, ISearchExpression? next) : ISearchExpression
-    {
-        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
-        {
-            if ((args.KeySB != null && args.KeySB.Equals(key)) || args.KeyStr == key)
-            {
-                nextSearchExpression = next;
-                return true;
-            }
-
-            nextSearchExpression = null;
-            return false;
-        }
-    }
-}

ParadoxSaveParser.Lib/SearchExpression/AnyMatchExpression.cs

@@ -0,0 +1,13 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>"*" — matches every node at this level.</summary>
+internal sealed class AnyMatchExpression(ISearchExpression? next) : ISearchExpression
+{
+    public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
+    {
+        nextSearchExpression = next;
+        return true;
+    }
+
+    public void ReEncode(Encoding encoding) => next?.ReEncode(encoding);
+}

ParadoxSaveParser.Lib/SearchExpression/ExactMatchExpression.cs

@@ -0,0 +1,37 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>A literal key, matched byte for byte.</summary>
+internal sealed class ExactMatchExpression : ISearchExpression
+{
+    public string Key { get; }
+
+    // the key as it appears in the save, so matching needs no encoding at all
+    public byte[] KeyBytes { get; private set; }
+    public ISearchExpression? Next { get; }
+
+    public ExactMatchExpression(string key, Encoding encoding, ISearchExpression? next)
+    {
+        Key = key;
+        Next = next;
+        KeyBytes = encoding.GetBytes(key);
+    }
+
+    public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
+    {
+        // comparing bytes keeps the candidate's key from becoming a string
+        if (candidate.Key.SequenceEqual(KeyBytes))
+        {
+            nextSearchExpression = Next;
+            return true;
+        }
+
+        nextSearchExpression = null;
+        return false;
+    }
+
+    public void ReEncode(Encoding encoding)
+    {
+        KeyBytes = encoding.GetBytes(Key);
+        Next?.ReEncode(encoding);
+    }
+}

ParadoxSaveParser.Lib/SearchExpression/ISearchExpression.cs

@@ -0,0 +1,50 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>
+///     One step of a compiled query. Each step points at the step its matched nodes
+///     are searched with, so the query runs in a single pass down the tree.
+/// </summary>
+public interface ISearchExpression
+{
+    /// <summary>Tests one node against this step of the query.</summary>
+    /// <param name="candidate">The node being tested, identified by its key or its position.</param>
+    /// <param name="nextSearchExpression">
+    ///     On a match, the step for the node's children, or <c>null</c> if the node is kept whole.
+    ///     Undefined when the method returns <c>false</c>.
+    /// </param>
+    /// <returns>true if the node is selected by this step</returns>
+    bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression);
+
+    /// <summary>
+    ///     Encodes the literal keys of this step and of every step below it.
+    ///     Called by <see cref="SearchExpressionCompilation" />, never during matching.
+    /// </summary>
+    void ReEncode(Encoding encoding);
+}
+
+/// <summary>
+///     A node the query is tested against: its raw key and its position in the parent.
+///     The key stays undecoded, so a rejected key never becomes a string.
+/// </summary>
+public readonly ref struct MatchCandidate
+{
+    /// <summary>Undecoded key of the node. Empty for list items.</summary>
+    public readonly ReadOnlySpan<byte> Key;
+
+    /// <summary>Position of the node among its siblings, counted from 0.</summary>
+    public readonly int Index;
+
+    /// <summary>A list item, which has a position but no key.</summary>
+    public MatchCandidate(int index)
+    {
+        Key = default;
+        Index = index;
+    }
+
+    /// <summary>A dictionary entry, which has both a key and a position.</summary>
+    public MatchCandidate(int index, ReadOnlySpan<byte> key)
+    {
+        Key = key;
+        Index = index;
+    }
+}

ParadoxSaveParser.Lib/SearchExpression/IndexMatchExpression.cs

@@ -0,0 +1,19 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>"[7]" — matches the node at one position.</summary>
+internal sealed class IndexMatchExpression(int index, ISearchExpression? next) : ISearchExpression
+{
+    public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
+    {
+        if (candidate.Index == index)
+        {
+            nextSearchExpression = next;
+            return true;
+        }
+
+        nextSearchExpression = null;
+        return false;
+    }
+
+    public void ReEncode(Encoding encoding) => next?.ReEncode(encoding);
+}

ParadoxSaveParser.Lib/SearchExpression/MultipleMatchExpression.cs

@@ -0,0 +1,22 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>"(a|b)" — matches if any alternative does.</summary>
+internal sealed class MultipleMatchExpression(List<ISearchExpression> subExprs) : ISearchExpression
+{
+    public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
+    {
+        // the first matching alternative wins
+        foreach (var e in subExprs)
+            if (e.DoesMatch(candidate, out nextSearchExpression))
+                return true;
+
+        nextSearchExpression = null;
+        return false;
+    }
+
+    public void ReEncode(Encoding encoding)
+    {
+        foreach (var e in subExprs)
+            e.ReEncode(encoding);
+    }
+}

ParadoxSaveParser.Lib/SearchExpression/NoMatchExpression.cs

@@ -0,0 +1,23 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>"~" — matches nothing, so the branch is skipped.</summary>
+internal sealed class NoMatchExpression : ISearchExpression
+{
+    // stateless, so one instance serves every "~" in every query
+    public static readonly NoMatchExpression Instance = new();
+
+    private NoMatchExpression()
+    {
+    }
+
+    public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
+    {
+        nextSearchExpression = null;
+        return false;
+    }
+
+    // no keys and no children, and shared between queries, so nothing may be stored here
+    public void ReEncode(Encoding encoding)
+    {
+    }
+}

ParadoxSaveParser.Lib/SearchExpression/SearchExpressionCompilation.cs

@@ -0,0 +1,198 @@
+using System.Threading;
+
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>Thrown when a query cannot be compiled because it is malformed or uses unsupported syntax.</summary>
+public class SearchExpressionException : Exception
+{
+    public SearchExpressionException(string message, ReadOnlySpan<char> part)
+        : base($"{message}: '{part}'")
+    {
+    }
+}
+
+/// <summary>
+///     A query such as <c>countries.*.technology</c>, and the expression tree compiled from it.
+///     The tree is built on the first <see cref="Compile" /> and reused afterwards; only a change
+///     of encoding touches it again, and then only to re-encode its literal keys.
+/// </summary>
+public class SearchExpressionCompilation
+{
+    private readonly string _query;
+
+    // guards the tree while it is built or re-encoded, so parsers on several threads
+    // can share one compilation
+    private readonly Lock _compileLock = new();
+
+    private ISearchExpression? _rootNode;
+    private Encoding? _currentEncoding;
+
+    /// <param name="query">Search query. Compiled on the first call to <see cref="Compile" />.</param>
+    /// <exception cref="ArgumentException">the query is empty</exception>
+    public SearchExpressionCompilation(string query)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(query);
+        _query = query;
+    }
+
+    /// <summary>
+    ///     Returns the expression tree, with its literal keys encoded the way the save is.
+    ///     Compiles the query on the first call, and re-encodes the tree when the encoding changes.
+    /// </summary>
+    /// <param name="encoding">Encoding of the strings inside the save being parsed.</param>
+    /// <exception cref="SearchExpressionException">the query is malformed</exception>
+    public ISearchExpression Compile(Encoding encoding)
+    {
+        lock (_compileLock)
+        {
+            if (_rootNode is null)
+                _rootNode = CompilePart(_query, encoding, null);
+            // keys are stored as bytes, so another encoding means encoding them again
+            else if (!encoding.Equals(_currentEncoding)) 
+                _rootNode.ReEncode(encoding);
+
+            _currentEncoding = encoding;
+            return _rootNode;
+        }
+    }
+
+    /// <summary>True if <paramref name="chars" /> holds <paramref name="c" /> at
+    /// <paramref name="i" /> as syntax, not as an escaped literal.</summary>
+    private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan<char> chars, int i)
+        // escaped if a backslash stands in either of the two preceding positions
+        => chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\');
+
+    /// <summary>Compiles one path step together with everything that follows it.</summary>
+    /// <param name="tail">
+    ///     Step that continues the path after this one ends, or <c>null</c> if nothing follows.
+    ///     Used for the part written after a group, as in <c>(a|b).c</c>, where every alternative
+    ///     of the group continues with the same <c>.c</c>.
+    /// </param>
+    private static ISearchExpression CompilePart(ReadOnlySpan<char> query, Encoding encoding,
+        ISearchExpression? tail)
+    {
+        // empty alternative, as in "(a||b)"
+        if (query.IsEmpty)
+            throw new SearchExpressionException("Empty expression", query);
+
+        // "(a|b.c)" — a group of alternatives
+        if (query[0] is '(')
+            return CompileGroup(query, encoding, tail);
+
+        // a plain step: find the '.' that ends it
+        int partBeforePointLength = 0;
+        while (partBeforePointLength < query.Length)
+        {
+            if (CharEqualsAndNotEscaped('.', query, partBeforePointLength))
+                break;
+            partBeforePointLength++;
+        }
+
+        // this step, and the rest of the path that becomes its child expression
+        var part = query.Slice(0, partBeforePointLength);
+        // empty step, as in "a..b" or ".b"
+        if (part.IsEmpty)
+            throw new SearchExpressionException("Empty path step", query);
+
+        ReadOnlySpan<char> remaining = default;
+        if (partBeforePointLength < query.Length)
+        {
+            remaining = query.Slice(partBeforePointLength + 1);
+            if (remaining.IsEmpty)
+                throw new SearchExpressionException("Path ends with '.'", query);
+        }
+
+        // when nothing follows this step, the path continues with the tail, which may be null too
+        var next = remaining.IsEmpty ? tail : CompilePart(remaining, encoding, tail);
+
+        // "*" — any node at this level
+        if (part is "*")
+            return new AnyMatchExpression(next);
+        // "~" — nothing at this level
+        if (part is "~")
+            return NoMatchExpression.Instance;
+
+        // a '*' anywhere else would be a wildcard inside a key
+        for (int j = 0; j < part.Length; j++)
+            if (CharEqualsAndNotEscaped('*', part, j))
+                throw new SearchExpressionException("Pattern matching other than '*' is not supported", part);
+
+        // "[7]" — match by position
+        if (part[0] is '[')
+        {
+            if (part[^1] is not ']')
+                throw new SearchExpressionException("Index step has no closing ']'", part);
+            // strip the brackets
+            var indexText = part.Slice(1, part.Length - 2);
+            if (!int.TryParse(indexText, out int index) || index < 0)
+                throw new SearchExpressionException("Index must be a non-negative number", part);
+            return new IndexMatchExpression(index, next);
+        }
+
+        // an ordinary key
+        return new ExactMatchExpression(part.ToString(), encoding, next);
+    }
+
+    /// <summary>Compiles "(a|b)", and the path written after it, if there is one.</summary>
+    private static ISearchExpression CompileGroup(ReadOnlySpan<char> query, Encoding encoding,
+        ISearchExpression? tail)
+    {
+        int close = FindGroupEnd(query);
+
+        // "(a|b).c" — what stands after the group continues every alternative of it
+        var afterGroup = query.Slice(close + 1);
+        if (!afterGroup.IsEmpty)
+        {
+            if (CharEqualsAndNotEscaped(')', afterGroup, 0))
+                throw new SearchExpressionException("Too many closing brackets", query);
+            if (!CharEqualsAndNotEscaped('.', afterGroup, 0))
+                throw new SearchExpressionException("Group must be followed by '.'", query);
+
+            var rest = afterGroup.Slice(1);
+            if (rest.IsEmpty)
+                throw new SearchExpressionException("Path ends with '.'", query);
+            tail = CompilePart(rest, encoding, tail);
+        }
+
+        // cut the group into alternatives and compile each of them with that same tail
+        var subExprs = new List<ISearchExpression>();
+        int begin = 1;
+        int depth = 0;
+        for (int i = 1; i < close; i++)
+        {
+            if (CharEqualsAndNotEscaped('(', query, i))
+                depth++;
+            else if (CharEqualsAndNotEscaped(')', query, i))
+                depth--;
+            // a deeper '|' belongs to a nested group and is compiled with it
+            else if (depth == 0 && CharEqualsAndNotEscaped('|', query, i))
+            {
+                subExprs.Add(CompilePart(query.Slice(begin, i - begin), encoding, tail));
+                begin = i + 1;
+            }
+        }
+
+        // the last alternative has no '|' after it
+        subExprs.Add(CompilePart(query.Slice(begin, close - begin), encoding, tail));
+        return new MultipleMatchExpression(subExprs);
+    }
+
+    /// <summary>Finds the ')' that closes the group opened by the first character.</summary>
+    private static int FindGroupEnd(ReadOnlySpan<char> query)
+    {
+        int depth = 0;
+        for (int i = 0; i < query.Length; i++)
+        {
+            if (CharEqualsAndNotEscaped('(', query, i))
+                depth++;
+            else if (CharEqualsAndNotEscaped(')', query, i))
+            {
+                depth--;
+                if (depth == 0)
+                    return i;
+            }
+        }
+
+        throw new SearchExpressionException("Too many opening brackets", query);
+    }
+}

ParadoxSaveParser.Lib/Tokenizer.cs

@@ -0,0 +1,456 @@
+using System.Buffers;
+
+namespace ParadoxSaveParser.Lib;
+
+internal enum TokenType : byte
+{
+    // default value, so a slot that was never scanned is recognizably empty
+    Invalid,
+    // any bare or quoted value: the tokenizer does not tell numbers from strings
+    StringOrNumber,
+    Equals,
+    BracketOpen,
+    BracketClose,
+    EndOfFile
+}
+
+/// <summary>
+///     Splits a Paradox text save into tokens, reading the stream through a reusable buffer.
+///     Token text is exposed as a span into that buffer, so a token costs no allocation at all
+///     unless it happens to straddle a buffer refill.
+///     <para>
+///         The parser can tell the tokenizer to throw away a whole <c>{...}</c> block with
+///         <see cref="SkipBlock" />. Blocks rejected by the search expression are then never
+///         turned into tokens or strings, which is what makes a query cheaper than a full parse.
+///     </para>
+/// </summary>
+internal sealed class Tokenizer
+{
+    /// <summary>Bytes that end an unquoted token.</summary>
+    private static readonly SearchValues<byte> TokenEnd = SearchValues.Create(" \t\r\n={}\""u8);
+
+    private static readonly SearchValues<byte> Whitespace = SearchValues.Create(" \t\r\n"u8);
+
+    /// <summary>Everything <see cref="SkipBlock" /> has to look at while counting depth.</summary>
+    private static readonly SearchValues<byte> BlockChars = SearchValues.Create("{}\""u8);
+
+    /// <summary>One scanned token. Reused forever, so scanning a token allocates nothing.</summary>
+    private sealed class Slot
+    {
+        public TokenType Type;
+        // position of the token's first byte in the file, for error messages
+        public int Line;
+        public short Column;
+
+        // text is either a range of the shared buffer, or, once a refill would overwrite it,
+        // a copy in Scratch
+        public int Start;
+        public int Length;
+        // grown on demand and kept between tokens, so long tokens stop reallocating after a while
+        public byte[] Scratch = [];
+        public bool InScratch;
+    }
+
+    private readonly Stream _stream;
+    private readonly byte[] _buffer;
+
+    // current token plus up to two lookahead tokens
+    private readonly Slot[] _slots = [new Slot(), new Slot(), new Slot()];
+    // index of the current token; the slots are used as a ring, so the array is never shifted
+    private int _current;
+    // how many slots after _current already hold a scanned, not yet consumed token
+    private int _lookahead;
+
+    // read position and amount of valid data in _buffer
+    private int _pos;
+    private int _length;
+    // set once the stream has no more bytes; stops Fill() from calling Read() again
+    private bool _eof;
+    // position of _pos in the file; _column is 0-based here and reported 1-based
+    private int _line = 1;
+    private int _column;
+
+    public Tokenizer(Stream stream, int bufferSize)
+    {
+        if (bufferSize < 16)
+            throw new ArgumentOutOfRangeException(nameof(bufferSize), bufferSize, "buffer is too small");
+        _stream = stream;
+        _buffer = new byte[bufferSize];
+    }
+
+    public TokenType Type => _slots[_current].Type;
+    public int Line => _slots[_current].Line;
+    public short Column => _slots[_current].Column;
+
+    /// <summary>
+    ///     Bytes of the current token, without quotes. Valid only until the next
+    ///     <see cref="Read" />, because the buffer underneath it gets reused.
+    /// </summary>
+    public ReadOnlySpan<byte> Text
+    {
+        get
+        {
+            var slot = _slots[_current];
+            // a token that survived a refill was copied out; everything else still points into the buffer
+            return slot.InScratch
+                ? slot.Scratch.AsSpan(0, slot.Length)
+                : _buffer.AsSpan(slot.Start, slot.Length);
+        }
+    }
+
+    /// <summary>Consumes the file's magic header and checks it, before any token is scanned.</summary>
+    public void ReadHeader(ReadOnlySpan<byte> expected, Encoding encoding)
+    {
+        // read straight from the stream: this runs before the buffer holds anything
+        Span<byte> head = stackalloc byte[expected.Length];
+        _stream.ReadExactly(head);
+        if (!head.SequenceEqual(expected))
+            throw new Exception($"Invalid gamestate header. " +
+                                $"Expected '{encoding.GetString(expected)}', got '{encoding.GetString(head)}'.");
+    }
+
+    /// <returns>false at end of file</returns>
+    public bool Read()
+    {
+        _current = NextSlot(_current);
+        // the next slot may already be filled by an earlier PeekType, then there is nothing to scan
+        if (_lookahead > 0)
+            _lookahead--;
+        else Scan(_slots[_current]);
+        return _slots[_current].Type != TokenType.EndOfFile;
+    }
+
+    /// <summary>
+    ///     Type of a token that has not been consumed yet, 1 or 2 tokens ahead of the current one.
+    ///     Only types are available: the parser never needs the text of a token it has not reached.
+    /// </summary>
+    public TokenType PeekType(int offset)
+    {
+        // scan only as far as asked, so lookahead never runs into a block SkipBlock is about to drop
+        while (_lookahead < offset)
+        {
+            // first slot after the ones that are already filled
+            int slot = _current;
+            for (int i = 0; i <= _lookahead; i++)
+                slot = NextSlot(slot);
+            Scan(_slots[slot]);
+            _lookahead++;
+        }
+
+        int index = _current;
+        for (int i = 0; i < offset; i++)
+            index = NextSlot(index);
+        return _slots[index].Type;
+    }
+
+    /// <summary>
+    ///     Throws away the block opened by the current <c>{</c> token without tokenizing it:
+    ///     raw bytes are scanned for braces until the depth returns to zero.
+    ///     Braces inside quoted strings are text and do not change the depth.
+    ///     Leaves the closing <c>}</c> as the current token.
+    /// </summary>
+    public void SkipBlock()
+    {
+        int depth = 1;
+
+        // tokens that lookahead already pulled out of the buffer still count towards the depth
+        while (depth > 0 && _lookahead > 0)
+        {
+            _current = NextSlot(_current);
+            _lookahead--;
+            switch (_slots[_current].Type)
+            {
+                case TokenType.BracketOpen:
+                    depth++;
+                    break;
+                case TokenType.BracketClose:
+                    depth--;
+                    break;
+                case TokenType.EndOfFile:
+                    return;
+            }
+        }
+
+        bool inQuotes = false;
+        while (depth > 0)
+        {
+            if (_pos >= _length && !Fill())
+                break; // unbalanced braces: the file ended inside the block
+
+            var span = _buffer.AsSpan(_pos, _length - _pos);
+            // inside a string only the closing quote matters, braces there are ordinary characters
+            int i = inQuotes ? span.IndexOf((byte)'\"') : span.IndexOfAny(BlockChars);
+            if (i < 0)
+            {
+                // nothing interesting in this bufferful, drop all of it and refill
+                Consume(span.Length);
+                continue;
+            }
+
+            byte b = span[i];
+            Consume(i + 1);
+            if (b == (byte)'\"')
+                inQuotes = !inQuotes;
+            else if (b == (byte)'{')
+                depth++;
+            else depth--;
+        }
+
+        // hand the parser the closing brace it expects, without having tokenized anything inside
+        var current = _slots[_current];
+        current.Type = depth == 0 ? TokenType.BracketClose : TokenType.EndOfFile;
+        current.Length = 0;
+        current.InScratch = false;
+        current.Line = _line;
+        current.Column = (short)_column;
+    }
+
+    public string Describe(Encoding encoding) => Type switch
+    {
+        TokenType.StringOrNumber => Text.Length == 0 ? "NULL" : encoding.GetString(Text),
+        TokenType.Equals => "=",
+        TokenType.BracketOpen => "{",
+        TokenType.BracketClose => "}",
+        TokenType.EndOfFile => "END_OF_FILE",
+        _ => "INVALID_TOKEN",
+    };
+
+    /// <summary>Slot indices wrap around: the three slots form a ring buffer.</summary>
+    private int NextSlot(int i) => i + 1 == _slots.Length ? 0 : i + 1;
+
+    /// <summary>Reads the next token from the stream into <paramref name="slot" />.</summary>
+    private void Scan(Slot slot)
+    {
+        // loops only to skip comments, which produce no token
+        while (true)
+        {
+            if (!SkipWhitespace())
+            {
+                slot.Type = TokenType.EndOfFile;
+                slot.Length = 0;
+                slot.InScratch = false;
+                slot.Line = _line;
+                slot.Column = (short)_column;
+                return;
+            }
+
+            slot.Line = _line;
+            slot.Column = (short)(_column + 1); // columns are reported 1-based
+            // SkipWhitespace guarantees at least one buffered byte here
+            byte b = _buffer[_pos];
+            switch (b)
+            {
+                case (byte)'=':
+                    ConsumeFlat(1);
+                    SetDelimiter(slot, TokenType.Equals);
+                    return;
+                case (byte)'{':
+                    ConsumeFlat(1);
+                    SetDelimiter(slot, TokenType.BracketOpen);
+                    return;
+                case (byte)'}':
+                    ConsumeFlat(1);
+                    SetDelimiter(slot, TokenType.BracketClose);
+                    return;
+                case (byte)'\"':
+                    ReadQuoted(slot);
+                    return;
+                default:
+                    ReadBare(slot);
+                    // comments are dropped, same as before: a token starting with '#' is not emitted
+                    if (slot.Length > 0 && FirstByte(slot) == (byte)'#')
+                        continue;
+                    return;
+            }
+        }
+    }
+
+    /// <summary>Single-character tokens carry no text of their own.</summary>
+    private static void SetDelimiter(Slot slot, TokenType type)
+    {
+        slot.Type = type;
+        slot.Length = 0;
+        slot.InScratch = false;
+    }
+
+    /// <summary>Reads an unquoted token, which ends at the first delimiter byte.</summary>
+    private void ReadBare(Slot slot)
+    {
+        StartText(slot);
+        while (true)
+        {
+            var span = _buffer.AsSpan(_pos, _length - _pos);
+            // one vectorized search replaces a loop over the token's bytes
+            int end = span.IndexOfAny(TokenEnd);
+            if (end >= 0)
+            {
+                // the delimiter itself is left for the next Scan to classify
+                AppendText(slot, span[..end]);
+                ConsumeFlat(end);
+                return;
+            }
+
+            // token runs to the end of the buffer and continues in the next one
+            AppendText(slot, span);
+            ConsumeFlat(span.Length);
+            if (!Fill())
+                return;
+        }
+    }
+
+    /// <summary>
+    ///     Reads a quoted token. Everything up to the next <c>"</c> is text, braces included;
+    ///     the format has no escape sequences, so a quote always closes the string.
+    /// </summary>
+    private void ReadQuoted(Slot slot)
+    {
+        ConsumeFlat(1); // opening quote
+        StartText(slot);
+        while (true)
+        {
+            if (_pos >= _length && !Fill())
+                return; // unterminated string at end of file
+
+            var span = _buffer.AsSpan(_pos, _length - _pos);
+            int end = span.IndexOf((byte)'\"');
+            if (end >= 0)
+            {
+                AppendText(slot, span[..end]);
+                Consume(end + 1); // content plus the closing quote
+                return;
+            }
+
+            // Consume, not ConsumeFlat: a quoted value is allowed to span several lines
+            AppendText(slot, span);
+            Consume(span.Length);
+        }
+    }
+
+    /// <summary>Begins a text token at the current read position.</summary>
+    private void StartText(Slot slot)
+    {
+        slot.Type = TokenType.StringOrNumber;
+        slot.Start = _pos;
+        slot.Length = 0;
+        slot.InScratch = false;
+    }
+
+    /// <summary>Extends a text token by one chunk of bytes taken from the buffer.</summary>
+    private void AppendText(Slot slot, ReadOnlySpan<byte> chunk)
+    {
+        if (!slot.InScratch)
+        {
+            // chunks always continue where the previous one ended, so the range just grows
+            // still contiguous in the buffer, nothing to copy
+            if (slot.Length == 0)
+                slot.Start = _pos;
+            slot.Length += chunk.Length;
+            return;
+        }
+
+        // the start of this token is already out of the buffer, so the rest has to follow it
+        EnsureScratch(slot, slot.Length + chunk.Length);
+        chunk.CopyTo(slot.Scratch.AsSpan(slot.Length));
+        slot.Length += chunk.Length;
+    }
+
+    /// <summary>First byte of a token, wherever its text currently lives. Used to spot comments.</summary>
+    private byte FirstByte(Slot slot) => slot.InScratch ? slot.Scratch[0] : _buffer[slot.Start];
+
+    private static void EnsureScratch(Slot slot, int size)
+    {
+        if (slot.Scratch.Length >= size)
+            return;
+        // doubling keeps a token that is appended chunk by chunk from resizing on every chunk
+        int capacity = Math.Max(size, slot.Scratch.Length * 2);
+        Array.Resize(ref slot.Scratch, capacity);
+    }
+
+    /// <summary>Moves the read position onto the next non-whitespace byte.</summary>
+    /// <returns>false at end of file</returns>
+    private bool SkipWhitespace()
+    {
+        while (true)
+        {
+            if (_pos >= _length && !Fill())
+                return false;
+
+            var span = _buffer.AsSpan(_pos, _length - _pos);
+            int i = span.IndexOfAnyExcept(Whitespace);
+            if (i < 0)
+            {
+                // whitespace to the end of the buffer, keep going in the next one
+                Consume(span.Length);
+                continue;
+            }
+
+            if (i > 0)
+                Consume(i);
+            return true;
+        }
+    }
+
+    /// <summary>Refills the buffer from the stream.</summary>
+    /// <returns>false if the stream is exhausted</returns>
+    private bool Fill()
+    {
+        if (_eof)
+            return false;
+
+        // text of live tokens lives in the buffer, so it has to be copied out before overwriting
+        MaterializeSlots();
+        // a short read is fine, the next Fill picks up the rest
+        _length = _stream.Read(_buffer, 0, _buffer.Length);
+        _pos = 0;
+        if (_length > 0)
+            return true;
+
+        _eof = true;
+        _length = 0;
+        return false;
+    }
+
+    /// <summary>
+    ///     Copies every token that still points into the buffer out to its own scratch array.
+    ///     Called just before a refill, which is the only moment that text can be lost.
+    /// </summary>
+    private void MaterializeSlots()
+    {
+        foreach (var slot in _slots)
+        {
+            // delimiters and empty tokens own no text, and a copied one needs no second copy
+            if (slot.InScratch || slot.Length == 0 || slot.Type != TokenType.StringOrNumber)
+                continue;
+            EnsureScratch(slot, slot.Length);
+            _buffer.AsSpan(slot.Start, slot.Length).CopyTo(slot.Scratch);
+            slot.InScratch = true;
+        }
+    }
+
+    /// <summary>Consumes bytes that cannot contain a line break.</summary>
+    private void ConsumeFlat(int count)
+    {
+        _pos += count;
+        _column += count;
+    }
+
+    /// <summary>Consumes bytes that may contain line breaks, keeping line and column exact.</summary>
+    private void Consume(int count)
+    {
+        var slice = _buffer.AsSpan(_pos, count);
+        // the last break decides the column, and only then is counting all of them worth it
+        int lastBreak = slice.LastIndexOf((byte)'\n');
+        if (lastBreak < 0)
+        {
+            _column += count;
+        }
+        else
+        {
+            _line += slice.Count((byte)'\n');
+            // bytes left after the final break
+            _column = count - lastBreak - 1;
+        }
+
+        _pos += count;
+    }
+}

ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs

@@ -76,6 +76,7 @@ public class SaveParsingOperation
         Dictionary<string, object> parsedData;
         await using (var gamestateStream = PathHelper.CreateTempFile())
         {
+            // TODO: ckeck if it is zip archive or plain text like in Search.cs
             using (var zipArchive = ZipFile.Open(_meta.GetSaveFilePath().Str, ZipArchiveMode.Read))
             {
                 var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");

ParadoxSaveParser.WebAPI/Program.cs

@@ -40,7 +40,7 @@ public static class Program
         #endif
         
         new LaunchArgumentParser(
-                new LaunchArgument(["-d", "--debug"], 
+                new LaunchArgument(["d", "debug"], 
                     "enables debug log output to console",
                     () => IsDebug = true)
             ).AllowNoArguments().ParseAndHandle(args);

ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs

@@ -3,7 +3,11 @@
 public interface ISaveDataFilter
 {
     public string SearchString { get; }
-    public ISearchExpression SearchExpression { get; }
+
+    /// <summary>
+    ///     Shared between all parsing operations: the query is compiled once and its tree reused.
+    /// </summary>
+    public SearchExpressionCompilation SearchExpression { get; }
     
     public void Apply(Dictionary<string, object> data);
 }

ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs

@@ -6,7 +6,7 @@ namespace ParadoxSaveParser.WebAPI.SaveDataFilters;
 public class SaveDataFilterEU4 : ISaveDataFilter
 {
     public string SearchString { get; }
-    public ISearchExpression SearchExpression { get; }
+    public SearchExpressionCompilation SearchExpression { get; }
     
     public SaveDataFilterEU4()
     {
@@ -32,7 +32,7 @@ public class SaveDataFilterEU4 : ISaveDataFilter
             )
             """
             .Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", "");
-        SearchExpression = SearchExpressionCompiler.Compile(SearchString);
+        SearchExpression = new SearchExpressionCompilation(SearchString);
     }
 
     

ParadoxSaveParser.sln

@@ -15,27 +15,80 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParadoxSaveParser.Lib.Tests
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParadoxSaveParser.CLI", "ParadoxSaveParser.CLI\ParadoxSaveParser.CLI.csproj", "{2D4448A6-390D-47F3-9BB7-6266669719DE}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParadoxSaveParser.Benchmarks", "ParadoxSaveParser.Benchmarks\ParadoxSaveParser.Benchmarks.csproj", "{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
 		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Debug|x64.Build.0 = Debug|Any CPU
+		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Debug|x86.Build.0 = Debug|Any CPU
 		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Release|x64.ActiveCfg = Release|Any CPU
+		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Release|x64.Build.0 = Release|Any CPU
+		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Release|x86.ActiveCfg = Release|Any CPU
+		{7D377558-CE40-4D7E-B0B1-FB33C475AE6A}.Release|x86.Build.0 = Release|Any CPU
 		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Debug|x64.Build.0 = Debug|Any CPU
+		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Debug|x86.Build.0 = Debug|Any CPU
 		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Release|x64.ActiveCfg = Release|Any CPU
+		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Release|x64.Build.0 = Release|Any CPU
+		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Release|x86.ActiveCfg = Release|Any CPU
+		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Release|x86.Build.0 = Release|Any CPU
 		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Debug|x64.Build.0 = Debug|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Debug|x86.Build.0 = Debug|Any CPU
 		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Release|Any CPU.Build.0 = Release|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Release|x64.ActiveCfg = Release|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Release|x64.Build.0 = Release|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Release|x86.ActiveCfg = Release|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Release|x86.Build.0 = Release|Any CPU
 		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Debug|x64.Build.0 = Debug|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Debug|x86.Build.0 = Debug|Any CPU
 		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Release|x64.ActiveCfg = Release|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Release|x64.Build.0 = Release|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Release|x86.ActiveCfg = Release|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Release|x86.Build.0 = Release|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Debug|x64.Build.0 = Debug|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Debug|x86.Build.0 = Debug|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Release|x64.ActiveCfg = Release|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Release|x64.Build.0 = Release|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Release|x86.ActiveCfg = Release|Any CPU
+		{0E35F6F0-3D74-4C18-BFC8-A1C45A04A410}.Release|x86.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
 	EndGlobalSection
 EndGlobal

README.md

@@ -5,14 +5,16 @@ Yet another save files parser.
 - EU4
 
 ## Project structure
+- **[ParadoxSaveParser.WebAPI](./ParadoxSaveParser.WebAPI)** -
+  Backend for my save file analytics website (TODO: add repo link).
+- **[ParadoxSaveParser.CLI](./ParadoxSaveParser.CLI)** -
+  Command line tool to parse save files. Can be used in interactive mode.
 - **[ParadoxSaveParser.Lib](./ParadoxSaveParser.Lib)** - 
     Parser itself
 - **[ParadoxSaveParser.Lib.Tests](./ParadoxSaveParser.Lib.Tests)** - 
     Tests for parser
-- **[ParadoxSaveParser.CLI](./ParadoxSaveParser.CLI)** - 
-    Command line tool to parse save files. Can be used in interactive mode.
-- **[ParadoxSaveParser.WebAPI](./ParadoxSaveParser.WebAPI)** - 
-    Backend for my save file analytics website (TODO: add repo link).
+- **[ParadoxSaveParser.Benchmarks](./ParadoxSaveParser.Benchmarks)** - 
+    Performance comparison of the search expression engine against regex engines and jq
 
 ## Building
 1. Install protobuf compiler https://protobuf.dev/installation/