refactored SearchExpression

ParadoxSaveParser.Lib/ParadoxEncodings.cs

@@ -0,0 +1,19 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>
+///     Encodings used by Paradox save files.
+///     Touching this class registers the code page provider, so no consumer has to do it.
+/// </summary>
+public static class ParadoxEncodings
+{
+    /// <summary>Encoding of EU4 save files.</summary>
+    public static Encoding Windows1252 { get; }
+
+    static ParadoxEncodings()
+    {
+        // .NET ships only ASCII, Latin1 and the Unicode encodings; code pages come from this provider
+        Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
+        // assigned here and not in an initializer, which would run before the provider is registered
+        Windows1252 = Encoding.GetEncoding(1252);
+    }
+}

ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj.DotSettings

@@ -0,0 +1,2 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=searchexpression/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

ParadoxSaveParser.Lib/SaveParserEU4.cs

@@ -14,13 +14,20 @@ public class SaveParserEU4
     private const int DefaultBufferSize = 64 * 1024;
     private static ReadOnlySpan<byte> Header => "EU4txt"u8;
 
+    /// <summary>
+    ///     Windows-1252 is the default encoding of the save files
+    /// </summary>
+    public static Encoding DefaultEncoding => ParadoxEncodings.Windows1252;
+
     private readonly Tokenizer _tokens;
+    private readonly SearchExpressionCompilation? _query;
+
+    // step of the query the parser is currently at, compiled by Parse()
     private ISearchExpression? _searchExprCurrent;
 
     /// <summary>
     ///     Encoding of the strings inside the save. Saves of different localizations use
-    ///     different ones, so it can be changed; <see cref="System.Text.Encoding.Latin1" /> maps
-    ///     every byte to one character and never fails, which makes it a safe default.
+    ///     different ones, so it can be changed; <see cref="DefaultEncoding" /> is what EU4 writes.
     /// </summary>
     public Encoding Encoding { get; }
 
@@ -28,21 +35,23 @@ public class SaveParserEU4
     ///     Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive
     /// </param>
     /// <param name="query">
-    ///     Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM,
+    ///     Parsing whole save may take a few seconds on mid pc and takes 1GB of RAM,
     ///     so you should specify what exactly you want to get from save file
     /// </param>
-    /// <param name="encoding">Encoding of the strings inside the save. Latin1 by default.</param>
+    /// <param name="encoding">Encoding of the strings inside the save. <see cref="DefaultEncoding"/></param>
     /// <param name="bufferSize">Size of the read buffer. Mostly useful for tests.</param>
-    public SaveParserEU4(Stream savefile, ISearchExpression? query,
+    public SaveParserEU4(Stream savefile, SearchExpressionCompilation? query,
         Encoding? encoding = null, int bufferSize = DefaultBufferSize)
     {
-        Encoding = encoding ?? Encoding.Latin1;
-        _searchExprCurrent = query;
+        Encoding = encoding ?? DefaultEncoding;
+        _query = query;
         _tokens = new Tokenizer(savefile, bufferSize);
     }
 
     public Dictionary<string, object> Parse()
     {
+        // compiled here, so the keys of the query are always encoded like the save
+        _searchExprCurrent = _query?.Compile(Encoding);
         _tokens.ReadHeader(Header, Encoding);
         return ParseDict();
     }
@@ -194,7 +203,7 @@ public class SaveParserEU4
             ISearchExpression? searchExprNext = null;
             bool matches = _searchExprCurrent == null
                            || _searchExprCurrent.DoesMatch(
-                               new MatchCandidate(localIndex, _tokens.Text, Encoding), out searchExprNext);
+                               new MatchCandidate(localIndex, _tokens.Text), out searchExprNext);
             string? keyStr = matches ? DecodeString(_tokens.Text) : null;
 
             // next token should be `=` or `{`

ParadoxSaveParser.Lib/SearchExpression.cs

@@ -1,188 +0,0 @@
-namespace ParadoxSaveParser.Lib;
-
-/// <summary>
-///     The node a search expression is tested against: its key inside the parent dictionary,
-///     still in the raw bytes of the save, and its position inside the parent list.
-///     Keys stay undecoded so that a key rejected by the query never becomes a string.
-/// </summary>
-public readonly ref struct MatchCandidate
-{
-    public readonly ReadOnlySpan<byte> Key;
-    public readonly int Index;
-
-    /// <summary>Encoding <see cref="Key" /> is written in.</summary>
-    public readonly Encoding Encoding;
-
-    /// <summary>A list item, which has a position but no key.</summary>
-    public MatchCandidate(int index)
-    {
-        Key = default;
-        Index = index;
-        Encoding = Encoding.Latin1;
-    }
-
-    public MatchCandidate(int index, ReadOnlySpan<byte> key, Encoding encoding)
-    {
-        Key = key;
-        Index = index;
-        Encoding = encoding;
-    }
-}
-
-public interface ISearchExpression
-{
-    bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression);
-}
-
-public static class SearchExpressionCompiler
-{
-    private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan<char> chars, int i)
-        => chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\');
-
-    public static ISearchExpression Compile(ReadOnlySpan<char> query)
-    {
-        if (query.IsEmpty)
-            throw new ArgumentNullException(nameof(query));
-
-        if (query[0] is '(')
-        {
-            var subExprs = new List<ISearchExpression>();
-            int supExprBegin = 1;
-            int bracketBalance = 1;
-            int i = supExprBegin;
-            for (; i < query.Length && bracketBalance != 0; i++)
-            {
-                if (CharEqualsAndNotEscaped('(', query, i))
-                {
-                    bracketBalance++;
-                }
-                else if (CharEqualsAndNotEscaped(')', query, i))
-                {
-                    bracketBalance--;
-                }
-                else if (bracketBalance == 1 && CharEqualsAndNotEscaped('|', query, i))
-                {
-                    var subPart = query.Slice(supExprBegin, i - supExprBegin);
-                    var subExpr = Compile(subPart);
-                    subExprs.Add(subExpr);
-                    supExprBegin = i + 1;
-                }
-            }
-
-            if (i != query.Length)
-                throw new NotImplementedException("Expressions after ')' are not supported");
-
-            if (bracketBalance > 0)
-                throw new Exception("Too many opening brackets");
-            if (bracketBalance < 0)
-                throw new Exception("Too many closing brackets");
-
-            var subPartLast = query.Slice(supExprBegin, i - 1 - supExprBegin);
-            var subExprLast = Compile(subPartLast);
-            subExprs.Add(subExprLast);
-            return new MultipleMatchExpression(subExprs);
-        }
-
-        int partBeforePointLength = 0;
-        while (partBeforePointLength < query.Length)
-        {
-            if (CharEqualsAndNotEscaped('.', query, partBeforePointLength))
-                break;
-            partBeforePointLength++;
-        }
-
-        var part = query.Slice(0, partBeforePointLength);
-        ReadOnlySpan<char> remaining = default;
-        if (partBeforePointLength < query.Length)
-            remaining = query.Slice(partBeforePointLength + 1);
-        if (part is "*") 
-            return new AnyMatchExpression(remaining.IsEmpty ? null : Compile(remaining));
-        if (part is "~")
-            return new NoMatchExpression();
-        
-        for (int j = 0; j < part.Length; j++)
-            if (CharEqualsAndNotEscaped('*', part, j))
-                throw new NotImplementedException("pattern matching other than '*' is not implemented yet");
-
-        if (part[0] is '[')
-        {
-            part = part.Slice(1, part.Length - 2);
-            return new IndexMatchExpression(int.Parse(part), remaining.IsEmpty ? null : Compile(remaining));
-        }
-
-        return new ExactMatchExpression(part.ToString(), remaining.IsEmpty ? null : Compile(remaining));
-    }
-
-
-    private record AnyMatchExpression(ISearchExpression? next) : ISearchExpression
-    {
-        public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
-        {
-            nextSearchExpression = next;
-            return true;
-        }
-    }
-
-    private record NoMatchExpression : ISearchExpression
-    {
-        public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
-        {
-            nextSearchExpression = null;
-            return false;
-        }
-    }
-
-    private record MultipleMatchExpression(List<ISearchExpression> subExprs) : ISearchExpression
-    {
-        public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
-        {
-            foreach (var e in subExprs)
-                if (e.DoesMatch(candidate, out nextSearchExpression))
-                    return true;
-
-            nextSearchExpression = null;
-            return false;
-        }
-    }
-
-    private record IndexMatchExpression(int index, ISearchExpression? next) : ISearchExpression
-    {
-        public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
-        {
-            if (candidate.Index == index)
-            {
-                nextSearchExpression = next;
-                return true;
-            }
-
-            nextSearchExpression = null;
-            return false;
-        }
-    }
-
-    private record ExactMatchExpression(string key, ISearchExpression? next) : ISearchExpression
-    {
-        // the key is compared as bytes, so it is encoded once for whatever encoding the
-        // parser reads the save in
-        private byte[]? _keyBytes;
-        private Encoding? _keyEncoding;
-
-        public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
-        {
-            if (!ReferenceEquals(_keyEncoding, candidate.Encoding))
-            {
-                _keyEncoding = candidate.Encoding;
-                _keyBytes = candidate.Encoding.GetBytes(key);
-            }
-
-            if (candidate.Key.SequenceEqual(_keyBytes))
-            {
-                nextSearchExpression = next;
-                return true;
-            }
-
-            nextSearchExpression = null;
-            return false;
-        }
-    }
-}

ParadoxSaveParser.Lib/SearchExpression/AnyMatchExpression.cs

@@ -0,0 +1,13 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>"*" — matches every node at this level.</summary>
+internal sealed class AnyMatchExpression(ISearchExpression? next) : ISearchExpression
+{
+    public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
+    {
+        nextSearchExpression = next;
+        return true;
+    }
+
+    public void ReEncode(Encoding encoding) => next?.ReEncode(encoding);
+}

ParadoxSaveParser.Lib/SearchExpression/ExactMatchExpression.cs

@@ -0,0 +1,37 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>A literal key, matched byte for byte.</summary>
+internal sealed class ExactMatchExpression : ISearchExpression
+{
+    public string Key { get; }
+
+    // the key as it appears in the save, so matching needs no encoding at all
+    public byte[] KeyBytes { get; private set; }
+    public ISearchExpression? Next { get; }
+
+    public ExactMatchExpression(string key, Encoding encoding, ISearchExpression? next)
+    {
+        Key = key;
+        Next = next;
+        KeyBytes = encoding.GetBytes(key);
+    }
+
+    public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
+    {
+        // comparing bytes keeps the candidate's key from becoming a string
+        if (candidate.Key.SequenceEqual(KeyBytes))
+        {
+            nextSearchExpression = Next;
+            return true;
+        }
+
+        nextSearchExpression = null;
+        return false;
+    }
+
+    public void ReEncode(Encoding encoding)
+    {
+        KeyBytes = encoding.GetBytes(Key);
+        Next?.ReEncode(encoding);
+    }
+}

ParadoxSaveParser.Lib/SearchExpression/ISearchExpression.cs

@@ -0,0 +1,50 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>
+///     One step of a compiled query. Each step points at the step its matched nodes
+///     are searched with, so the query runs in a single pass down the tree.
+/// </summary>
+public interface ISearchExpression
+{
+    /// <summary>Tests one node against this step of the query.</summary>
+    /// <param name="candidate">The node being tested, identified by its key or its position.</param>
+    /// <param name="nextSearchExpression">
+    ///     On a match, the step for the node's children, or <c>null</c> if the node is kept whole.
+    ///     Undefined when the method returns <c>false</c>.
+    /// </param>
+    /// <returns>true if the node is selected by this step</returns>
+    bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression);
+
+    /// <summary>
+    ///     Encodes the literal keys of this step and of every step below it.
+    ///     Called by <see cref="SearchExpressionCompilation" />, never during matching.
+    /// </summary>
+    void ReEncode(Encoding encoding);
+}
+
+/// <summary>
+///     A node the query is tested against: its raw key and its position in the parent.
+///     The key stays undecoded, so a rejected key never becomes a string.
+/// </summary>
+public readonly ref struct MatchCandidate
+{
+    /// <summary>Undecoded key of the node. Empty for list items.</summary>
+    public readonly ReadOnlySpan<byte> Key;
+
+    /// <summary>Position of the node among its siblings, counted from 0.</summary>
+    public readonly int Index;
+
+    /// <summary>A list item, which has a position but no key.</summary>
+    public MatchCandidate(int index)
+    {
+        Key = default;
+        Index = index;
+    }
+
+    /// <summary>A dictionary entry, which has both a key and a position.</summary>
+    public MatchCandidate(int index, ReadOnlySpan<byte> key)
+    {
+        Key = key;
+        Index = index;
+    }
+}

ParadoxSaveParser.Lib/SearchExpression/IndexMatchExpression.cs

@@ -0,0 +1,19 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>"[7]" — matches the node at one position.</summary>
+internal sealed class IndexMatchExpression(int index, ISearchExpression? next) : ISearchExpression
+{
+    public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
+    {
+        if (candidate.Index == index)
+        {
+            nextSearchExpression = next;
+            return true;
+        }
+
+        nextSearchExpression = null;
+        return false;
+    }
+
+    public void ReEncode(Encoding encoding) => next?.ReEncode(encoding);
+}

ParadoxSaveParser.Lib/SearchExpression/MultipleMatchExpression.cs

@@ -0,0 +1,22 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>"(a|b)" — matches if any alternative does.</summary>
+internal sealed class MultipleMatchExpression(List<ISearchExpression> subExprs) : ISearchExpression
+{
+    public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
+    {
+        // the first matching alternative wins
+        foreach (var e in subExprs)
+            if (e.DoesMatch(candidate, out nextSearchExpression))
+                return true;
+
+        nextSearchExpression = null;
+        return false;
+    }
+
+    public void ReEncode(Encoding encoding)
+    {
+        foreach (var e in subExprs)
+            e.ReEncode(encoding);
+    }
+}

ParadoxSaveParser.Lib/SearchExpression/NoMatchExpression.cs

@@ -0,0 +1,23 @@
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>"~" — matches nothing, so the branch is skipped.</summary>
+internal sealed class NoMatchExpression : ISearchExpression
+{
+    // stateless, so one instance serves every "~" in every query
+    public static readonly NoMatchExpression Instance = new();
+
+    private NoMatchExpression()
+    {
+    }
+
+    public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
+    {
+        nextSearchExpression = null;
+        return false;
+    }
+
+    // no keys and no children, and shared between queries, so nothing may be stored here
+    public void ReEncode(Encoding encoding)
+    {
+    }
+}

ParadoxSaveParser.Lib/SearchExpression/SearchExpressionCompilation.cs

@@ -0,0 +1,198 @@
+using System.Threading;
+
+namespace ParadoxSaveParser.Lib;
+
+/// <summary>Thrown when a query cannot be compiled because it is malformed or uses unsupported syntax.</summary>
+public class SearchExpressionException : Exception
+{
+    public SearchExpressionException(string message, ReadOnlySpan<char> part)
+        : base($"{message}: '{part}'")
+    {
+    }
+}
+
+/// <summary>
+///     A query such as <c>countries.*.technology</c>, and the expression tree compiled from it.
+///     The tree is built on the first <see cref="Compile" /> and reused afterwards; only a change
+///     of encoding touches it again, and then only to re-encode its literal keys.
+/// </summary>
+public class SearchExpressionCompilation
+{
+    private readonly string _query;
+
+    // guards the tree while it is built or re-encoded, so parsers on several threads
+    // can share one compilation
+    private readonly Lock _compileLock = new();
+
+    private ISearchExpression? _rootNode;
+    private Encoding? _currentEncoding;
+
+    /// <param name="query">Search query. Compiled on the first call to <see cref="Compile" />.</param>
+    /// <exception cref="ArgumentException">the query is empty</exception>
+    public SearchExpressionCompilation(string query)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(query);
+        _query = query;
+    }
+
+    /// <summary>
+    ///     Returns the expression tree, with its literal keys encoded the way the save is.
+    ///     Compiles the query on the first call, and re-encodes the tree when the encoding changes.
+    /// </summary>
+    /// <param name="encoding">Encoding of the strings inside the save being parsed.</param>
+    /// <exception cref="SearchExpressionException">the query is malformed</exception>
+    public ISearchExpression Compile(Encoding encoding)
+    {
+        lock (_compileLock)
+        {
+            if (_rootNode is null)
+                _rootNode = CompilePart(_query, encoding, null);
+            // keys are stored as bytes, so another encoding means encoding them again
+            else if (!encoding.Equals(_currentEncoding)) 
+                _rootNode.ReEncode(encoding);
+
+            _currentEncoding = encoding;
+            return _rootNode;
+        }
+    }
+
+    /// <summary>True if <paramref name="chars" /> holds <paramref name="c" /> at
+    /// <paramref name="i" /> as syntax, not as an escaped literal.</summary>
+    private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan<char> chars, int i)
+        // escaped if a backslash stands in either of the two preceding positions
+        => chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\');
+
+    /// <summary>Compiles one path step together with everything that follows it.</summary>
+    /// <param name="tail">
+    ///     Step that continues the path after this one ends, or <c>null</c> if nothing follows.
+    ///     Used for the part written after a group, as in <c>(a|b).c</c>, where every alternative
+    ///     of the group continues with the same <c>.c</c>.
+    /// </param>
+    private static ISearchExpression CompilePart(ReadOnlySpan<char> query, Encoding encoding,
+        ISearchExpression? tail)
+    {
+        // empty alternative, as in "(a||b)"
+        if (query.IsEmpty)
+            throw new SearchExpressionException("Empty expression", query);
+
+        // "(a|b.c)" — a group of alternatives
+        if (query[0] is '(')
+            return CompileGroup(query, encoding, tail);
+
+        // a plain step: find the '.' that ends it
+        int partBeforePointLength = 0;
+        while (partBeforePointLength < query.Length)
+        {
+            if (CharEqualsAndNotEscaped('.', query, partBeforePointLength))
+                break;
+            partBeforePointLength++;
+        }
+
+        // this step, and the rest of the path that becomes its child expression
+        var part = query.Slice(0, partBeforePointLength);
+        // empty step, as in "a..b" or ".b"
+        if (part.IsEmpty)
+            throw new SearchExpressionException("Empty path step", query);
+
+        ReadOnlySpan<char> remaining = default;
+        if (partBeforePointLength < query.Length)
+        {
+            remaining = query.Slice(partBeforePointLength + 1);
+            if (remaining.IsEmpty)
+                throw new SearchExpressionException("Path ends with '.'", query);
+        }
+
+        // when nothing follows this step, the path continues with the tail, which may be null too
+        var next = remaining.IsEmpty ? tail : CompilePart(remaining, encoding, tail);
+
+        // "*" — any node at this level
+        if (part is "*")
+            return new AnyMatchExpression(next);
+        // "~" — nothing at this level
+        if (part is "~")
+            return NoMatchExpression.Instance;
+
+        // a '*' anywhere else would be a wildcard inside a key
+        for (int j = 0; j < part.Length; j++)
+            if (CharEqualsAndNotEscaped('*', part, j))
+                throw new SearchExpressionException("Pattern matching other than '*' is not supported", part);
+
+        // "[7]" — match by position
+        if (part[0] is '[')
+        {
+            if (part[^1] is not ']')
+                throw new SearchExpressionException("Index step has no closing ']'", part);
+            // strip the brackets
+            var indexText = part.Slice(1, part.Length - 2);
+            if (!int.TryParse(indexText, out int index) || index < 0)
+                throw new SearchExpressionException("Index must be a non-negative number", part);
+            return new IndexMatchExpression(index, next);
+        }
+
+        // an ordinary key
+        return new ExactMatchExpression(part.ToString(), encoding, next);
+    }
+
+    /// <summary>Compiles "(a|b)", and the path written after it, if there is one.</summary>
+    private static ISearchExpression CompileGroup(ReadOnlySpan<char> query, Encoding encoding,
+        ISearchExpression? tail)
+    {
+        int close = FindGroupEnd(query);
+
+        // "(a|b).c" — what stands after the group continues every alternative of it
+        var afterGroup = query.Slice(close + 1);
+        if (!afterGroup.IsEmpty)
+        {
+            if (CharEqualsAndNotEscaped(')', afterGroup, 0))
+                throw new SearchExpressionException("Too many closing brackets", query);
+            if (!CharEqualsAndNotEscaped('.', afterGroup, 0))
+                throw new SearchExpressionException("Group must be followed by '.'", query);
+
+            var rest = afterGroup.Slice(1);
+            if (rest.IsEmpty)
+                throw new SearchExpressionException("Path ends with '.'", query);
+            tail = CompilePart(rest, encoding, tail);
+        }
+
+        // cut the group into alternatives and compile each of them with that same tail
+        var subExprs = new List<ISearchExpression>();
+        int begin = 1;
+        int depth = 0;
+        for (int i = 1; i < close; i++)
+        {
+            if (CharEqualsAndNotEscaped('(', query, i))
+                depth++;
+            else if (CharEqualsAndNotEscaped(')', query, i))
+                depth--;
+            // a deeper '|' belongs to a nested group and is compiled with it
+            else if (depth == 0 && CharEqualsAndNotEscaped('|', query, i))
+            {
+                subExprs.Add(CompilePart(query.Slice(begin, i - begin), encoding, tail));
+                begin = i + 1;
+            }
+        }
+
+        // the last alternative has no '|' after it
+        subExprs.Add(CompilePart(query.Slice(begin, close - begin), encoding, tail));
+        return new MultipleMatchExpression(subExprs);
+    }
+
+    /// <summary>Finds the ')' that closes the group opened by the first character.</summary>
+    private static int FindGroupEnd(ReadOnlySpan<char> query)
+    {
+        int depth = 0;
+        for (int i = 0; i < query.Length; i++)
+        {
+            if (CharEqualsAndNotEscaped('(', query, i))
+                depth++;
+            else if (CharEqualsAndNotEscaped(')', query, i))
+            {
+                depth--;
+                if (depth == 0)
+                    return i;
+            }
+        }
+
+        throw new SearchExpressionException("Too many opening brackets", query);
+    }
+}