diff --git a/ParadoxSaveParser.Benchmarks/BenchData.cs b/ParadoxSaveParser.Benchmarks/BenchData.cs index 7eac775..15fa659 100644 --- a/ParadoxSaveParser.Benchmarks/BenchData.cs +++ b/ParadoxSaveParser.Benchmarks/BenchData.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using System.Text; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -35,10 +34,8 @@ public static class BenchData /// /// The save decoded as text for the regex engines. - /// Latin1 is used because EU4 saves contain non-UTF8 bytes inside dynasty names, - /// and it maps every byte to exactly one char. /// - public static string PdxText => _pdxText ??= Encoding.Latin1.GetString(PdxBytes); + public static string PdxText => _pdxText ??= SaveParserEU4.DefaultEncoding.GetString(PdxBytes); private static string FindRepoRoot() { diff --git a/ParadoxSaveParser.Benchmarks/ExtractionBenchmarks.cs b/ParadoxSaveParser.Benchmarks/ExtractionBenchmarks.cs index b175b84..705d1b1 100644 --- a/ParadoxSaveParser.Benchmarks/ExtractionBenchmarks.cs +++ b/ParadoxSaveParser.Benchmarks/ExtractionBenchmarks.cs @@ -18,7 +18,7 @@ public class ExtractionBenchmarks private byte[] _pdx = null!; private string _pdxText = null!; private byte[] _json = null!; - private ISearchExpression _query = null!; + private SearchExpressionCompilation _query = null!; private Regex _technologyBlock = null!; private Regex _pathAwareCompiled = null!; @@ -32,7 +32,7 @@ public class ExtractionBenchmarks _pdx = BenchData.PdxBytes; _pdxText = BenchData.PdxText; _json = File.ReadAllBytes(BenchData.JsonPath); - _query = SearchExpressionCompiler.Compile(BenchData.PdxQuery); + _query = new SearchExpressionCompilation(BenchData.PdxQuery); _technologyBlock = new Regex(Extractors.TechnologyBlockPattern, RegexOptions.Compiled); _pathAwareCompiled = new Regex(Extractors.PathAwarePattern, RegexOptions.Compiled); diff --git a/ParadoxSaveParser.Benchmarks/Extractors.cs b/ParadoxSaveParser.Benchmarks/Extractors.cs index 25ee59d..73b2bdb 100644 --- a/ParadoxSaveParser.Benchmarks/Extractors.cs +++ b/ParadoxSaveParser.Benchmarks/Extractors.cs @@ -22,7 +22,7 @@ public static partial class Extractors // ---------------------------------------------------------------- this project /// The parser of this repo: single streaming pass, guided by a compiled search expression. - public static long SearchExpression(byte[] pdx, ISearchExpression query) + public static long SearchExpression(byte[] pdx, SearchExpressionCompilation query) { using var stream = new MemoryStream(pdx, false); var root = new SaveParserEU4(stream, query).Parse(); @@ -30,7 +30,7 @@ public static partial class Extractors } /// Same query, but the parser reads from the given stream instead of a preloaded byte[]. - public static long SearchExpressionFromStream(Stream stream, ISearchExpression query) + public static long SearchExpressionFromStream(Stream stream, SearchExpressionCompilation query) { var root = new SaveParserEU4(stream, query).Parse(); return ChecksumOfParsedTree(root); diff --git a/ParadoxSaveParser.Benchmarks/Mispairing.cs b/ParadoxSaveParser.Benchmarks/Mispairing.cs index 475ee47..fcdf8fb 100644 --- a/ParadoxSaveParser.Benchmarks/Mispairing.cs +++ b/ParadoxSaveParser.Benchmarks/Mispairing.cs @@ -18,7 +18,7 @@ public static class Mispairing { var result = new Dictionary(); using var stream = new MemoryStream(pdx, false); - var root = new SaveParserEU4(stream, SearchExpressionCompiler.Compile(BenchData.PdxQuery)).Parse(); + var root = new SaveParserEU4(stream, new SearchExpressionCompilation(BenchData.PdxQuery)).Parse(); foreach (var (tag, value) in (Dictionary)root["countries"]) { if (value is Dictionary c diff --git a/ParadoxSaveParser.Benchmarks/ParserInputBenchmarks.cs b/ParadoxSaveParser.Benchmarks/ParserInputBenchmarks.cs index 642a164..27e254d 100644 --- a/ParadoxSaveParser.Benchmarks/ParserInputBenchmarks.cs +++ b/ParadoxSaveParser.Benchmarks/ParserInputBenchmarks.cs @@ -15,10 +15,10 @@ namespace ParadoxSaveParser.Benchmarks; [SimpleJob(RunStrategy.Monitoring, launchCount: 1, warmupCount: 1, iterationCount: 5)] public class ParserInputBenchmarks { - private ISearchExpression _query = null!; + private SearchExpressionCompilation _query = null!; [GlobalSetup] - public void Setup() => _query = SearchExpressionCompiler.Compile(BenchData.PdxQuery); + public void Setup() => _query = new SearchExpressionCompilation(BenchData.PdxQuery); [Benchmark(Description = "SearchExpression over FileStream")] public long FromFile() diff --git a/ParadoxSaveParser.Benchmarks/Program.cs b/ParadoxSaveParser.Benchmarks/Program.cs index 3f7f681..d0bb695 100644 --- a/ParadoxSaveParser.Benchmarks/Program.cs +++ b/ParadoxSaveParser.Benchmarks/Program.cs @@ -39,7 +39,7 @@ public static class Program var pdx = BenchData.PdxBytes; var text = BenchData.PdxText; var json = File.ReadAllBytes(BenchData.JsonPath); - var query = SearchExpressionCompiler.Compile(BenchData.PdxQuery); + var query = new SearchExpressionCompilation(BenchData.PdxQuery); Time("SearchExpression", () => Extractors.SearchExpression(pdx, query)); Time("FullParse", () => Extractors.FullParseThenSelect(pdx)); diff --git a/ParadoxSaveParser.CLI/Modes/Search.cs b/ParadoxSaveParser.CLI/Modes/Search.cs index b4c89cd..736b083 100644 --- a/ParadoxSaveParser.CLI/Modes/Search.cs +++ b/ParadoxSaveParser.CLI/Modes/Search.cs @@ -33,7 +33,7 @@ internal static partial class Modes ? Console.OpenStandardOutput() : File.OpenWrite(outputPath.Value); - var searchExpression = SearchExpressionCompiler.Compile(searchQuery); + var searchExpression = new SearchExpressionCompilation(searchQuery); var parser = new SaveParserEU4(inputStream, searchExpression); var parsedValue = parser.Parse(); diff --git a/ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs b/ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs index 10ce9b5..07578cd 100644 --- a/ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs +++ b/ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.IO; using System.Text; @@ -66,7 +67,7 @@ public class SearchExpressionTests { byte[] data = "EU4txt a={ name=\"x{y}z\" b=1 }".ToBytes(); using var saveStream = new MemoryStream(data, false); - var parser = new SaveParserEU4(saveStream, SearchExpressionCompiler.Compile("a")); + var parser = new SaveParserEU4(saveStream, Compile("a")); var a = (Dictionary)parser.Parse()["a"]; Assert.Multiple(() => { @@ -80,7 +81,7 @@ public class SearchExpressionTests { byte[] data = "EU4txt a={ s=\"{{{\" } b=2".ToBytes(); using var saveStream = new MemoryStream(data, false); - var parser = new SaveParserEU4(saveStream, SearchExpressionCompiler.Compile("b")); + var parser = new SaveParserEU4(saveStream, Compile("b")); Assert.That(parser.Parse()["b"], Is.EqualTo(2L)); } @@ -89,15 +90,102 @@ public class SearchExpressionTests { byte[] data = Encoding.UTF8.GetBytes("EU4txt a={ name=\"Ä\" }"); using var saveStream = new MemoryStream(data, false); - var parser = new SaveParserEU4(saveStream, SearchExpressionCompiler.Compile("a"), Encoding.UTF8); + var parser = new SaveParserEU4(saveStream, Compile("a", Encoding.UTF8), Encoding.UTF8); var a = (Dictionary)parser.Parse()["a"]; Assert.That(a["name"], Is.EqualTo("Ä")); } + /// + /// A path written after a group applies to every alternative of that group. + /// + [TestCase("a.(b|zz).c", "a={ b={ c=0 } }")] + [TestCase("(a).b.(d|e)", "a={ b={ d=1 e=2 } }")] + [TestCase("a.(b|f).c", "a={ b={ c=0 } f=3 }")] + [TestCase("(a.(b|zz)|yy).(c|d)", "a={ b={ c=0 d=1 } }")] + public void PathAfterGroupContinuesEveryAlternative(string input, string expectedOutput) + { + Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput)); + } + + /// + /// Groups with more alternatives than the other tests use: a key that matches nothing, + /// a repeated key, and a literal key standing before a "*". + /// + [TestCase("a.b.(c|d|e|zz)", "a={ b={ c=0 d=1 e=2 } }")] + [TestCase("a.b.(c|c|c|c)", "a={ b={ c=0 } }")] + [TestCase("a.(b|zz|yy|xx|*)", "a={ b={ c=0 d=1 e=2 } f=3 }")] + public void GroupWithManyAlternatives(string input, string expectedOutput) + { + Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput)); + } + + /// + /// Bytes 0x80-0x9F are punctuation in Windows-1252 and control characters in Latin1, + /// and the parser strips control characters, so the two encodings are told apart here. + /// + [Test] + public void DefaultEncodingIsWindows1252() + { + // 93 and 94 are curly quotes, 96 is an en dash + byte[] data = [.. "EU4txt a={ name=\""u8, 0x93, 0x96, 0x94, .. "\" }"u8]; + using var saveStream = new MemoryStream(data, false); + var parser = new SaveParserEU4(saveStream, Compile("a")); + var a = (Dictionary)parser.Parse()["a"]; + Assert.That(a["name"], Is.EqualTo("“–”")); + } + + /// One compilation, two saves whose keys are written in different encodings. + [Test] + public void CompilationReEncodesKeysForAnotherEncoding() + { + var compilation = new SearchExpressionCompilation("ä"); + foreach (var encoding in new[] { SaveParserEU4.DefaultEncoding, Encoding.UTF8, SaveParserEU4.DefaultEncoding }) + { + byte[] data = encoding.GetBytes("EU4txt ä={ b=1 }"); + using var saveStream = new MemoryStream(data, false); + // the parser re-encodes the shared compilation to its own encoding + var parser = new SaveParserEU4(saveStream, compilation, encoding); + var found = (Dictionary)parser.Parse()["ä"]; + Assert.That(found["b"], Is.EqualTo(1L), encoding.EncodingName); + } + } + + [TestCase("a..b")] // empty step + [TestCase(".b")] // empty first step + [TestCase("a.")] // trailing point + [TestCase("(a||b)")] // empty alternative + [TestCase("(a|b")] // unclosed group + [TestCase("(a|b))")] // extra closing bracket + [TestCase("(a|b)c")] // group not followed by '.' + [TestCase("(a|b).")] // group followed by nothing + [TestCase("a.[1")] // index step without ']' + [TestCase("a.[x]")] // index that is not a number + [TestCase("a.[-1]")] // negative index + [TestCase("a.b*c")] // wildcard inside a key + public void MalformedQueryIsReported(string query) + { + Assert.Throws(() => Compile(query)); + } + + [Test] + public void EmptyQueryIsAnArgumentError() + { + // ReSharper disable once ObjectCreationAsStatement + Assert.Throws(() => new SearchExpressionCompilation("")); + } + + /// Compiles right away, so that a malformed query throws here and not inside a parser. + private static SearchExpressionCompilation Compile(string query, Encoding? encoding = null) + { + var compilation = new SearchExpressionCompilation(query); + compilation.Compile(encoding ?? SaveParserEU4.DefaultEncoding); + return compilation; + } + private static string Search(byte[] saveData, string query, int bufferSize = 64 * 1024) { using var saveStream = new MemoryStream(saveData, false); - var se = SearchExpressionCompiler.Compile(query); + var se = Compile(query); var parser = new SaveParserEU4(saveStream, se, bufferSize: bufferSize); var rootNode = parser.Parse(); string json = JsonSerializer.Serialize(rootNode, _smallSaveSerializerOptions); diff --git a/ParadoxSaveParser.Lib/ParadoxEncodings.cs b/ParadoxSaveParser.Lib/ParadoxEncodings.cs new file mode 100644 index 0000000..1083602 --- /dev/null +++ b/ParadoxSaveParser.Lib/ParadoxEncodings.cs @@ -0,0 +1,19 @@ +namespace ParadoxSaveParser.Lib; + +/// +/// Encodings used by Paradox save files. +/// Touching this class registers the code page provider, so no consumer has to do it. +/// +public static class ParadoxEncodings +{ + /// Encoding of EU4 save files. + public static Encoding Windows1252 { get; } + + static ParadoxEncodings() + { + // .NET ships only ASCII, Latin1 and the Unicode encodings; code pages come from this provider + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + // assigned here and not in an initializer, which would run before the provider is registered + Windows1252 = Encoding.GetEncoding(1252); + } +} diff --git a/ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj.DotSettings b/ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj.DotSettings new file mode 100644 index 0000000..d022f3a --- /dev/null +++ b/ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj.DotSettings @@ -0,0 +1,2 @@ + + True \ No newline at end of file diff --git a/ParadoxSaveParser.Lib/SaveParserEU4.cs b/ParadoxSaveParser.Lib/SaveParserEU4.cs index f2dd1f0..5d431fc 100644 --- a/ParadoxSaveParser.Lib/SaveParserEU4.cs +++ b/ParadoxSaveParser.Lib/SaveParserEU4.cs @@ -14,13 +14,20 @@ public class SaveParserEU4 private const int DefaultBufferSize = 64 * 1024; private static ReadOnlySpan Header => "EU4txt"u8; + /// + /// Windows-1252 is the default encoding of the save files + /// + public static Encoding DefaultEncoding => ParadoxEncodings.Windows1252; + private readonly Tokenizer _tokens; + private readonly SearchExpressionCompilation? _query; + + // step of the query the parser is currently at, compiled by Parse() private ISearchExpression? _searchExprCurrent; /// /// Encoding of the strings inside the save. Saves of different localizations use - /// different ones, so it can be changed; maps - /// every byte to one character and never fails, which makes it a safe default. + /// different ones, so it can be changed; is what EU4 writes. /// public Encoding Encoding { get; } @@ -28,21 +35,23 @@ public class SaveParserEU4 /// Uncompressed stream of gamestate file which can be extracted from save archive /// /// - /// Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM, + /// Parsing whole save may take a few seconds on mid pc and takes 1GB of RAM, /// so you should specify what exactly you want to get from save file /// - /// Encoding of the strings inside the save. Latin1 by default. + /// Encoding of the strings inside the save. /// Size of the read buffer. Mostly useful for tests. - public SaveParserEU4(Stream savefile, ISearchExpression? query, + public SaveParserEU4(Stream savefile, SearchExpressionCompilation? query, Encoding? encoding = null, int bufferSize = DefaultBufferSize) { - Encoding = encoding ?? Encoding.Latin1; - _searchExprCurrent = query; + Encoding = encoding ?? DefaultEncoding; + _query = query; _tokens = new Tokenizer(savefile, bufferSize); } public Dictionary Parse() { + // compiled here, so the keys of the query are always encoded like the save + _searchExprCurrent = _query?.Compile(Encoding); _tokens.ReadHeader(Header, Encoding); return ParseDict(); } @@ -194,7 +203,7 @@ public class SaveParserEU4 ISearchExpression? searchExprNext = null; bool matches = _searchExprCurrent == null || _searchExprCurrent.DoesMatch( - new MatchCandidate(localIndex, _tokens.Text, Encoding), out searchExprNext); + new MatchCandidate(localIndex, _tokens.Text), out searchExprNext); string? keyStr = matches ? DecodeString(_tokens.Text) : null; // next token should be `=` or `{` diff --git a/ParadoxSaveParser.Lib/SearchExpression.cs b/ParadoxSaveParser.Lib/SearchExpression.cs deleted file mode 100644 index ef1a55b..0000000 --- a/ParadoxSaveParser.Lib/SearchExpression.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace ParadoxSaveParser.Lib; - -/// -/// The node a search expression is tested against: its key inside the parent dictionary, -/// still in the raw bytes of the save, and its position inside the parent list. -/// Keys stay undecoded so that a key rejected by the query never becomes a string. -/// -public readonly ref struct MatchCandidate -{ - public readonly ReadOnlySpan Key; - public readonly int Index; - - /// Encoding is written in. - public readonly Encoding Encoding; - - /// A list item, which has a position but no key. - public MatchCandidate(int index) - { - Key = default; - Index = index; - Encoding = Encoding.Latin1; - } - - public MatchCandidate(int index, ReadOnlySpan key, Encoding encoding) - { - Key = key; - Index = index; - Encoding = encoding; - } -} - -public interface ISearchExpression -{ - bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression); -} - -public static class SearchExpressionCompiler -{ - private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan chars, int i) - => chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\'); - - public static ISearchExpression Compile(ReadOnlySpan query) - { - if (query.IsEmpty) - throw new ArgumentNullException(nameof(query)); - - if (query[0] is '(') - { - var subExprs = new List(); - int supExprBegin = 1; - int bracketBalance = 1; - int i = supExprBegin; - for (; i < query.Length && bracketBalance != 0; i++) - { - if (CharEqualsAndNotEscaped('(', query, i)) - { - bracketBalance++; - } - else if (CharEqualsAndNotEscaped(')', query, i)) - { - bracketBalance--; - } - else if (bracketBalance == 1 && CharEqualsAndNotEscaped('|', query, i)) - { - var subPart = query.Slice(supExprBegin, i - supExprBegin); - var subExpr = Compile(subPart); - subExprs.Add(subExpr); - supExprBegin = i + 1; - } - } - - if (i != query.Length) - throw new NotImplementedException("Expressions after ')' are not supported"); - - if (bracketBalance > 0) - throw new Exception("Too many opening brackets"); - if (bracketBalance < 0) - throw new Exception("Too many closing brackets"); - - var subPartLast = query.Slice(supExprBegin, i - 1 - supExprBegin); - var subExprLast = Compile(subPartLast); - subExprs.Add(subExprLast); - return new MultipleMatchExpression(subExprs); - } - - int partBeforePointLength = 0; - while (partBeforePointLength < query.Length) - { - if (CharEqualsAndNotEscaped('.', query, partBeforePointLength)) - break; - partBeforePointLength++; - } - - var part = query.Slice(0, partBeforePointLength); - ReadOnlySpan remaining = default; - if (partBeforePointLength < query.Length) - remaining = query.Slice(partBeforePointLength + 1); - if (part is "*") - return new AnyMatchExpression(remaining.IsEmpty ? null : Compile(remaining)); - if (part is "~") - return new NoMatchExpression(); - - for (int j = 0; j < part.Length; j++) - if (CharEqualsAndNotEscaped('*', part, j)) - throw new NotImplementedException("pattern matching other than '*' is not implemented yet"); - - if (part[0] is '[') - { - part = part.Slice(1, part.Length - 2); - return new IndexMatchExpression(int.Parse(part), remaining.IsEmpty ? null : Compile(remaining)); - } - - return new ExactMatchExpression(part.ToString(), remaining.IsEmpty ? null : Compile(remaining)); - } - - - private record AnyMatchExpression(ISearchExpression? next) : ISearchExpression - { - public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) - { - nextSearchExpression = next; - return true; - } - } - - private record NoMatchExpression : ISearchExpression - { - public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) - { - nextSearchExpression = null; - return false; - } - } - - private record MultipleMatchExpression(List subExprs) : ISearchExpression - { - public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) - { - foreach (var e in subExprs) - if (e.DoesMatch(candidate, out nextSearchExpression)) - return true; - - nextSearchExpression = null; - return false; - } - } - - private record IndexMatchExpression(int index, ISearchExpression? next) : ISearchExpression - { - public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) - { - if (candidate.Index == index) - { - nextSearchExpression = next; - return true; - } - - nextSearchExpression = null; - return false; - } - } - - private record ExactMatchExpression(string key, ISearchExpression? next) : ISearchExpression - { - // the key is compared as bytes, so it is encoded once for whatever encoding the - // parser reads the save in - private byte[]? _keyBytes; - private Encoding? _keyEncoding; - - public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) - { - if (!ReferenceEquals(_keyEncoding, candidate.Encoding)) - { - _keyEncoding = candidate.Encoding; - _keyBytes = candidate.Encoding.GetBytes(key); - } - - if (candidate.Key.SequenceEqual(_keyBytes)) - { - nextSearchExpression = next; - return true; - } - - nextSearchExpression = null; - return false; - } - } -} \ No newline at end of file diff --git a/ParadoxSaveParser.Lib/SearchExpression/AnyMatchExpression.cs b/ParadoxSaveParser.Lib/SearchExpression/AnyMatchExpression.cs new file mode 100644 index 0000000..57d71ba --- /dev/null +++ b/ParadoxSaveParser.Lib/SearchExpression/AnyMatchExpression.cs @@ -0,0 +1,13 @@ +namespace ParadoxSaveParser.Lib; + +/// "*" — matches every node at this level. +internal sealed class AnyMatchExpression(ISearchExpression? next) : ISearchExpression +{ + public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) + { + nextSearchExpression = next; + return true; + } + + public void ReEncode(Encoding encoding) => next?.ReEncode(encoding); +} \ No newline at end of file diff --git a/ParadoxSaveParser.Lib/SearchExpression/ExactMatchExpression.cs b/ParadoxSaveParser.Lib/SearchExpression/ExactMatchExpression.cs new file mode 100644 index 0000000..c77b6e7 --- /dev/null +++ b/ParadoxSaveParser.Lib/SearchExpression/ExactMatchExpression.cs @@ -0,0 +1,37 @@ +namespace ParadoxSaveParser.Lib; + +/// A literal key, matched byte for byte. +internal sealed class ExactMatchExpression : ISearchExpression +{ + public string Key { get; } + + // the key as it appears in the save, so matching needs no encoding at all + public byte[] KeyBytes { get; private set; } + public ISearchExpression? Next { get; } + + public ExactMatchExpression(string key, Encoding encoding, ISearchExpression? next) + { + Key = key; + Next = next; + KeyBytes = encoding.GetBytes(key); + } + + public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) + { + // comparing bytes keeps the candidate's key from becoming a string + if (candidate.Key.SequenceEqual(KeyBytes)) + { + nextSearchExpression = Next; + return true; + } + + nextSearchExpression = null; + return false; + } + + public void ReEncode(Encoding encoding) + { + KeyBytes = encoding.GetBytes(Key); + Next?.ReEncode(encoding); + } +} \ No newline at end of file diff --git a/ParadoxSaveParser.Lib/SearchExpression/ISearchExpression.cs b/ParadoxSaveParser.Lib/SearchExpression/ISearchExpression.cs new file mode 100644 index 0000000..382583f --- /dev/null +++ b/ParadoxSaveParser.Lib/SearchExpression/ISearchExpression.cs @@ -0,0 +1,50 @@ +namespace ParadoxSaveParser.Lib; + +/// +/// One step of a compiled query. Each step points at the step its matched nodes +/// are searched with, so the query runs in a single pass down the tree. +/// +public interface ISearchExpression +{ + /// Tests one node against this step of the query. + /// The node being tested, identified by its key or its position. + /// + /// On a match, the step for the node's children, or null if the node is kept whole. + /// Undefined when the method returns false. + /// + /// true if the node is selected by this step + bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression); + + /// + /// Encodes the literal keys of this step and of every step below it. + /// Called by , never during matching. + /// + void ReEncode(Encoding encoding); +} + +/// +/// A node the query is tested against: its raw key and its position in the parent. +/// The key stays undecoded, so a rejected key never becomes a string. +/// +public readonly ref struct MatchCandidate +{ + /// Undecoded key of the node. Empty for list items. + public readonly ReadOnlySpan Key; + + /// Position of the node among its siblings, counted from 0. + public readonly int Index; + + /// A list item, which has a position but no key. + public MatchCandidate(int index) + { + Key = default; + Index = index; + } + + /// A dictionary entry, which has both a key and a position. + public MatchCandidate(int index, ReadOnlySpan key) + { + Key = key; + Index = index; + } +} diff --git a/ParadoxSaveParser.Lib/SearchExpression/IndexMatchExpression.cs b/ParadoxSaveParser.Lib/SearchExpression/IndexMatchExpression.cs new file mode 100644 index 0000000..b54730f --- /dev/null +++ b/ParadoxSaveParser.Lib/SearchExpression/IndexMatchExpression.cs @@ -0,0 +1,19 @@ +namespace ParadoxSaveParser.Lib; + +/// "[7]" — matches the node at one position. +internal sealed class IndexMatchExpression(int index, ISearchExpression? next) : ISearchExpression +{ + public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) + { + if (candidate.Index == index) + { + nextSearchExpression = next; + return true; + } + + nextSearchExpression = null; + return false; + } + + public void ReEncode(Encoding encoding) => next?.ReEncode(encoding); +} \ No newline at end of file diff --git a/ParadoxSaveParser.Lib/SearchExpression/MultipleMatchExpression.cs b/ParadoxSaveParser.Lib/SearchExpression/MultipleMatchExpression.cs new file mode 100644 index 0000000..3721251 --- /dev/null +++ b/ParadoxSaveParser.Lib/SearchExpression/MultipleMatchExpression.cs @@ -0,0 +1,22 @@ +namespace ParadoxSaveParser.Lib; + +/// "(a|b)" — matches if any alternative does. +internal sealed class MultipleMatchExpression(List subExprs) : ISearchExpression +{ + public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) + { + // the first matching alternative wins + foreach (var e in subExprs) + if (e.DoesMatch(candidate, out nextSearchExpression)) + return true; + + nextSearchExpression = null; + return false; + } + + public void ReEncode(Encoding encoding) + { + foreach (var e in subExprs) + e.ReEncode(encoding); + } +} \ No newline at end of file diff --git a/ParadoxSaveParser.Lib/SearchExpression/NoMatchExpression.cs b/ParadoxSaveParser.Lib/SearchExpression/NoMatchExpression.cs new file mode 100644 index 0000000..7de84e7 --- /dev/null +++ b/ParadoxSaveParser.Lib/SearchExpression/NoMatchExpression.cs @@ -0,0 +1,23 @@ +namespace ParadoxSaveParser.Lib; + +/// "~" — matches nothing, so the branch is skipped. +internal sealed class NoMatchExpression : ISearchExpression +{ + // stateless, so one instance serves every "~" in every query + public static readonly NoMatchExpression Instance = new(); + + private NoMatchExpression() + { + } + + public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression) + { + nextSearchExpression = null; + return false; + } + + // no keys and no children, and shared between queries, so nothing may be stored here + public void ReEncode(Encoding encoding) + { + } +} \ No newline at end of file diff --git a/ParadoxSaveParser.Lib/SearchExpression/SearchExpressionCompilation.cs b/ParadoxSaveParser.Lib/SearchExpression/SearchExpressionCompilation.cs new file mode 100644 index 0000000..6e7ac20 --- /dev/null +++ b/ParadoxSaveParser.Lib/SearchExpression/SearchExpressionCompilation.cs @@ -0,0 +1,198 @@ +using System.Threading; + +namespace ParadoxSaveParser.Lib; + +/// Thrown when a query cannot be compiled because it is malformed or uses unsupported syntax. +public class SearchExpressionException : Exception +{ + public SearchExpressionException(string message, ReadOnlySpan part) + : base($"{message}: '{part}'") + { + } +} + +/// +/// A query such as countries.*.technology, and the expression tree compiled from it. +/// The tree is built on the first and reused afterwards; only a change +/// of encoding touches it again, and then only to re-encode its literal keys. +/// +public class SearchExpressionCompilation +{ + private readonly string _query; + + // guards the tree while it is built or re-encoded, so parsers on several threads + // can share one compilation + private readonly Lock _compileLock = new(); + + private ISearchExpression? _rootNode; + private Encoding? _currentEncoding; + + /// Search query. Compiled on the first call to . + /// the query is empty + public SearchExpressionCompilation(string query) + { + ArgumentException.ThrowIfNullOrEmpty(query); + _query = query; + } + + /// + /// Returns the expression tree, with its literal keys encoded the way the save is. + /// Compiles the query on the first call, and re-encodes the tree when the encoding changes. + /// + /// Encoding of the strings inside the save being parsed. + /// the query is malformed + public ISearchExpression Compile(Encoding encoding) + { + lock (_compileLock) + { + if (_rootNode is null) + _rootNode = CompilePart(_query, encoding, null); + // keys are stored as bytes, so another encoding means encoding them again + else if (!encoding.Equals(_currentEncoding)) + _rootNode.ReEncode(encoding); + + _currentEncoding = encoding; + return _rootNode; + } + } + + /// True if holds at + /// as syntax, not as an escaped literal. + private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan chars, int i) + // escaped if a backslash stands in either of the two preceding positions + => chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\'); + + /// Compiles one path step together with everything that follows it. + /// + /// Step that continues the path after this one ends, or null if nothing follows. + /// Used for the part written after a group, as in (a|b).c, where every alternative + /// of the group continues with the same .c. + /// + private static ISearchExpression CompilePart(ReadOnlySpan query, Encoding encoding, + ISearchExpression? tail) + { + // empty alternative, as in "(a||b)" + if (query.IsEmpty) + throw new SearchExpressionException("Empty expression", query); + + // "(a|b.c)" — a group of alternatives + if (query[0] is '(') + return CompileGroup(query, encoding, tail); + + // a plain step: find the '.' that ends it + int partBeforePointLength = 0; + while (partBeforePointLength < query.Length) + { + if (CharEqualsAndNotEscaped('.', query, partBeforePointLength)) + break; + partBeforePointLength++; + } + + // this step, and the rest of the path that becomes its child expression + var part = query.Slice(0, partBeforePointLength); + // empty step, as in "a..b" or ".b" + if (part.IsEmpty) + throw new SearchExpressionException("Empty path step", query); + + ReadOnlySpan remaining = default; + if (partBeforePointLength < query.Length) + { + remaining = query.Slice(partBeforePointLength + 1); + if (remaining.IsEmpty) + throw new SearchExpressionException("Path ends with '.'", query); + } + + // when nothing follows this step, the path continues with the tail, which may be null too + var next = remaining.IsEmpty ? tail : CompilePart(remaining, encoding, tail); + + // "*" — any node at this level + if (part is "*") + return new AnyMatchExpression(next); + // "~" — nothing at this level + if (part is "~") + return NoMatchExpression.Instance; + + // a '*' anywhere else would be a wildcard inside a key + for (int j = 0; j < part.Length; j++) + if (CharEqualsAndNotEscaped('*', part, j)) + throw new SearchExpressionException("Pattern matching other than '*' is not supported", part); + + // "[7]" — match by position + if (part[0] is '[') + { + if (part[^1] is not ']') + throw new SearchExpressionException("Index step has no closing ']'", part); + // strip the brackets + var indexText = part.Slice(1, part.Length - 2); + if (!int.TryParse(indexText, out int index) || index < 0) + throw new SearchExpressionException("Index must be a non-negative number", part); + return new IndexMatchExpression(index, next); + } + + // an ordinary key + return new ExactMatchExpression(part.ToString(), encoding, next); + } + + /// Compiles "(a|b)", and the path written after it, if there is one. + private static ISearchExpression CompileGroup(ReadOnlySpan query, Encoding encoding, + ISearchExpression? tail) + { + int close = FindGroupEnd(query); + + // "(a|b).c" — what stands after the group continues every alternative of it + var afterGroup = query.Slice(close + 1); + if (!afterGroup.IsEmpty) + { + if (CharEqualsAndNotEscaped(')', afterGroup, 0)) + throw new SearchExpressionException("Too many closing brackets", query); + if (!CharEqualsAndNotEscaped('.', afterGroup, 0)) + throw new SearchExpressionException("Group must be followed by '.'", query); + + var rest = afterGroup.Slice(1); + if (rest.IsEmpty) + throw new SearchExpressionException("Path ends with '.'", query); + tail = CompilePart(rest, encoding, tail); + } + + // cut the group into alternatives and compile each of them with that same tail + var subExprs = new List(); + int begin = 1; + int depth = 0; + for (int i = 1; i < close; i++) + { + if (CharEqualsAndNotEscaped('(', query, i)) + depth++; + else if (CharEqualsAndNotEscaped(')', query, i)) + depth--; + // a deeper '|' belongs to a nested group and is compiled with it + else if (depth == 0 && CharEqualsAndNotEscaped('|', query, i)) + { + subExprs.Add(CompilePart(query.Slice(begin, i - begin), encoding, tail)); + begin = i + 1; + } + } + + // the last alternative has no '|' after it + subExprs.Add(CompilePart(query.Slice(begin, close - begin), encoding, tail)); + return new MultipleMatchExpression(subExprs); + } + + /// Finds the ')' that closes the group opened by the first character. + private static int FindGroupEnd(ReadOnlySpan query) + { + int depth = 0; + for (int i = 0; i < query.Length; i++) + { + if (CharEqualsAndNotEscaped('(', query, i)) + depth++; + else if (CharEqualsAndNotEscaped(')', query, i)) + { + depth--; + if (depth == 0) + return i; + } + } + + throw new SearchExpressionException("Too many opening brackets", query); + } +} \ No newline at end of file diff --git a/ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs b/ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs index 8aed66c..c0bb883 100644 --- a/ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs +++ b/ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs @@ -3,7 +3,11 @@ public interface ISaveDataFilter { public string SearchString { get; } - public ISearchExpression SearchExpression { get; } + + /// + /// Shared between all parsing operations: the query is compiled once and its tree reused. + /// + public SearchExpressionCompilation SearchExpression { get; } public void Apply(Dictionary data); } \ No newline at end of file diff --git a/ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs b/ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs index 1188c37..4e7d1e0 100644 --- a/ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs +++ b/ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs @@ -6,7 +6,7 @@ namespace ParadoxSaveParser.WebAPI.SaveDataFilters; public class SaveDataFilterEU4 : ISaveDataFilter { public string SearchString { get; } - public ISearchExpression SearchExpression { get; } + public SearchExpressionCompilation SearchExpression { get; } public SaveDataFilterEU4() { @@ -32,7 +32,7 @@ public class SaveDataFilterEU4 : ISaveDataFilter ) """ .Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", ""); - SearchExpression = SearchExpressionCompiler.Compile(SearchString); + SearchExpression = new SearchExpressionCompilation(SearchString); }