Parser rewrite

ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs

@@ -1,4 +1,6 @@
+using System.Collections.Generic;
 using System.IO;
+using System.Text;
 using System.Text.Encodings.Web;
 using System.Text.Json;
 using System.Text.Json.Serialization;
@@ -41,12 +43,64 @@ public class SearchExpressionTests
     [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")]
     public void TestSearchOnSmallData(string input, string expectedOutput)
     {
-        using var saveStream = new MemoryStream(_smallSaveData, false);
-        var se = SearchExpressionCompiler.Compile(input);
-        var parser = new SaveParserEU4(saveStream, se);
+        Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput));
+    }
+
+    /// <summary>
+    ///     The same queries, but with a read buffer so small that tokens, quoted strings and
+    ///     skipped blocks are split across refills.
+    /// </summary>
+    [TestCase("a", "a={ b={ c=0 d=1 e=2 } f=3 }")]
+    [TestCase("a.b", "a={ b={ c=0 d=1 e=2 } }")]
+    [TestCase("a.[1]", "a={ f=3 }")]
+    [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")]
+    public void TestSearchAcrossBufferRefills(string input, string expectedOutput)
+    {
+        foreach (int bufferSize in new[] { 16, 17, 23, 64 })
+            Assert.That(Search(_smallSaveData, input, bufferSize), Is.EqualTo(expectedOutput),
+                $"buffer size {bufferSize}");
+    }
+
+    [Test]
+    public void BracesInsideQuotedStringAreText()
+    {
+        byte[] data = "EU4txt a={ name=\"x{y}z\" b=1 }".ToBytes();
+        using var saveStream = new MemoryStream(data, false);
+        var parser = new SaveParserEU4(saveStream, SearchExpressionCompiler.Compile("a"));
+        var a = (Dictionary<string, object>)parser.Parse()["a"];
+        Assert.Multiple(() =>
+        {
+            Assert.That(a["name"], Is.EqualTo("x{y}z"));
+            Assert.That(a["b"], Is.EqualTo(1L));
+        });
+    }
+
+    [Test]
+    public void SkippedBlockIgnoresBracesInsideQuotedStrings()
+    {
+        byte[] data = "EU4txt a={ s=\"{{{\" } b=2".ToBytes();
+        using var saveStream = new MemoryStream(data, false);
+        var parser = new SaveParserEU4(saveStream, SearchExpressionCompiler.Compile("b"));
+        Assert.That(parser.Parse()["b"], Is.EqualTo(2L));
+    }
+
+    [Test]
+    public void EncodingIsConfigurable()
+    {
+        byte[] data = Encoding.UTF8.GetBytes("EU4txt a={ name=\"Ä\" }");
+        using var saveStream = new MemoryStream(data, false);
+        var parser = new SaveParserEU4(saveStream, SearchExpressionCompiler.Compile("a"), Encoding.UTF8);
+        var a = (Dictionary<string, object>)parser.Parse()["a"];
+        Assert.That(a["name"], Is.EqualTo("Ä"));
+    }
+
+    private static string Search(byte[] saveData, string query, int bufferSize = 64 * 1024)
+    {
+        using var saveStream = new MemoryStream(saveData, false);
+        var se = SearchExpressionCompiler.Compile(query);
+        var parser = new SaveParserEU4(saveStream, se, bufferSize: bufferSize);
         var rootNode = parser.Parse();
         string json = JsonSerializer.Serialize(rootNode, _smallSaveSerializerOptions);
-        string pdx = JsonToPdx(json);
-        Assert.That(pdx, Is.EqualTo(expectedOutput));
+        return JsonToPdx(json);
     }
 }