using System; using System.Collections.Generic; using System.IO; using System.Text; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; using DTLib.Extensions; namespace ParadoxSaveParser.Lib.Tests; [TestFixture] [TestOf(typeof(ISearchExpression))] public class SearchExpressionTests { [SetUp] public void Setup() { _smallSaveData = "EU4txt a={ b={ c=0 d=1 e=2 } f=3 }".ToBytes(); } private byte[] _smallSaveData; private static readonly JsonSerializerOptions _smallSaveSerializerOptions = new() { WriteIndented = false, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, MaxDepth = 1024, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, }; internal static string JsonToPdx(string json) => json.Substring(1, json.Length - 2) .Replace(",", " ").Replace("{", "{ ").Replace("}", " }") .Replace("\"", "").Replace("[", "").Replace("]", "").Replace(":", "="); [TestCase("a", "a={ b={ c=0 d=1 e=2 } f=3 }")] [TestCase("a.*", "a={ b={ c=0 d=1 e=2 } f=3 }")] [TestCase("a.b", "a={ b={ c=0 d=1 e=2 } }")] [TestCase("a.[0].c", "a={ b={ c=0 } }")] [TestCase("a.[1]", "a={ f=3 }")] [TestCase("a.b.(c|d)", "a={ b={ c=0 d=1 } }")] [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")] public void TestSearchOnSmallData(string input, string expectedOutput) { Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput)); } /// /// The same queries, but with a read buffer so small that tokens, quoted strings and /// skipped blocks are split across refills. /// [TestCase("a", "a={ b={ c=0 d=1 e=2 } f=3 }")] [TestCase("a.b", "a={ b={ c=0 d=1 e=2 } }")] [TestCase("a.[1]", "a={ f=3 }")] [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")] public void TestSearchAcrossBufferRefills(string input, string expectedOutput) { foreach (int bufferSize in new[] { 16, 17, 23, 64 }) Assert.That(Search(_smallSaveData, input, bufferSize), Is.EqualTo(expectedOutput), $"buffer size {bufferSize}"); } [Test] public void BracesInsideQuotedStringAreText() { byte[] data = "EU4txt a={ name=\"x{y}z\" b=1 }".ToBytes(); using var saveStream = new MemoryStream(data, false); var parser = new SaveParserEU4(saveStream, Compile("a")); var a = (Dictionary)parser.Parse()["a"]; Assert.Multiple(() => { Assert.That(a["name"], Is.EqualTo("x{y}z")); Assert.That(a["b"], Is.EqualTo(1L)); }); } [Test] public void SkippedBlockIgnoresBracesInsideQuotedStrings() { byte[] data = "EU4txt a={ s=\"{{{\" } b=2".ToBytes(); using var saveStream = new MemoryStream(data, false); var parser = new SaveParserEU4(saveStream, Compile("b")); Assert.That(parser.Parse()["b"], Is.EqualTo(2L)); } [Test] public void EncodingIsConfigurable() { byte[] data = Encoding.UTF8.GetBytes("EU4txt a={ name=\"Ä\" }"); using var saveStream = new MemoryStream(data, false); var parser = new SaveParserEU4(saveStream, Compile("a", Encoding.UTF8), Encoding.UTF8); var a = (Dictionary)parser.Parse()["a"]; Assert.That(a["name"], Is.EqualTo("Ä")); } /// /// A path written after a group applies to every alternative of that group. /// [TestCase("a.(b|zz).c", "a={ b={ c=0 } }")] [TestCase("(a).b.(d|e)", "a={ b={ d=1 e=2 } }")] [TestCase("a.(b|f).c", "a={ b={ c=0 } f=3 }")] [TestCase("(a.(b|zz)|yy).(c|d)", "a={ b={ c=0 d=1 } }")] public void PathAfterGroupContinuesEveryAlternative(string input, string expectedOutput) { Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput)); } /// /// Groups with more alternatives than the other tests use: a key that matches nothing, /// a repeated key, and a literal key standing before a "*". /// [TestCase("a.b.(c|d|e|zz)", "a={ b={ c=0 d=1 e=2 } }")] [TestCase("a.b.(c|c|c|c)", "a={ b={ c=0 } }")] [TestCase("a.(b|zz|yy|xx|*)", "a={ b={ c=0 d=1 e=2 } f=3 }")] public void GroupWithManyAlternatives(string input, string expectedOutput) { Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput)); } /// /// Bytes 0x80-0x9F are punctuation in Windows-1252 and control characters in Latin1, /// and the parser strips control characters, so the two encodings are told apart here. /// [Test] public void DefaultEncodingIsWindows1252() { // 93 and 94 are curly quotes, 96 is an en dash byte[] data = [.. "EU4txt a={ name=\""u8, 0x93, 0x96, 0x94, .. "\" }"u8]; using var saveStream = new MemoryStream(data, false); var parser = new SaveParserEU4(saveStream, Compile("a")); var a = (Dictionary)parser.Parse()["a"]; Assert.That(a["name"], Is.EqualTo("“–”")); } /// One compilation, two saves whose keys are written in different encodings. [Test] public void CompilationReEncodesKeysForAnotherEncoding() { var compilation = new SearchExpressionCompilation("ä"); foreach (var encoding in new[] { SaveParserEU4.DefaultEncoding, Encoding.UTF8, SaveParserEU4.DefaultEncoding }) { byte[] data = encoding.GetBytes("EU4txt ä={ b=1 }"); using var saveStream = new MemoryStream(data, false); // the parser re-encodes the shared compilation to its own encoding var parser = new SaveParserEU4(saveStream, compilation, encoding); var found = (Dictionary)parser.Parse()["ä"]; Assert.That(found["b"], Is.EqualTo(1L), encoding.EncodingName); } } [TestCase("a..b")] // empty step [TestCase(".b")] // empty first step [TestCase("a.")] // trailing point [TestCase("(a||b)")] // empty alternative [TestCase("(a|b")] // unclosed group [TestCase("(a|b))")] // extra closing bracket [TestCase("(a|b)c")] // group not followed by '.' [TestCase("(a|b).")] // group followed by nothing [TestCase("a.[1")] // index step without ']' [TestCase("a.[x]")] // index that is not a number [TestCase("a.[-1]")] // negative index [TestCase("a.b*c")] // wildcard inside a key public void MalformedQueryIsReported(string query) { Assert.Throws(() => Compile(query)); } [Test] public void EmptyQueryIsAnArgumentError() { // ReSharper disable once ObjectCreationAsStatement Assert.Throws(() => new SearchExpressionCompilation("")); } /// Compiles right away, so that a malformed query throws here and not inside a parser. private static SearchExpressionCompilation Compile(string query, Encoding? encoding = null) { var compilation = new SearchExpressionCompilation(query); compilation.Compile(encoding ?? SaveParserEU4.DefaultEncoding); return compilation; } private static string Search(byte[] saveData, string query, int bufferSize = 64 * 1024) { using var saveStream = new MemoryStream(saveData, false); var se = Compile(query); var parser = new SaveParserEU4(saveStream, se, bufferSize: bufferSize); var rootNode = parser.Parse(); string json = JsonSerializer.Serialize(rootNode, _smallSaveSerializerOptions); return JsonToPdx(json); } }