ParadoxSaveParser/ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Serialization;
using DTLib.Extensions;

namespace ParadoxSaveParser.Lib.Tests;

[TestFixture]
[TestOf(typeof(ISearchExpression))]
public class SearchExpressionTests
{
    [SetUp]
    public void Setup()
    {
        _smallSaveData = "EU4txt a={ b={ c=0 d=1 e=2 } f=3 }".ToBytes();
    }

    private byte[] _smallSaveData;


    private static readonly JsonSerializerOptions _smallSaveSerializerOptions = new()
    {
        WriteIndented = false,
        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
        MaxDepth = 1024,
        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
    };

    internal static string JsonToPdx(string json)
        => json.Substring(1, json.Length - 2)
            .Replace(",", " ").Replace("{", "{ ").Replace("}", " }")
            .Replace("\"", "").Replace("[", "").Replace("]", "").Replace(":", "=");

    [TestCase("a", "a={ b={ c=0 d=1 e=2 } f=3 }")]
    [TestCase("a.*", "a={ b={ c=0 d=1 e=2 } f=3 }")]
    [TestCase("a.b", "a={ b={ c=0 d=1 e=2 } }")]
    [TestCase("a.[0].c", "a={ b={ c=0 } }")]
    [TestCase("a.[1]", "a={ f=3 }")]
    [TestCase("a.b.(c|d)", "a={ b={ c=0 d=1 } }")]
    [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")]
    public void TestSearchOnSmallData(string input, string expectedOutput)
    {
        Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput));
    }

    /// <summary>
    ///     The same queries, but with a read buffer so small that tokens, quoted strings and
    ///     skipped blocks are split across refills.
    /// </summary>
    [TestCase("a", "a={ b={ c=0 d=1 e=2 } f=3 }")]
    [TestCase("a.b", "a={ b={ c=0 d=1 e=2 } }")]
    [TestCase("a.[1]", "a={ f=3 }")]
    [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")]
    public void TestSearchAcrossBufferRefills(string input, string expectedOutput)
    {
        foreach (int bufferSize in new[] { 16, 17, 23, 64 })
            Assert.That(Search(_smallSaveData, input, bufferSize), Is.EqualTo(expectedOutput),
                $"buffer size {bufferSize}");
    }

    [Test]
    public void BracesInsideQuotedStringAreText()
    {
        byte[] data = "EU4txt a={ name=\"x{y}z\" b=1 }".ToBytes();
        using var saveStream = new MemoryStream(data, false);
        var parser = new SaveParserEU4(saveStream, Compile("a"));
        var a = (Dictionary<string, object>)parser.Parse()["a"];
        Assert.Multiple(() =>
        {
            Assert.That(a["name"], Is.EqualTo("x{y}z"));
            Assert.That(a["b"], Is.EqualTo(1L));
        });
    }

    [Test]
    public void SkippedBlockIgnoresBracesInsideQuotedStrings()
    {
        byte[] data = "EU4txt a={ s=\"{{{\" } b=2".ToBytes();
        using var saveStream = new MemoryStream(data, false);
        var parser = new SaveParserEU4(saveStream, Compile("b"));
        Assert.That(parser.Parse()["b"], Is.EqualTo(2L));
    }

    [Test]
    public void EncodingIsConfigurable()
    {
        byte[] data = Encoding.UTF8.GetBytes("EU4txt a={ name=\"Ä\" }");
        using var saveStream = new MemoryStream(data, false);
        var parser = new SaveParserEU4(saveStream, Compile("a", Encoding.UTF8), Encoding.UTF8);
        var a = (Dictionary<string, object>)parser.Parse()["a"];
        Assert.That(a["name"], Is.EqualTo("Ä"));
    }

    /// <summary>
    ///     A path written after a group applies to every alternative of that group.
    /// </summary>
    [TestCase("a.(b|zz).c", "a={ b={ c=0 } }")]
    [TestCase("(a).b.(d|e)", "a={ b={ d=1 e=2 } }")]
    [TestCase("a.(b|f).c", "a={ b={ c=0 } f=3 }")]
    [TestCase("(a.(b|zz)|yy).(c|d)", "a={ b={ c=0 d=1 } }")]
    public void PathAfterGroupContinuesEveryAlternative(string input, string expectedOutput)
    {
        Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput));
    }

    /// <summary>
    ///     Groups with more alternatives than the other tests use: a key that matches nothing,
    ///     a repeated key, and a literal key standing before a "*".
    /// </summary>
    [TestCase("a.b.(c|d|e|zz)", "a={ b={ c=0 d=1 e=2 } }")]
    [TestCase("a.b.(c|c|c|c)", "a={ b={ c=0 } }")]
    [TestCase("a.(b|zz|yy|xx|*)", "a={ b={ c=0 d=1 e=2 } f=3 }")]
    public void GroupWithManyAlternatives(string input, string expectedOutput)
    {
        Assert.That(Search(_smallSaveData, input), Is.EqualTo(expectedOutput));
    }

    /// <summary>
    ///     Bytes 0x80-0x9F are punctuation in Windows-1252 and control characters in Latin1,
    ///     and the parser strips control characters, so the two encodings are told apart here.
    /// </summary>
    [Test]
    public void DefaultEncodingIsWindows1252()
    {
        // 93 and 94 are curly quotes, 96 is an en dash
        byte[] data = [.. "EU4txt a={ name=\""u8, 0x93, 0x96, 0x94, .. "\" }"u8];
        using var saveStream = new MemoryStream(data, false);
        var parser = new SaveParserEU4(saveStream, Compile("a"));
        var a = (Dictionary<string, object>)parser.Parse()["a"];
        Assert.That(a["name"], Is.EqualTo("“–”"));
    }

    /// <summary>One compilation, two saves whose keys are written in different encodings.</summary>
    [Test]
    public void CompilationReEncodesKeysForAnotherEncoding()
    {
        var compilation = new SearchExpressionCompilation("ä");
        foreach (var encoding in new[] { SaveParserEU4.DefaultEncoding, Encoding.UTF8, SaveParserEU4.DefaultEncoding })
        {
            byte[] data = encoding.GetBytes("EU4txt ä={ b=1 }");
            using var saveStream = new MemoryStream(data, false);
            // the parser re-encodes the shared compilation to its own encoding
            var parser = new SaveParserEU4(saveStream, compilation, encoding);
            var found = (Dictionary<string, object>)parser.Parse()["ä"];
            Assert.That(found["b"], Is.EqualTo(1L), encoding.EncodingName);
        }
    }

    [TestCase("a..b")]       // empty step
    [TestCase(".b")]         // empty first step
    [TestCase("a.")]         // trailing point
    [TestCase("(a||b)")]     // empty alternative
    [TestCase("(a|b")]       // unclosed group
    [TestCase("(a|b))")]     // extra closing bracket
    [TestCase("(a|b)c")]     // group not followed by '.'
    [TestCase("(a|b).")]     // group followed by nothing
    [TestCase("a.[1")]       // index step without ']'
    [TestCase("a.[x]")]      // index that is not a number
    [TestCase("a.[-1]")]     // negative index
    [TestCase("a.b*c")]      // wildcard inside a key
    public void MalformedQueryIsReported(string query)
    {
        Assert.Throws<SearchExpressionException>(() => Compile(query));
    }

    [Test]
    public void EmptyQueryIsAnArgumentError()
    {
        // ReSharper disable once ObjectCreationAsStatement
        Assert.Throws<ArgumentException>(() => new SearchExpressionCompilation(""));
    }

    /// <summary>Compiles right away, so that a malformed query throws here and not inside a parser.</summary>
    private static SearchExpressionCompilation Compile(string query, Encoding? encoding = null)
    {
        var compilation = new SearchExpressionCompilation(query);
        compilation.Compile(encoding ?? SaveParserEU4.DefaultEncoding);
        return compilation;
    }

    private static string Search(byte[] saveData, string query, int bufferSize = 64 * 1024)
    {
        using var saveStream = new MemoryStream(saveData, false);
        var se = Compile(query);
        var parser = new SaveParserEU4(saveStream, se, bufferSize: bufferSize);
        var rootNode = parser.Parse();
        string json = JsonSerializer.Serialize(rootNode, _smallSaveSerializerOptions);
        return JsonToPdx(json);
    }
}