code reformat and cleanup

ParadoxSaveParser.Lib.Tests/ParadoxSaveParser.Lib.Tests.csproj

@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+
+        <IsPackable>false</IsPackable>
+        <IsTestProject>true</IsTestProject>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="coverlet.collector" Version="6.0.0"/>
+        <PackageReference Include="DTLib" Version="1.6.5"/>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0"/>
+        <PackageReference Include="NUnit" Version="3.14.0"/>
+        <PackageReference Include="NUnit.Analyzers" Version="3.9.0"/>
+        <PackageReference Include="NUnit3TestAdapter" Version="4.5.0"/>
+    </ItemGroup>
+
+    <ItemGroup>
+        <Using Include="NUnit.Framework"/>
+    </ItemGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj"/>
+    </ItemGroup>
+
+</Project>

ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs

@@ -0,0 +1,49 @@
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using DTLib.Extensions;
+
+namespace ParadoxSaveParser.Lib.Tests;
+
+[TestFixture]
+[TestOf(typeof(ISearchExpression))]
+public class SearchExpressionTests
+{
+    [SetUp]
+    public void Setup()
+    {
+        _smallSaveData = "EU4txt a={ b={ c=0 d=1 e=2 } f=3 }".ToBytes();
+    }
+
+    private byte[] _smallSaveData;
+
+
+    private static readonly JsonSerializerOptions _smallSaveSerializerOptions = new()
+    {
+        WriteIndented = false,
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        MaxDepth = 1024
+    };
+
+    internal static string JsonToPdx(string json)
+        => json.Substring(1, json.Length - 2)
+            .Replace(",", " ").Replace("{", "{ ").Replace("}", " }")
+            .Replace("\"", "").Replace("[", "").Replace("]", "").Replace(":", "=");
+
+    [TestCase("a", "a={ b={ c=0 d=1 e=2 } f=3 }")]
+    [TestCase("a.*", "a={ b={ c=0 d=1 e=2 } f=3 }")]
+    [TestCase("a.b", "a={ b={ c=0 d=1 e=2 } }")]
+    [TestCase("a.[0].c", "a={ b={ c=0 } }")]
+    [TestCase("a.[1]", "a={ f=3 }")]
+    [TestCase("a.b.(c|d)", "a={ b={ c=0 d=1 } }")]
+    [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")]
+    public void TestSearchOnSmallData(string input, string expectedOutput)
+    {
+        using var saveStream = new MemoryStream(_smallSaveData, false);
+        var se = SearchExpressionCompiler.Compile(input);
+        var parser = new SaveParserEU4(saveStream, se);
+        var rootNode = parser.Parse();
+        string json = JsonSerializer.Serialize(rootNode, _smallSaveSerializerOptions);
+        string pdx = JsonToPdx(json);
+        Assert.That(pdx, Is.EqualTo(expectedOutput));
+    }
+}

ParadoxSaveParser.Lib/BufferedEnumerator.cs

@@ -39,11 +39,11 @@ namespace ParadoxSaveParser.Lib;
 /// </code>
 public class BufferedEnumerator<T> : IEnumerator<LinkedListNode<T>>
 {
-    private IEnumerator<T> _enumerator;
-    private int _bufferSize;
-    LinkedList<T> _llist = new();
+    private readonly int _bufferSize;
     private LinkedListNode<T>? _currentNode;
     private int _currentNodeIndex = -1;
+    private readonly IEnumerator<T> _enumerator;
+    private readonly LinkedList<T> _llist = new();
 
     public BufferedEnumerator(IEnumerator<T> enumerator, int bufferSize)
     {
@@ -53,7 +53,7 @@ public class BufferedEnumerator<T> : IEnumerator<LinkedListNode<T>>
 
     public bool MoveNext()
     {
-        if(_currentNodeIndex >= _bufferSize / 2)
+        if (_currentNodeIndex >= _bufferSize / 2)
             _llist.RemoveFirst();
         
         while (_llist.Count < _bufferSize && _enumerator.MoveNext())

ParadoxSaveParser.Lib/SaveParserEU4.cs

@@ -6,74 +6,26 @@ global using System.Text;
 namespace ParadoxSaveParser.Lib;
 
 /// <summary>
-/// Sequential parser that doesn't cache anything.
+///     Sequential parser that doesn't cache anything.
 /// </summary>
 public class SaveParserEU4
 {
     protected Stream _saveFile;
-    private BufferedEnumerator<Token> _tokens;
-    private SearchExpression _query;
-    private int _currentDepth;
+    private ISearchExpression? _searchExprCurrent;
+    private readonly BufferedEnumerator<Token> _tokens;
 
-    /// <param name="savefile">Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive</param>
-    /// <param name="query">Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM,
-    /// so you should specify what exactly you want to get from save file</param>
-    public SaveParserEU4(Stream savefile, SearchExpression query)
+    /// <param name="savefile">
+    ///     Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive
+    /// </param>
+    /// <param name="query">
+    ///     Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM,
+    ///     so you should specify what exactly you want to get from save file
+    /// </param>
+    public SaveParserEU4(Stream savefile, ISearchExpression? query)
     {
         _tokens = new BufferedEnumerator<Token>(LexTextSave(), 5);
         _saveFile = savefile;
-        _query = query;
-    }
-
-    protected enum TokenType : byte
-    {
-        Invalid,
-        StringOrNumber,
-        Equals,
-        BracketOpen,
-        BracketClose,
-    }
-
-    protected struct Token
-    {
-        public required TokenType type;
-        public required short column;
-        public required int line;
-        public string? value;
-
-        public override string ToString()
-        {
-            string s;
-            switch (type)
-            {
-                case TokenType.Invalid:
-                    s = "INVALID_TOKEN";
-                    break;
-                case TokenType.StringOrNumber:
-                    s = value ?? "NULL";
-                    break;
-                case TokenType.Equals:
-                    s = "=";
-                    break;
-                case TokenType.BracketOpen:
-                    s = "{";
-                    break;
-                case TokenType.BracketClose:
-                    s = "}";
-                    break;
-                default:
-                    throw new ArgumentOutOfRangeException(type.ToString());
-            }
-            
-            return $"{line}:{column} '{s}'";
-        }
-    }
-
-    protected class UnexpectedTokenException : Exception
-    {
-        public UnexpectedTokenException(Token token) :
-            base($"Unexpected token: {token}")
-        {}
+        _searchExprCurrent = query;
     }
 
     protected IEnumerator<Token> LexTextSave()
@@ -125,7 +77,7 @@ public class SaveParserEU4
             switch (c)
             {
                 case -1:
-                    if(TryCompleteStringToken())
+                    if (TryCompleteStringToken())
                         yield return strToken;
                     yield break;
                 case '\"':
@@ -135,17 +87,17 @@ public class SaveParserEU4
                 case ' ':
                 case '\t':
                 case '\r':
-                    if(TryCompleteStringToken())
+                    if (TryCompleteStringToken())
                         yield return strToken;
                     break;
                 case '\n':
-                    if(TryCompleteStringToken())
+                    if (TryCompleteStringToken())
                         yield return strToken;
                     line++;
                     column = 0;
                     break;
                 case '=':
-                    if(TryCompleteStringToken())
+                    if (TryCompleteStringToken())
                         yield return strToken;
                     yield return new Token
                     {
@@ -154,7 +106,7 @@ public class SaveParserEU4
                     };
                     break;
                 case '{':
-                    if(TryCompleteStringToken())
+                    if (TryCompleteStringToken())
                         yield return strToken;
                     yield return new Token
                     {
@@ -163,7 +115,7 @@ public class SaveParserEU4
                     };
                     break;
                 case '}':
-                    if(TryCompleteStringToken())
+                    if (TryCompleteStringToken())
                         yield return strToken;
                     yield return new Token
                     {
@@ -174,7 +126,7 @@ public class SaveParserEU4
                 default:
                     // Skip control characters, which are invisible and causing frontend bugs.
                     // I dont know why there are so many of them in strings.
-                    if(c >= 0x20)
+                    if (c >= 0x20)
                         str.Append((char)c);
                     break;
             }
@@ -185,23 +137,21 @@ public class SaveParserEU4
     // doesn't move next
     private object? ParseValue()
     {
-        Token tok = _tokens.Current.Value;
+        var tok = _tokens.Current.Value;
         switch (tok.type)
         {
             case TokenType.StringOrNumber:
-                if(string.IsNullOrEmpty(tok.value))
+                if (string.IsNullOrEmpty(tok.value))
                     return string.Empty;
                 if (tok.value[0] != '-' && !char.IsDigit(tok.value[0]))
                     return tok.value;
-                if(tok.value.Contains('.') && Double.TryParse(tok.value, out double d)) 
+                if (tok.value.Contains('.') && double.TryParse(tok.value, out double d))
                     return d;
-                if (Int64.TryParse(tok.value, out long l))
+                if (long.TryParse(tok.value, out long l))
                     return l;
                 return tok.value;
             case TokenType.BracketOpen:
-                _currentDepth++;
-                var obj = ParseListOrDict();
-                _currentDepth--;
+                object obj = ParseListOrDict();
                 return obj;
             case TokenType.BracketClose:
                 return null;
@@ -215,7 +165,7 @@ public class SaveParserEU4
     /// <returns>true if skipped value, false if current token is closing bracket</returns>
     private bool SkipValue()
     {
-        Token tok = _tokens.Current.Value;
+        var tok = _tokens.Current.Value;
         if (tok.type == TokenType.BracketOpen)
         {
             SkipObject();
@@ -230,7 +180,7 @@ public class SaveParserEU4
     {
         while (bracketBalance != 0 && _tokens.MoveNext())
         {
-            Token tok = _tokens.Current.Value;
+            var tok = _tokens.Current.Value;
             if (tok.type == TokenType.BracketOpen)
                 bracketBalance++;
             else if (tok.type == TokenType.BracketClose)
@@ -253,15 +203,16 @@ public class SaveParserEU4
     private List<object> ParseList()
     {
         List<object> list = new();
-        while(true)
+        while (true)
         {
-            if(!_tokens.MoveNext())
+            if (!_tokens.MoveNext())
                 throw new Exception("Unexpected end of file");
             object? value = ParseValue();
             if (value is null)
                 break;
             list.Add(value);
         }
+
         return list;
     }
 
@@ -273,7 +224,7 @@ public class SaveParserEU4
         // root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count
         for (int localIndex = 0; _tokens.MoveNext(); localIndex++)
         {
-            Token tok = _tokens.Current.Value;
+            var tok = _tokens.Current.Value;
             // end of dictionary 
             if (tok.type == TokenType.BracketClose)
                 break;
@@ -288,13 +239,13 @@ public class SaveParserEU4
                 continue;
             }
 
-            if(tok.type != TokenType.StringOrNumber)
+            if (tok.type != TokenType.StringOrNumber)
                 throw new UnexpectedTokenException(tok);
 
             string key = tok.value!;
 
             // next token should be `=` or `{`
-            if(!_tokens.MoveNext())
+            if (!_tokens.MoveNext())
                 throw new UnexpectedTokenException(tok);
             tok = _tokens.Current.Value;
             if (tok.type == TokenType.Equals)
@@ -306,23 +257,31 @@ public class SaveParserEU4
             // Saves may contain object definition without `=`.
             // Example: `map_area_data {` instead of `map_area_data = {`
             else if (tok.type != TokenType.BracketOpen)
+            {
                 throw new UnexpectedTokenException(tok);
+            }
 
-            if (!_query.DoesMatch(new SearchArgs(key, _currentDepth, localIndex)))
+            ISearchExpression? searchExprNext = null;
+            if (_searchExprCurrent != null
+                && !_searchExprCurrent.DoesMatch(new SearchArgs(key, localIndex), out searchExprNext))
             {
                 SkipValue();
                 continue;
             }
 
+            var searExpressionPrevious = _searchExprCurrent;
+            _searchExprCurrent = searchExprNext;
             object? value = ParseValue();
             if (value is null)
                 throw new UnexpectedTokenException(_tokens.Current.Value);
+            _searchExprCurrent = searExpressionPrevious;
 
-            if(!dict.TryGetValue(key, out List<object>? list))
+            if (!dict.TryGetValue(key, out var list))
             {
                 list = new List<object>();
                 dict.Add(key, list);
             }
+
             list.Add(value);
         }
 
@@ -334,4 +293,56 @@ public class SaveParserEU4
         var root = ParseDict();
         return root;
     }
+
+    protected enum TokenType : byte
+    {
+        Invalid,
+        StringOrNumber,
+        Equals,
+        BracketOpen,
+        BracketClose
+    }
+
+    protected struct Token
+    {
+        public required TokenType type;
+        public required short column;
+        public required int line;
+        public string? value;
+
+        public override string ToString()
+        {
+            string s;
+            switch (type)
+            {
+                case TokenType.Invalid:
+                    s = "INVALID_TOKEN";
+                    break;
+                case TokenType.StringOrNumber:
+                    s = value ?? "NULL";
+                    break;
+                case TokenType.Equals:
+                    s = "=";
+                    break;
+                case TokenType.BracketOpen:
+                    s = "{";
+                    break;
+                case TokenType.BracketClose:
+                    s = "}";
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(type.ToString());
+            }
+
+            return $"{line}:{column} '{s}'";
+        }
+    }
+
+    protected class UnexpectedTokenException : Exception
+    {
+        public UnexpectedTokenException(Token token) :
+            base($"Unexpected token: {token}")
+        {
+        }
+    }
 }

ParadoxSaveParser.Lib/SearchExpression.cs

@@ -1,139 +1,137 @@
-using System.Diagnostics;
-using System.Linq;
+namespace ParadoxSaveParser.Lib;
 
-namespace ParadoxSaveParser.Lib;
-
-public record SearchArgs(string key, int currentDepth, int localIndex);
+public record SearchArgs(string key, int localIndex);
 
 public interface ISearchExpression
 {
-    bool DoesMatch(SearchArgs args);
+    bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression);
 }
 
-public class SearchExpression : ISearchExpression
+public static class SearchExpressionCompiler
 {
-    private List<ISearchExpression> _compiledExpression;
-    private int _expressionDepth;
+    private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan<char> chars, int i)
+        => chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\');
 
-    private SearchExpression(List<ISearchExpression> compiledExpression, int expressionDepth)
+    public static ISearchExpression Compile(ReadOnlySpan<char> query)
     {
-        _compiledExpression = compiledExpression;
-        _expressionDepth = expressionDepth;
-    }
+        if (query.IsEmpty)
+            throw new ArgumentNullException(nameof(query));
 
-    
-    public bool DoesMatch(SearchArgs args)
-    {
-        int index = args.currentDepth - _expressionDepth;
-        if (index < 0 || index >= _compiledExpression.Count)
-            return true;
-
-        return _compiledExpression[index].DoesMatch(args);
-    }
-
-    
-    private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan<char> chars, int i) => 
-        chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\');
-
-    public static SearchExpression Parse(string query) => ParseInternal(query, 0);
-    
-    private static SearchExpression ParseInternal(ReadOnlySpan<char> query, int expressionDepth)
-    {
-        var compiledExpression = new List<ISearchExpression>();
-        ISearchExpression exprPart;
-        int partBegin = 0;
-        int bracketBalance = 0;
-        int expressionDepthIncrement = 0;
-        
-        for (int i = 0; i < query.Length; i++)
-        {
-            if (CharEqualsAndNotEscaped('(', query, i))
-                bracketBalance++;
-            else if (CharEqualsAndNotEscaped(')', query, i))
-                bracketBalance--;
-            else if (bracketBalance == 0 && CharEqualsAndNotEscaped('.', query, i))
-            {
-                var part = query.Slice(partBegin, i - partBegin);
-                expressionDepthIncrement++;
-                exprPart = ParsePart(part, query, partBegin, 
-                    expressionDepth + expressionDepthIncrement);
-                compiledExpression.Add(exprPart);
-                partBegin = i + 1;
-            }
-        }
-
-        exprPart = ParsePart(query.Slice(partBegin), query, partBegin, 
-            expressionDepth + expressionDepthIncrement);
-        compiledExpression.Add(exprPart);
-        
-        return new SearchExpression(compiledExpression, expressionDepth);
-    }
-
-    private static ISearchExpression ParsePart(ReadOnlySpan<char> part, 
-        ReadOnlySpan<char> query, int partBegin, int expressionDepth)
-    {
-        if (part is "*")
-        {
-            return new AnyMatchExpression();
-        }
-        
-        if (CharEqualsAndNotEscaped('[', query, partBegin))
-        {
-            part = part.Slice(1, part.Length - 2);
-            return new IndexMatchExpression(int.Parse(part));
-        }
-        
-        if(part[0] is '(')
+        if (query[0] is '(')
         {
             var subExprs = new List<ISearchExpression>();
-            ISearchExpression subExpr;
-            part = part.Slice(1, part.Length - 2);
-            int supExprBegin = 0;
-            for (int j = 0; j < part.Length; j++)
+            int supExprBegin = 1;
+            int bracketBalance = 1;
+            for (int i = supExprBegin; i < query.Length && bracketBalance != 0; i++)
             {
-                if (CharEqualsAndNotEscaped('|', part, j))
+                if (CharEqualsAndNotEscaped('(', query, i))
                 {
-                    subExpr = ParseInternal(part.Slice(supExprBegin, j - supExprBegin), 
-                        expressionDepth);
+                    bracketBalance++;
+                }
+                else if (CharEqualsAndNotEscaped(')', query, i))
+                {
+                    bracketBalance--;
+                }
+                else if (bracketBalance == 1 && CharEqualsAndNotEscaped('|', query, i))
+                {
+                    var subPart = query.Slice(supExprBegin, i - supExprBegin);
+                    var subExpr = Compile(subPart);
                     subExprs.Add(subExpr);
-                    supExprBegin = j + 1;
+                    supExprBegin = i + 1;
                 }
             }
 
-            subExpr = ParseInternal(part.Slice(supExprBegin), expressionDepth);
-            subExprs.Add(subExpr);
+            if (query[^1] != ')')
+                throw new NotImplementedException("Expressions after ')' are not supported");
+
+            if (bracketBalance > 0)
+                throw new Exception("Too many opening brackets");
+            if (bracketBalance < 0)
+                throw new Exception("Too many closing brackets");
+
+            var subPartLast = query.Slice(supExprBegin, query.Length - supExprBegin - 1);
+            var subExprLast = Compile(subPartLast);
+            subExprs.Add(subExprLast);
             return new MultipleMatchExpression(subExprs);
         }
 
-        return new ExactMatchExpression(part.ToString());
+        int partBeforePointLength = 0;
+        while (partBeforePointLength < query.Length)
+        {
+            if (CharEqualsAndNotEscaped('.', query, partBeforePointLength))
+                break;
+            partBeforePointLength++;
+        }
+
+        var part = query.Slice(0, partBeforePointLength);
+        ReadOnlySpan<char> remaining = default;
+        if (partBeforePointLength < query.Length)
+            remaining = query.Slice(partBeforePointLength + 1);
+        if (part is "*") return new AnyMatchExpression(remaining.IsEmpty ? null : Compile(remaining));
+
+        for (int j = 0; j < part.Length; j++)
+            if (CharEqualsAndNotEscaped('*', part, j))
+                throw new NotImplementedException("pattern matching other than '*' is not implemented yet");
+
+        if (part[0] is '[')
+        {
+            part = part.Slice(1, part.Length - 2);
+            return new IndexMatchExpression(int.Parse(part), remaining.IsEmpty ? null : Compile(remaining));
+        }
+
+        return new ExactMatchExpression(part.ToString(), remaining.IsEmpty ? null : Compile(remaining));
     }
 
-    private record AnyMatchExpression : ISearchExpression
+
+    private record AnyMatchExpression(ISearchExpression? next) : ISearchExpression
     {
-        public bool DoesMatch(SearchArgs args) => true;
+        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
+        {
+            nextSearchExpression = next;
+            return true;
+        }
     }
 
     private record MultipleMatchExpression(List<ISearchExpression> subExprs) : ISearchExpression
     {
-        public bool DoesMatch(SearchArgs args)
+        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
         {
             foreach (var e in subExprs)
-            {
-                if(e.DoesMatch(args))
+                if (e.DoesMatch(args, out nextSearchExpression))
                     return true;
-            }
+
+            nextSearchExpression = null;
             return false;
         }
     }
 
-    private record IndexMatchExpression(int index) : ISearchExpression
+    private record IndexMatchExpression(int index, ISearchExpression? next) : ISearchExpression
     {
-        public bool DoesMatch(SearchArgs args) => args.localIndex == index;
+        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
+        {
+            if (args.localIndex == index)
+            {
+                nextSearchExpression = next;
+                return true;
+            }
+
+            nextSearchExpression = null;
+            return false;
+        }
     }
 
-    private record ExactMatchExpression(string key) : ISearchExpression
+    private record ExactMatchExpression(string key, ISearchExpression? next) : ISearchExpression
     {
-        public bool DoesMatch(SearchArgs args) => args.key == key;
-    }
+        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
+        {
+            if (args.key == key)
+            {
+                nextSearchExpression = next;
+                return true;
+            }
 
+            nextSearchExpression = null;
+            return false;
+        }
+    }
 }

ParadoxSaveParser.WebAPI/Config.cs

@@ -5,9 +5,9 @@ namespace ParadoxSaveParser.WebAPI;
 public class Config
 {
     public const int ActualVersion = 1;
+    public string BaseUrl = "http://127.0.0.1:5226/";
 
     public int Version = ActualVersion;
-    public string BaseUrl = "http://127.0.0.1:5226/";
 
     public static Config FromDtsod(DtsodV23 d)
     {
@@ -18,16 +18,16 @@ public class Config
         };
         if (cfg.Version < ActualVersion)
             throw new Exception($"config is obsolete (config v{cfg.Version} < program v{ActualVersion})");
-        if(cfg.Version > ActualVersion)
+        if (cfg.Version > ActualVersion)
             throw new Exception($"program is obsolete (config v{cfg.Version} > program v{ActualVersion})");
         return cfg;
     }
 
-    public DtsodV23 ToDtsod() =>
-        new()
+    public DtsodV23 ToDtsod()
+        => new()
         {
             { "version", Version },
-            { "baseUrl", BaseUrl },
+            { "baseUrl", BaseUrl }
         };
 
     public override string ToString() => ToDtsod().ToString();

ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.csproj

@@ -9,10 +9,10 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj" />
+        <ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj"/>
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="DTLib.Web" Version="1.2.2" />
+        <PackageReference Include="DTLib.Web" Version="1.2.2"/>
     </ItemGroup>
 </Project>

ParadoxSaveParser.WebAPI/PathHelper.cs

@@ -4,7 +4,10 @@ public static class PathHelper
 {
     public static readonly IOPath DATA_DIR = "data";
     public static readonly IOPath SAVES_DIR = Path.Concat(DATA_DIR, "saves");
+
     public static IOPath GetMetaFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".meta.json");
+
     public static IOPath GetSaveFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".eu4");
+
     public static IOPath GetParsedSaveFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".parsed.json");
 }

ParadoxSaveParser.WebAPI/Program.HttpHelpers.cs

@@ -0,0 +1,97 @@
+using System.Linq;
+using System.Net;
+using System.Text.Encodings.Web;
+using System.Text.Json.Serialization;
+using DTLib.Extensions;
+
+namespace ParadoxSaveParser.WebAPI;
+
+public partial class Program
+{
+    private static readonly JsonSerializerOptions _responseJsonSerializerOptions = new()
+    {
+        WriteIndented = false,
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        MaxDepth = 1024
+    };
+
+    public static async Task<HttpStatusCode> ReturnResponseString(HttpListenerContext ctx,
+        string value, HttpStatusCode statusCode = HttpStatusCode.OK)
+    {
+        ctx.Response.StatusCode = (int)statusCode;
+        ctx.Response.ContentType = "text/plain";
+        await ctx.Response.OutputStream.WriteAsync(
+            value.ToBytes(),
+            _mainCancel.Token);
+        return statusCode;
+    }
+
+    public static async Task<HttpStatusCode> ReturnResponseJson(HttpListenerContext ctx,
+        object value, HttpStatusCode statusCode = HttpStatusCode.OK)
+    {
+        ctx.Response.StatusCode = (int)statusCode;
+        ctx.Response.ContentType = "application/json";
+        await JsonSerializer.SerializeAsync(
+            ctx.Response.OutputStream,
+            value,
+            value.GetType(),
+            _responseJsonSerializerOptions,
+            _mainCancel.Token);
+        return statusCode;
+    }
+
+    public static async Task<HttpStatusCode> ReturnResponseError(HttpListenerContext ctx, ErrorMessage error)
+    {
+        ctx.Response.StatusCode = (int)error.StatusCode;
+        ctx.Response.ContentType = "application/json";
+        await JsonSerializer.SerializeAsync(
+            ctx.Response.OutputStream,
+            error,
+            typeof(ErrorMessage),
+            _responseJsonSerializerOptions,
+            _mainCancel.Token);
+        return error.StatusCode;
+    }
+
+
+    internal static ValueOrError<string> GetRequestQueryValue(HttpListenerContext ctx, string paramName)
+    {
+        string[]? values = ctx.Request.QueryString.GetValues(paramName);
+        string? value = values?.FirstOrDefault();
+        if (string.IsNullOrEmpty(value))
+            return new ErrorMessage(HttpStatusCode.BadRequest,
+                $"No request parameter '{paramName}' provided");
+        return value;
+    }
+
+    public record ErrorMessage
+    {
+        public ErrorMessage(HttpStatusCode statusCode, string message)
+        {
+            StatusCode = statusCode;
+            Message = message;
+        }
+
+        [JsonIgnore] public HttpStatusCode StatusCode { get; }
+
+        [JsonPropertyName("errorMessage")] public string Message { get; }
+    }
+
+    public class ValueOrError<T>
+    {
+        public readonly ErrorMessage? Error;
+        public readonly T? Value;
+
+        private ValueOrError(T? value, ErrorMessage? error)
+        {
+            Value = value;
+            Error = error;
+        }
+
+        public bool HasError => Error is not null;
+
+        public static implicit operator ValueOrError<T>(T v) => new(v, null);
+
+        public static implicit operator ValueOrError<T>(ErrorMessage e) => new(default, e);
+    }
+}

ParadoxSaveParser.WebAPI/Program.RequestHandlers.cs

@@ -0,0 +1,117 @@
+using System.IO.Compression;
+using System.Linq;
+using System.Net;
+
+namespace ParadoxSaveParser.WebAPI;
+
+public partial class Program
+{
+    private static async Task<HttpStatusCode> UploadSaveHandler(HttpListenerContext ctx)
+    {
+        string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault();
+        if (contentType != "application/octet-stream")
+            return await ReturnResponseError(ctx, new ErrorMessage(
+                HttpStatusCode.BadRequest,
+                $"Invalid request Content-Type: '{contentType}'"));
+
+        string saveId = Guid.NewGuid().ToString();
+        var metaFilePath = PathHelper.GetMetaFilePath(saveId);
+        if (File.Exists(metaFilePath))
+            return await ReturnResponseError(ctx, new ErrorMessage(
+                HttpStatusCode.InternalServerError,
+                $"Guid collision! file' {metaFilePath}' already exists."));
+
+        var meta = new SaveFileMetadata
+            { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized };
+        if (!_saveMetadataStorage.TryAdd(saveId, meta))
+            return await ReturnResponseError(ctx, new ErrorMessage(
+                HttpStatusCode.InternalServerError,
+                $"Guid collision! Can't create metadata with id {saveId}"));
+
+        meta.status = SaveFileProcessingStatus.Uploading;
+        var saveFilePath = PathHelper.GetSaveFilePath(meta.id);
+        await using var saveFile = File.OpenWrite(saveFilePath);
+        await using var remoteStream = ctx.Request.InputStream;
+        await remoteStream.CopyToAsync(saveFile, _mainCancel.Token);
+        meta.status = SaveFileProcessingStatus.Uploaded;
+
+        return await ReturnResponseString(ctx, saveId);
+    }
+
+    private static ValueOrError<SaveFileMetadata> GetMetaFromRequestId(HttpListenerContext ctx,
+        string requestParamName)
+    {
+        var idOrError = GetRequestQueryValue(ctx, requestParamName);
+        if (idOrError.HasError)
+            return idOrError.Error!;
+
+        if (!_saveMetadataStorage.TryGetValue(idOrError.Value!, out var meta))
+            return new ErrorMessage(HttpStatusCode.InternalServerError,
+                $"Save with id {idOrError.Value} not found");
+
+        return meta;
+    }
+
+    private static async Task<HttpStatusCode> GetSaveStatusHandler(HttpListenerContext ctx)
+    {
+        var metaOrError = GetMetaFromRequestId(ctx, "id");
+        if (metaOrError.HasError)
+            return await ReturnResponseError(ctx, metaOrError.Error!);
+
+        return await ReturnResponseJson(ctx, metaOrError.Value!);
+    }
+
+    private static async Task<HttpStatusCode> ParseSaveEU4Handler(HttpListenerContext ctx)
+    {
+        var metaOrError = GetMetaFromRequestId(ctx, "id");
+        if (metaOrError.HasError)
+            return await ReturnResponseError(ctx, metaOrError.Error!);
+        var meta = metaOrError.Value!;
+
+        var searchQueryOrError = GetRequestQueryValue(ctx, "search");
+        if (searchQueryOrError.HasError)
+            return await ReturnResponseError(ctx, searchQueryOrError.Error!);
+        string searchQuery = searchQueryOrError.Value!;
+
+        try
+        {
+            string extractedGamestatePath = PathHelper.GetSaveFilePath(meta.id) + ".gamestate";
+            using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(meta.id).Str, ZipArchiveMode.Read))
+            {
+                var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
+                if (zipEntry is null)
+                    return await ReturnResponseError(ctx, new ErrorMessage(
+                        HttpStatusCode.BadRequest,
+                        "Invalid save format: no 'gamestate' file found"));
+
+                zipEntry.ExtractToFile(extractedGamestatePath, true);
+            }
+
+            var gamestateStream = File.OpenRead(extractedGamestatePath);
+
+            meta.status = SaveFileProcessingStatus.Parsing;
+            var se = SearchExpressionCompiler.Compile(searchQuery);
+            var parser = new SaveParserEU4(gamestateStream, se);
+            var result = parser.Parse();
+
+            meta.status = SaveFileProcessingStatus.SavingResults;
+            var resultFilePath = PathHelper.GetParsedSaveFilePath(meta.id);
+            await using var resultFile = File.OpenWrite(resultFilePath);
+            await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions, _mainCancel.Token);
+            meta.status = SaveFileProcessingStatus.Done;
+            meta.SaveToFile();
+        }
+        catch (Exception ex)
+        {
+            string errorMesage = ex.ToStringDemystified();
+            _loggerRoot.LogWarn(nameof(ParseSaveEU4Handler), errorMesage);
+            return await ReturnResponseError(ctx, new ErrorMessage(HttpStatusCode.BadRequest, errorMesage));
+        }
+        finally
+        {
+            GC.Collect();
+        }
+
+        return await ReturnResponseJson(ctx, meta);
+    }
+}

ParadoxSaveParser.WebAPI/Program.Responses.cs

@@ -1,125 +0,0 @@
-using System.IO.Compression;
-using System.Linq;
-using System.Net;
-using DTLib.Extensions;
-
-namespace ParadoxSaveParser.WebAPI;
-
-public partial class Program
-{
-    // ReSharper disable once NotAccessedPositionalProperty.Global
-    public record ErrorMessage(string errorMessage);
-
-    private static async Task<HttpStatusCode> ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode,
-        object response)
-    {
-        await JsonSerializer.SerializeAsync(ctx.Response.OutputStream, response, response.GetType(),
-            JsonSerializerOptions.Default, _mainCancel.Token);
-        ctx.Response.StatusCode = (int)statusCode;
-        return statusCode;
-    }
-
-    private static async Task<HttpStatusCode> ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode,
-        string response)
-    {
-        await ctx.Response.OutputStream.WriteAsync(response.ToBytes(), _mainCancel.Token);
-        ctx.Response.StatusCode = (int)statusCode;
-        return statusCode;
-    }
-
-    private static async Task<HttpStatusCode> UploadSaveHandler(HttpListenerContext ctx)
-    {
-        string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault();
-        if (contentType != "application/octet-stream")
-            return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
-                new ErrorMessage($"Invalid request Content-Type: '{contentType}'"));
-
-        string saveId = Guid.NewGuid().ToString();
-        IOPath metaFilePath = PathHelper.GetMetaFilePath(saveId);
-        if (File.Exists(metaFilePath))
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError,
-                new ErrorMessage($"Guid collision! file' {metaFilePath}' already exists."));
-
-        var meta = new SaveFileMetadata
-            { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized, };
-        if (!_saveMetadataStorage.TryAdd(saveId, meta))
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError,
-                new ErrorMessage($"Guid collision! Can't create metadata with id {saveId}"));
-
-        meta.status = SaveFileProcessingStatus.Uploading;
-        IOPath saveFilePath = PathHelper.GetSaveFilePath(meta.id);
-        await using var saveFile = File.OpenWrite(saveFilePath);
-        await using var remoteStream = ctx.Request.InputStream;
-        await remoteStream.CopyToAsync(saveFile, _mainCancel.Token);
-        meta.status = SaveFileProcessingStatus.Uploaded;
-
-        return await ReturnResponse(ctx, HttpStatusCode.OK, saveId);
-    }
-
-    private static (SaveFileMetadata? meta, ErrorMessage? errorMesage) GetMetaFromRequestId(HttpListenerContext ctx,
-        string requestParamName)
-    {
-        var ids = ctx.Request.QueryString.GetValues(requestParamName);
-        string? id = ids?.FirstOrDefault();
-        if (string.IsNullOrEmpty(id))
-            return (null, new ErrorMessage($"No request parameter '{requestParamName}' provided"));
-
-        if (!_saveMetadataStorage.TryGetValue(id, out var meta))
-            return (null, new ErrorMessage($"Save with {id} not found"));
-
-        return (meta, null);
-    }
-
-    private static async Task<HttpStatusCode> GetSaveStatusHandler(HttpListenerContext ctx)
-    {
-        var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id");
-        if (errorMessage is not null)
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage);
-
-        return await ReturnResponse(ctx, HttpStatusCode.OK, meta!);
-    }
-
-    private static async Task<HttpStatusCode> ParseSaveEU4Handler(HttpListenerContext ctx)
-    {
-        var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id");
-        if (errorMessage is not null)
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage);
-
-        //TODO: get actual query
-        string searchQuery = "";
-        
-        try
-        {
-            using var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(meta!.id).Str, ZipArchiveMode.Read);
-            var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
-            if (zipEntry is null)
-                return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
-                    new ErrorMessage("Invalid save format: no 'gamestate' file found"));
-
-            string extractedGamestatePath = PathHelper.GetSaveFilePath(meta.id) + ".gamestate";
-            zipEntry.ExtractToFile(extractedGamestatePath, true);
-            var gamestateStream = File.OpenRead(extractedGamestatePath);
-
-            meta.status = SaveFileProcessingStatus.Parsing;
-            var se = SearchExpression.Parse(searchQuery);
-            var parser = new SaveParserEU4(gamestateStream, se);
-            var result = parser.Parse();
-
-            meta.status = SaveFileProcessingStatus.SavingResults;
-            IOPath resultFilePath = PathHelper.GetParsedSaveFilePath(meta.id);
-            await using var resultFile = File.OpenWrite(resultFilePath);
-            await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions, _mainCancel.Token);
-            meta.status = SaveFileProcessingStatus.Done;
-            meta.SaveToFile();
-        }
-        catch (Exception ex)
-        {
-            string errorMesage = ex.ToStringDemystified();
-            _loggerRoot.LogWarn(nameof(ParseSaveEU4Handler), errorMesage);
-            return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
-                new ErrorMessage(errorMesage));
-        }
-
-        return await ReturnResponse(ctx, HttpStatusCode.OK, meta);
-    }
-}

ParadoxSaveParser.WebAPI/Program.cs

@@ -15,13 +15,12 @@ using System.Collections.Concurrent;
 using System.IO;
 using System.Text.Encodings.Web;
 using DTLib.Dtsod;
-using DTLib.Extensions;
 using DTLib.Web;
 using DTLib.Web.Routes;
 
 namespace ParadoxSaveParser.WebAPI;
 
-public partial class Program
+public static partial class Program
 {
     private static readonly IOPath _configPath = "./config.dtsod";
     private static Config _config = new();
@@ -31,77 +30,21 @@ public partial class Program
         new FileLogger("logs", "ParadoxSaveParser.WebAPI"));
 
     private static readonly CancellationTokenSource _mainCancel = new();
-    private static ConcurrentDictionary<string, SaveFileMetadata> _saveMetadataStorage = new();
+    private static readonly ConcurrentDictionary<string, SaveFileMetadata> _saveMetadataStorage = new();
 
-    private static JsonSerializerOptions _saveSerializerOptions = new()
+    private static readonly JsonSerializerOptions _saveSerializerOptions = new()
     {
-        WriteIndented = false,
+        WriteIndented = true,
         Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
-        MaxDepth = 1024,
+        MaxDepth = 1024
     };
 
-    static void TestSearchExpression(Stream saveStream, TestCase tc)
-    {
-        saveStream.Seek(0, SeekOrigin.Begin);
-        var se = SearchExpression.Parse(tc.q);
-        var parser = new SaveParserEU4(saveStream, se);
-        var rootNode = parser.Parse();
-        string json = JsonSerializer.Serialize(rootNode, _saveSerializerOptions);
-        string pdx = json.Substring(1, json.Length - 2)
-            .Replace(",", " ").Replace("{", "{ ").Replace("}", " }")
-            .Replace("\"", "").Replace("[", "").Replace("]", "").Replace(":", "=");
-        if(pdx == tc.a)
-        {
-            Console.WriteLine($"[OK] q:'{tc.q}' a:'{tc.a}'");
-        }
-        else
-        {
-            Console.WriteLine($"[Error] q:'{tc.q}' a:'{tc.a}' r:'{pdx}'");
-        }
-    }
-
-    record TestCase(string q, string a);
-    
     public static void Main(string[] args)
     {
-        
-        using var saveStream = new MemoryStream(
-            "EU4txt a={ b={ c=0 d=1 e=2 } f=3 }".ToBytes(), 
-            false);
-
-        TestCase[] testCases = [
-            new("a", 
-                "a={ b={ c=0 d=1 e=2 } f=3 }"),
-            
-            new("a.*", 
-                "a={ b={ c=0 d=1 e=2 } f=3 }"),
-            
-            new("a.b", 
-                "a={ b={ c=0 d=1 e=2 } }"),
-            
-            new("a.[0].c",
-                "a={ b={ c=0 } }"),
-            
-            new("a.[1]",
-                "a={ f=3 }"),
-            
-            new("a.b.(c|d)", 
-                "a={ b={ c=0 d=1 } }"),
-            
-            new("a.(b.e|f)",
-                "a={ b={ e=2 } f=3 }"),
-        ];
-
-        foreach (var test in testCases)
-        {
-            TestSearchExpression(saveStream, test);
-        }
-
-/*
         Console.InputEncoding = Encoding.UTF8;
         Console.OutputEncoding = Encoding.UTF8;
         Console.CursorVisible = false;
-        ContextLogger logger = new ContextLogger(nameof(Main), _loggerRoot);
+        var logger = new ContextLogger(nameof(Main), _loggerRoot);
         Console.CancelKeyPress += (_, e) =>
         {
             e.Cancel = true;
@@ -118,7 +61,10 @@ public partial class Program
                 File.WriteAllText(_configPath, _config.ToString());
                 logger.LogWarn($"created default at {_configPath}.");
             }
-            else _config = Config.FromDtsod(new DtsodV23(File.ReadAllText(_configPath)));
+            else
+            {
+                _config = Config.FromDtsod(new DtsodV23(File.ReadAllText(_configPath)));
+            }
 
             PrepareLocalFiles();
 
@@ -140,14 +86,13 @@ public partial class Program
         {
             logger.LogError(ex.ToStringDemystified());
         }
-*/
     }
 
     public static void PrepareLocalFiles()
     {
         Directory.Create(PathHelper.DATA_DIR);
         Directory.Create(PathHelper.SAVES_DIR);
-        foreach (var metaFilePath in System.IO.Directory.GetFiles(
+        foreach (string metaFilePath in System.IO.Directory.GetFiles(
                      PathHelper.SAVES_DIR.Str, "*.meta.json",
                      SearchOption.TopDirectoryOnly))
         {
@@ -155,13 +100,11 @@ public partial class Program
             var meta = JsonSerializer.Deserialize<SaveFileMetadata>(metaFile) ??
                        throw new NullReferenceException(metaFilePath);
             if (meta.status != SaveFileProcessingStatus.Done)
-            {
-                _loggerRoot.LogWarn(nameof(PrepareLocalFiles), $"metadata file '{metaFilePath}' status has invalid status {meta.status}");
-            }
+                _loggerRoot.LogWarn(nameof(PrepareLocalFiles),
+                    $"metadata file '{metaFilePath}' status has invalid status {meta.status}");
 
-            if(!_saveMetadataStorage.TryAdd(meta.id, meta))
+            if (!_saveMetadataStorage.TryAdd(meta.id, meta))
                 throw new Exception("Guid collision!");
         }
     }
-    
 }

ParadoxSaveParser.WebAPI/SaveFileMetadata.cs

@@ -4,16 +4,23 @@ namespace ParadoxSaveParser.WebAPI;
 
 public enum SaveFileProcessingStatus
 {
-    Initialized, Uploading, Uploaded, Parsing, SavingResults, Done
+    Initialized,
+    Uploading,
+    Uploaded,
+    Parsing,
+    SavingResults,
+    Done
 }
 
 public enum Game
 {
-    Unknown, EU4
+    Unknown,
+    EU4
 }
 
 public class SaveFileMetadata
 {
+    private static readonly JsonSerializerOptions _jsonOptions = new() { WriteIndented = true };
     public required string id { get; init; }
 
     [JsonConverter(typeof(JsonStringEnumConverter))]
@@ -22,8 +29,6 @@ public class SaveFileMetadata
     [JsonConverter(typeof(JsonStringEnumConverter))]
     public required SaveFileProcessingStatus status { get; set; }
 
-    
-    private static readonly JsonSerializerOptions _jsonOptions = new() { WriteIndented = true };
     public void SaveToFile()
     {
         using var metaFile = File.OpenWrite(PathHelper.GetMetaFilePath(id));

ParadoxSaveParser.sln

@@ -10,6 +10,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SolutionFolder", "SolutionF
 		TODO.txt = TODO.txt
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParadoxSaveParser.Lib.Tests", "ParadoxSaveParser.Lib.Tests\ParadoxSaveParser.Lib.Tests.csproj", "{23F4BE1B-3043-4821-9F65-74FF5F57FA59}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -24,5 +26,9 @@ Global
 		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

TODO.txt

@@ -4,6 +4,8 @@ Main:
 ParseSaveHandler:
 	Make this method run as background task instead of POST query
 	Add debug log
+	Save parsed in protobuf
+	Re-parse if saved data was parsed with another query
 	
 Parser:
-	Add query support to parse only needed information
+	Optimize it (5 sec per query isn't good)