commented protobuf compilation

ParadoxSaveParser.CLI/Modes/Interactive.cs

@@ -0,0 +1,138 @@
+using Path = DTLib.Filesystem.Path;
+
+namespace ParadoxSaveParser.CLI;
+
+internal enum Mode
+{
+    Unset, Search, Interactive
+}
+
+internal static partial class Modes
+{
+    internal static void Interactive()
+    {
+        ColoredConsole.Clear();
+        ColoredConsole.WriteTitle("interactive mode", fg: ConsoleColor.Cyan);
+        ColoredConsole.WriteLine($"working directory: '{Environment.CurrentDirectory}'", ConsoleColor.Gray);
+        IOPath? inputPath = null;
+        IOPath? outputPath = null;
+        
+        while (true)
+        {
+            try
+            {
+                ColoredConsole.Write("> ", ConsoleColor.Blue);
+                string input = ColoredConsole.ReadLine(ConsoleColor.Gray);
+                if (string.IsNullOrEmpty(input))
+                    continue;
+
+                const string helpMessage =
+                    """
+                    Commands dont have arguments. Just write command name ant it will ask you for more information.
+                    Avaliable commands:
+                        h, help - show this message
+                        q, quit, exit - close the program
+                        pwd - show working directory
+                        cd - change working directory
+                        ls - show list of files in working directory
+                        i, input - set input file path
+                        o, output - set output file path
+                        s, search - perform search in input file using expression 
+                    """;
+
+                if (input.Contains(' '))
+                {
+                    ColoredConsole.WriteLine("Commands dont have arguments." 
+                                             + " Just write command name ant it will ask you for more information.", 
+                        ConsoleColor.Red);
+                    continue;
+                }
+                
+                switch (input)
+                {
+                    default:
+                        ColoredConsole.WriteLine("Unknown command, use 'help' for a list of commands.",
+                            ConsoleColor.Red);
+                        continue;
+
+                    case "q":
+                    case "quit":
+                    case "exit":
+                        return;
+
+                    case "h":
+                    case "help":
+                        ColoredConsole.WriteLine(helpMessage, ConsoleColor.White);
+                        break;
+
+                    case "i":
+                    case "input":
+                        ColoredConsole.Write("Input file path: ", ConsoleColor.Blue);
+                        input = ColoredConsole.ReadLine(ConsoleColor.Gray);
+                        if (string.IsNullOrEmpty(input))
+                            throw new ArgumentException("Input file path is required");
+                        inputPath = input;
+                        break;
+
+                    case "o":
+                    case "output":
+                        ColoredConsole.Write("Output file path [default=stdout]: ", ConsoleColor.Blue);
+                        input = ColoredConsole.ReadLine(ConsoleColor.Gray);
+                        if(string.IsNullOrEmpty(input))
+                            outputPath = null;
+                        else outputPath = input;
+                        break;
+
+                    case "s":
+                    case "search":
+                        if (inputPath is null)
+                            throw new ArgumentException("Input file path is required");
+
+                        ColoredConsole.Write("search expression: ", ConsoleColor.Blue);
+                        var searchQuery = ColoredConsole.ReadLine(ConsoleColor.Gray);
+                        if (string.IsNullOrEmpty(searchQuery))
+                            throw new ArgumentException("Search expression is required");
+
+                        ColoredConsole.WriteHLine('-', ConsoleColor.Cyan);
+                        Console.ResetColor();
+                        Search(searchQuery, inputPath.Value, outputPath);
+                        ColoredConsole.WriteHLine('-', ConsoleColor.Green);
+                        break;
+
+                    case "pwd":
+                        ColoredConsole.WriteLine(Environment.CurrentDirectory, ConsoleColor.White);
+                        break;
+
+                    case "cd":
+                        ColoredConsole.Write("Change working directory to: ", ConsoleColor.Blue);
+                        input = ColoredConsole.ReadLine(ConsoleColor.Gray);
+                        if (!string.IsNullOrEmpty(input))
+                        {
+                            Environment.CurrentDirectory = new IOPath(input).Str;
+                            ColoredConsole.WriteLine(Environment.CurrentDirectory, ConsoleColor.Green);
+                        }
+
+                        break;
+
+                    case "ls":
+                        IOPath curdir = Directory.GetCurrent();
+                        foreach (var dir in Directory.GetDirectories(curdir))
+                        {
+                            ColoredConsole.WriteLine(dir.RemoveBase(curdir).Str + Path.Sep, ConsoleColor.Cyan);
+                        }
+
+                        foreach (var file in Directory.GetFiles(curdir))
+                        {
+                            ColoredConsole.WriteLine(file.RemoveBase(curdir).Str, ConsoleColor.White);
+                        }
+
+                        break;
+                }
+            }
+            catch (Exception ex)
+            {
+                ColoredConsole.WriteLine(ex.ToStringDemystified(), ConsoleColor.Red);
+            }
+        }
+    }
+}

ParadoxSaveParser.CLI/Modes/Search.cs

@@ -0,0 +1,49 @@
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Text.Json;
+using ParadoxSaveParser.Lib;
+
+namespace ParadoxSaveParser.CLI;
+
+internal static partial class Modes
+{
+    internal static void Search(string searchQuery, IOPath inputPath, IOPath? outputPath)
+    {
+        Stream inputStream = File.OpenRead(inputPath);
+        try
+        {
+            byte[] head4 = new byte[4];
+            inputStream.ReadExactly(head4);
+            inputStream.Seek(0, SeekOrigin.Begin);
+            if (head4.SequenceEqual<byte>([(byte)'P', (byte)'K', 3, 4]))
+            {
+                var zipArchive = new ZipArchive(inputStream, ZipArchiveMode.Read);
+                var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
+                if (zipEntry is null)
+                    throw new Exception("'gamestate' file not found in zip archive");
+                var unzipped = new MemoryStream((int)zipEntry.Length);
+                zipEntry.Open().CopyTo(unzipped);
+                zipArchive.Dispose(); // closes inputStream
+                unzipped.Seek(0, SeekOrigin.Begin);
+                inputStream = unzipped;
+            }
+
+            using var outputStream = outputPath is null
+                ? Console.OpenStandardOutput()
+                : File.OpenWrite(outputPath.Value);
+
+            var searchExpression = SearchExpressionCompiler.Compile(searchQuery);
+
+            var parser = new SaveParserEU4(inputStream, searchExpression);
+            var parsedValue = parser.Parse();
+            JsonSerializer.Serialize(outputStream, parsedValue,
+                ParsedValueJsonContext.Default.DictionaryStringObject);
+            outputStream.WriteByte((byte)'\n');
+        }
+        finally
+        {
+            inputStream.Dispose();
+        }
+    }
+}

ParadoxSaveParser.CLI/ParadoxSaveParser.CLI.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
+        <LangVersion>latest</LangVersion>
+        <Nullable>enable</Nullable>
+        <ImplicitUsings>disable</ImplicitUsings>
+        <InvariantGlobalization>true</InvariantGlobalization>
+        <PublishAot>true</PublishAot>
+    </PropertyGroup>
+    
+    <ItemGroup>
+        <ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj"/>
+    </ItemGroup>
+
+    <ItemGroup>
+        <PackageReference Include="DTLib" Version="1.7.4" />
+    </ItemGroup>
+</Project>

ParadoxSaveParser.CLI/ParadoxSaveParser.CLI.csproj.DotSettings

@@ -0,0 +1,2 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=modes/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

ParadoxSaveParser.CLI/Program.cs

@@ -0,0 +1,70 @@
+global using System;
+global using DTLib.Console;
+global using DTLib.Demystifier;
+global using DTLib.Filesystem;
+global using Directory = DTLib.Filesystem.Directory;
+global using File = DTLib.Filesystem.File;
+using ParadoxSaveParser.CLI;
+
+try
+{
+    Mode mode = Mode.Unset;
+    IOPath? inputPath = null;
+    IOPath? outputPath = null;
+    string? searchQuery = null;
+
+    new LaunchArgumentParser(
+            new LaunchArgument(["-i", "--input"],
+                "Set input file path",
+                s => inputPath = s,
+                "gamestate or zip file"),
+
+            new LaunchArgument(["-o", "--output"],
+                "Set output file path",
+                s => outputPath = s,
+                "json file [default=stdout]"),
+
+            new LaunchArgument(["-s", "--search"],
+                "Search in input file",
+                s =>
+                {
+                    searchQuery = s;
+                    mode = Mode.Search;
+                },
+                "search expression")
+        )
+        .AllowNoArguments()
+        .ParseAndHandle(args);
+
+    if (args.Length == 0)
+        mode = Mode.Interactive;
+
+    switch (mode)
+    {
+        default:
+            throw new ArgumentOutOfRangeException(nameof(mode));
+        case Mode.Unset:
+            throw new Exception("No action specified");
+        case Mode.Search:
+            if (string.IsNullOrEmpty(searchQuery))
+                throw new ArgumentException("Search expression is required");
+            if (inputPath is null)
+                throw new ArgumentException("Input file path is required");
+            Modes.Search(searchQuery, inputPath.Value, outputPath);
+            break;
+        case Mode.Interactive:
+            Modes.Interactive();
+            break;
+    }
+}
+catch (LaunchArgumentParser.ExitAfterHelpException)
+{
+    // this exception is throwed after -h argument to close the program
+}
+catch (Exception ex)
+{
+    ColoredConsole.WriteLine(ex.ToStringDemystified(), ConsoleColor.Red);
+    Console.ResetColor();
+    Environment.Exit(1);
+}
+Console.ResetColor();

ParadoxSaveParser.Lib.Tests/ParadoxSaveParser.Lib.Tests.csproj

@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <LangVersion>latest</LangVersion>
+        <Nullable>enable</Nullable>
+        <ImplicitUsings>disable</ImplicitUsings>
+        <InvariantGlobalization>true</InvariantGlobalization>
+
+        <IsPackable>false</IsPackable>
+        <IsTestProject>true</IsTestProject>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj"/>
+    </ItemGroup>
+    
+    <ItemGroup>
+        <Using Include="NUnit.Framework"/>
+    </ItemGroup>
+    
+    <ItemGroup>
+        <PackageReference Include="coverlet.collector" Version="6.0.4">
+          <PrivateAssets>all</PrivateAssets>
+          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
+        <PackageReference Include="DTLib" Version="1.7.4" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
+        <PackageReference Include="NUnit" Version="4.3.2" />
+        <PackageReference Include="NUnit.Analyzers" Version="4.7.0">
+          <PrivateAssets>all</PrivateAssets>
+          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
+        <PackageReference Include="NUnit3TestAdapter" Version="5.0.0" />
+    </ItemGroup>
+</Project>

ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs

@@ -0,0 +1,52 @@
+using System.IO;
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using DTLib.Extensions;
+
+namespace ParadoxSaveParser.Lib.Tests;
+
+[TestFixture]
+[TestOf(typeof(ISearchExpression))]
+public class SearchExpressionTests
+{
+    [SetUp]
+    public void Setup()
+    {
+        _smallSaveData = "EU4txt a={ b={ c=0 d=1 e=2 } f=3 }".ToBytes();
+    }
+
+    private byte[] _smallSaveData;
+
+
+    private static readonly JsonSerializerOptions _smallSaveSerializerOptions = new()
+    {
+        WriteIndented = false,
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        MaxDepth = 1024,
+        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+    };
+
+    internal static string JsonToPdx(string json)
+        => json.Substring(1, json.Length - 2)
+            .Replace(",", " ").Replace("{", "{ ").Replace("}", " }")
+            .Replace("\"", "").Replace("[", "").Replace("]", "").Replace(":", "=");
+
+    [TestCase("a", "a={ b={ c=0 d=1 e=2 } f=3 }")]
+    [TestCase("a.*", "a={ b={ c=0 d=1 e=2 } f=3 }")]
+    [TestCase("a.b", "a={ b={ c=0 d=1 e=2 } }")]
+    [TestCase("a.[0].c", "a={ b={ c=0 } }")]
+    [TestCase("a.[1]", "a={ f=3 }")]
+    [TestCase("a.b.(c|d)", "a={ b={ c=0 d=1 } }")]
+    [TestCase("a.(b.e|f)", "a={ b={ e=2 } f=3 }")]
+    public void TestSearchOnSmallData(string input, string expectedOutput)
+    {
+        using var saveStream = new MemoryStream(_smallSaveData, false);
+        var se = SearchExpressionCompiler.Compile(input);
+        var parser = new SaveParserEU4(saveStream, se);
+        var rootNode = parser.Parse();
+        string json = JsonSerializer.Serialize(rootNode, _smallSaveSerializerOptions);
+        string pdx = JsonToPdx(json);
+        Assert.That(pdx, Is.EqualTo(expectedOutput));
+    }
+}

ParadoxSaveParser.Lib/BufferedEnumerator.cs

@@ -37,35 +37,73 @@ namespace ParadoxSaveParser.Lib;
 /// 2 3 | 4 | 5
 /// 3 4 | 5 |
 /// </code>
-public class BufferedEnumerator<T> : IEnumerator<LinkedListNode<T>>
+public class BufferedEnumerator<T> : IEnumerator<BufferedEnumerator<T>.Node>
 {
-    private IEnumerator<T> _enumerator;
+    public class Node
-    private int _bufferSize;
+    {
-    LinkedList<T> _llist = new();
+#nullable disable
-    private LinkedListNode<T>? _currentNode;
+        public Node Previous;
-    private int _currentNodeIndex = -1;
+        public Node Next;
+        public T Value;
+#nullable enable
+    }
+    
+    private readonly IEnumerator<T> _enumerator;
+    private readonly Node[] _ringBuffer;
+    private Node? _currentNode;
+    private int _currentBufferIndex = -1;
+    private int _lastValueIndex = -1;
 
     public BufferedEnumerator(IEnumerator<T> enumerator, int bufferSize)
     {
         _enumerator = enumerator;
-        _bufferSize = bufferSize;
+        _ringBuffer = new Node[bufferSize];
+    }
+
+    private void InitBuffer()
+    {
+        _ringBuffer[0] = new Node
+        {
+            Value = default!
+        };
+        for (int i = 1; i < _ringBuffer.Length; i++)
+        {
+            _ringBuffer[i] = new Node
+            {
+                Previous = _ringBuffer[i - 1],
+                Value = default!,
+            };
+            _ringBuffer[i - 1].Next = _ringBuffer[i];
+        }
+        _ringBuffer[^1].Next = _ringBuffer[0];
+        _ringBuffer[0].Previous = _ringBuffer[^1];
     }
     
     public bool MoveNext()
     {
-        if(_currentNodeIndex >= _bufferSize / 2)
+        if (_currentBufferIndex == -1)
-            _llist.RemoveFirst();
-        
-        while (_llist.Count < _bufferSize && _enumerator.MoveNext())
         {
-            _llist.AddLast(_enumerator.Current);
+            InitBuffer();
+            
+            int beforeMidpoint = _ringBuffer.Length / 2 - 1;
+            for (int i = 0; i <= beforeMidpoint && _enumerator.MoveNext(); i++)
+            {
+                _ringBuffer[i].Value = _enumerator.Current;
             }
-        if (_llist.Count == 0)
+        }
+        
+        _currentBufferIndex = (_currentBufferIndex + 1) % _ringBuffer.Length;
+        if (_enumerator.MoveNext())
+        {
+            int midpoint = (_currentBufferIndex + _ringBuffer.Length / 2) % _ringBuffer.Length;
+            _ringBuffer[midpoint].Value = _enumerator.Current;
+            _lastValueIndex = midpoint;
+        }
+        if(_currentBufferIndex == (_lastValueIndex + 1) % _ringBuffer.Length)
             return false;
         
-        _currentNodeIndex++;
+        _currentNode = _ringBuffer[_currentBufferIndex];
-        _currentNode = _currentNode is null ? _llist.First : _currentNode.Next;
+        return true;
-        return _currentNode is not null;
     }
 
     public void Reset()
@@ -73,7 +111,7 @@ public class BufferedEnumerator<T> : IEnumerator<LinkedListNode<T>>
         throw new NotImplementedException();
     }
 
-    public LinkedListNode<T> Current => _currentNode!;
+    public Node Current => _currentNode!;
 
     object IEnumerator.Current => Current;
 

ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj

@@ -1,8 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
     <PropertyGroup>
         <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>disable</ImplicitUsings>
+        <LangVersion>latest</LangVersion>
         <Nullable>enable</Nullable>
+        <ImplicitUsings>disable</ImplicitUsings>
+        <InvariantGlobalization>true</InvariantGlobalization>
     </PropertyGroup>
     
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.ObjectPool" Version="9.0.5" />
+    </ItemGroup>
 </Project>

ParadoxSaveParser.Lib/ParsedValueJsonContext.cs

@@ -0,0 +1,14 @@
+using System.Text.Json.Serialization;
+
+namespace ParadoxSaveParser.Lib;
+
+[JsonSourceGenerationOptions(MaxDepth = 1024, WriteIndented = true)]
+[JsonSerializable(typeof(Dictionary<string, object>))]
+[JsonSerializable(typeof(List<object>))]
+[JsonSerializable(typeof(string))]
+[JsonSerializable(typeof(long))]
+[JsonSerializable(typeof(double))]
+[JsonSerializable(typeof(bool))]
+public partial class ParsedValueJsonContext : JsonSerializerContext
+{
+}

ParadoxSaveParser.Lib/SaveParserEU4.cs

@@ -2,6 +2,7 @@
 global using System.Collections.Generic;
 global using System.IO;
 global using System.Text;
+using Microsoft.Extensions.ObjectPool;
 
 namespace ParadoxSaveParser.Lib;
 
@@ -10,70 +11,30 @@ namespace ParadoxSaveParser.Lib;
 /// </summary>
 public class SaveParserEU4
 {
-    protected Stream _saveFile;
+    protected readonly Stream _saveFile;
-    private BufferedEnumerator<Token> _tokens;
+    private readonly BufferedEnumerator<Token> _tokens;
-    private SearchExpression _query;
+    private readonly ObjectPool<StringBuilder> _stringBuilderPool;
-    private int _currentDepth;
+    private ISearchExpression? _searchExprCurrent;
 
-    /// <param name="savefile">Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive</param>
+    /// <param name="savefile">
-    /// <param name="query">Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM,
+    ///     Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive
-    /// so you should specify what exactly you want to get from save file</param>
+    /// </param>
-    public SaveParserEU4(Stream savefile, SearchExpression query)
+    /// <param name="query">
+    ///     Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM,
+    ///     so you should specify what exactly you want to get from save file
+    /// </param>
+    public SaveParserEU4(Stream savefile, ISearchExpression? query)
     {
-        _tokens = new BufferedEnumerator<Token>(LexTextSave(), 5);
         _saveFile = savefile;
-        _query = query;
+        _searchExprCurrent = query;
-    }
+        const int tokenBufSize = 5;
-
+        _tokens = new BufferedEnumerator<Token>(LexTextSave(), tokenBufSize);
-    protected enum TokenType : byte
+        _stringBuilderPool = new DefaultObjectPool<StringBuilder>(
+            new StringBuilderPooledObjectPolicy
             {
-        Invalid,
+                InitialCapacity = tokenBufSize * 13,
-        StringOrNumber,
+                MaximumRetainedCapacity = tokenBufSize * 13,
-        Equals,
+            });
-        BracketOpen,
-        BracketClose,
-    }
-
-    protected struct Token
-    {
-        public required TokenType type;
-        public required short column;
-        public required int line;
-        public string? value;
-
-        public override string ToString()
-        {
-            string s;
-            switch (type)
-            {
-                case TokenType.Invalid:
-                    s = "INVALID_TOKEN";
-                    break;
-                case TokenType.StringOrNumber:
-                    s = value ?? "NULL";
-                    break;
-                case TokenType.Equals:
-                    s = "=";
-                    break;
-                case TokenType.BracketOpen:
-                    s = "{";
-                    break;
-                case TokenType.BracketClose:
-                    s = "}";
-                    break;
-                default:
-                    throw new ArgumentOutOfRangeException(type.ToString());
-            }
-            
-            return $"{line}:{column} '{s}'";
-        }
-    }
-
-    protected class UnexpectedTokenException : Exception
-    {
-        public UnexpectedTokenException(Token token) :
-            base($"Unexpected token: {token}")
-        {}
     }
 
     protected IEnumerator<Token> LexTextSave()
@@ -85,7 +46,7 @@ public class SaveParserEU4
         if (headStr != expectedHeader)
             throw new Exception($"Invalid gamestate header. Expected '{expectedHeader}', got '{headStr}'.");
 
-        StringBuilder str = new();
+        StringBuilder strb = _stringBuilderPool.Get();
         int line = 2;
         int column = 0;
         bool isQuoteOpen = false;
@@ -94,7 +55,8 @@ public class SaveParserEU4
         {
             type = TokenType.Invalid,
             column = -1,
-            line = -1
+            line = -1,
+            value = null,
         };
 
         bool TryCompleteStringToken()
@@ -103,17 +65,17 @@ public class SaveParserEU4
                 return false;
 
             // strings in quotes may be empty
-            if (!isStrInQuotes && (str.Length <= 0 || str[0] == '#')) 
+            if (!isStrInQuotes && (strb.Length <= 0 || strb[0] == '#'))
                 return false;
 
             strToken = new Token
             {
                 type = TokenType.StringOrNumber,
-                column = (short)(column - str.Length),
+                column = (short)(column - strb.Length),
                 line = line,
-                value = str.ToString()
+                value = strb,
             };
-            str.Clear();
+            strb = _stringBuilderPool.Get();
             isStrInQuotes = false;
             return true;
         }
@@ -127,6 +89,7 @@ public class SaveParserEU4
                 case -1:
                     if (TryCompleteStringToken())
                         yield return strToken;
+                    _stringBuilderPool.Return(strb);
                     yield break;
                 case '\"':
                     isQuoteOpen = !isQuoteOpen;
@@ -175,33 +138,47 @@ public class SaveParserEU4
                     // Skip control characters, which are invisible and causing frontend bugs.
                     // I dont know why there are so many of them in strings.
                     if (c >= 0x20)
-                        str.Append((char)c);
+                        strb.Append((char)c);
                     break;
             }
         }
+
+        _stringBuilderPool.Return(strb);
     }
 
 
     // doesn't move next
     private object? ParseValue()
     {
-        Token tok = _tokens.Current.Value;
+        var tok = _tokens.Current.Value;
         switch (tok.type)
         {
             case TokenType.StringOrNumber:
-                if(string.IsNullOrEmpty(tok.value))
+                try
+                {
+                    // string values can be empty
+                    if (tok.value!.Length == 0)
                         return string.Empty;
-                if (tok.value[0] != '-' && !char.IsDigit(tok.value[0])) 
+                    if (tok.value.Equals("yes"))
-                    return tok.value;
+                        return true;
-                if(tok.value.Contains('.') && Double.TryParse(tok.value, out double d)) 
+                    if (tok.value.Equals("no"))
+                        return false;
+                    
+                    string tokStr = tok.value.ToString();
+                    if (tokStr[0] != '-' && !char.IsDigit(tokStr[0]))
+                        return tokStr;
+                    if (tokStr.Contains('.') && double.TryParse(tokStr, out double d))
                         return d;
-                if (Int64.TryParse(tok.value, out long l))
+                    if (long.TryParse(tokStr, out long l))
                         return l;
-                return tok.value;
+                    return tokStr;
+                }
+                finally
+                {
+                    _stringBuilderPool.Return(tok.value!);
+                }
             case TokenType.BracketOpen:
-                _currentDepth++;
+                object obj = ParseListOrDict();
-                var obj = ParseListOrDict();
-                _currentDepth--;
                 return obj;
             case TokenType.BracketClose:
                 return null;
@@ -215,14 +192,20 @@ public class SaveParserEU4
     /// <returns>true if skipped value, false if current token is closing bracket</returns>
     private bool SkipValue()
     {
-        Token tok = _tokens.Current.Value;
+        var tok = _tokens.Current.Value;
-        if (tok.type == TokenType.BracketOpen)
+        switch (tok.type)
         {
+            case TokenType.BracketOpen:
                 SkipObject();
                 return true;
+            case TokenType.StringOrNumber:
+                _stringBuilderPool.Return(tok.value!);
+                return true;
+            case TokenType.BracketClose:
+                return false;
+            default:
+                throw new UnexpectedTokenException(tok);
         }
-
-        return tok.type != TokenType.BracketClose;
     }
 
     // skips all tokens inside curly braces block
@@ -230,13 +213,20 @@ public class SaveParserEU4
     {
         while (bracketBalance != 0 && _tokens.MoveNext())
         {
-            Token tok = _tokens.Current.Value;
+            var tok = _tokens.Current.Value;
             if (tok.type == TokenType.BracketOpen)
                 bracketBalance++;
             else if (tok.type == TokenType.BracketClose)
                 bracketBalance--;
+            else if (tok.type == TokenType.StringOrNumber)
+            {
+                _stringBuilderPool.Return(tok.value!);
             }
         }
+    }
+
+    private static bool IsEmptyCollection(object value)
+        => value is Dictionary<string, object> { Count: 0 } or List<object> { Count: 0 };
 
     // doesn't move next
     private object ParseListOrDict()
@@ -253,27 +243,45 @@ public class SaveParserEU4
     private List<object> ParseList()
     {
         List<object> list = new();
-        while(true)
+        for (int i = 0; ; i++)
         {
             if (!_tokens.MoveNext())
                 throw new Exception("Unexpected end of file");
+
+            ISearchExpression? searchExprNext = null;
+            if (_searchExprCurrent != null 
+                && !_searchExprCurrent.DoesMatch(new SearchArgs(i, string.Empty), out searchExprNext))
+            {
+                if(!SkipValue())
+                    break;
+                continue;
+            }
+            var searchExprPrev = _searchExprCurrent;
+            _searchExprCurrent = searchExprNext;
             object? value = ParseValue();
+            _searchExprCurrent = searchExprPrev;
             if (value is null)
                 break;
+
+            // do dot add empty collections into list
+            if (IsEmptyCollection(value))
+                continue;
+            
             list.Add(value);
         }
+
         return list;
     }
 
     // moves next
-    private Dictionary<string, List<object>> ParseDict()
+    private Dictionary<string, object> ParseDict()
     {
-        Dictionary<string, List<object>> dict = new();
+        Dictionary<string, object> dict = new();
 
         // root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count
         for (int localIndex = 0; _tokens.MoveNext(); localIndex++)
         {
-            Token tok = _tokens.Current.Value;
+            var tok = _tokens.Current.Value;
             // end of dictionary 
             if (tok.type == TokenType.BracketClose)
                 break;
@@ -291,7 +299,7 @@ public class SaveParserEU4
             if (tok.type != TokenType.StringOrNumber)
                 throw new UnexpectedTokenException(tok);
 
-            string key = tok.value!;
+            var keySB = tok.value!;
 
             // next token should be `=` or `{`
             if (!_tokens.MoveNext())
@@ -306,32 +314,111 @@ public class SaveParserEU4
             // Saves may contain object definition without `=`.
             // Example: `map_area_data {` instead of `map_area_data = {`
             else if (tok.type != TokenType.BracketOpen)
-                throw new UnexpectedTokenException(tok);
-            
-            if (!_query.DoesMatch(new SearchArgs(key, _currentDepth, localIndex)))
             {
-                SkipValue();
+                throw new UnexpectedTokenException(tok);
+            }
+
+            ISearchExpression? searchExprNext = null;
+            if (_searchExprCurrent != null
+                && !_searchExprCurrent.DoesMatch(new SearchArgs(localIndex, keySB), out searchExprNext))
+            {
+                if(!SkipValue())
+                    throw new UnexpectedTokenException(_tokens.Current.Value);
+                _stringBuilderPool.Return(keySB);
                 continue;
             }
 
+            var searExpressionPrevious = _searchExprCurrent;
+            _searchExprCurrent = searchExprNext;
             object? value = ParseValue();
             if (value is null)
                 throw new UnexpectedTokenException(_tokens.Current.Value);
+            _searchExprCurrent = searExpressionPrevious;
 
-            if(!dict.TryGetValue(key, out List<object>? list))
+            string keyStr = keySB.ToString();
+            _stringBuilderPool.Return(keySB);
+            
+            // Paradox save format has another way of defining list:
+            // a = 1
+            // a = 2
+            // It means `a = { 1 2 }`
+            if (dict.TryGetValue(keyStr, out var firstValue))
             {
-                list = new List<object>();
+                // Do dot add empty collections into list.
-                dict.Add(key, list);
+                // `key:{}` is okay, but i don't want to see `key:[{},{},{},{},{},{}]`
+                if (IsEmptyCollection(value))
+                    continue;
+                
+                if (firstValue is List<object> existingList)
+                    existingList.Add(value);
+                else dict[keyStr] = new List<object> { firstValue, value };
+            }
+            else
+            {
+                dict.Add(keyStr, value);
             }
-            list.Add(value);
         }
 
         return dict;
     }
 
-    public Dictionary<string, List<object>> Parse()
+    public Dictionary<string, object> Parse()
     {
         var root = ParseDict();
         return root;
     }
+
+    protected enum TokenType : byte
+    {
+        Invalid,
+        StringOrNumber,
+        Equals,
+        BracketOpen,
+        BracketClose
+    }
+
+    protected struct Token
+    {
+        public required TokenType type;
+        public required short column;
+        public required int line;
+        public StringBuilder? value;
+
+        public override string ToString()
+        {
+            string s;
+            switch (type)
+            {
+                case TokenType.Invalid:
+                    s = "INVALID_TOKEN";
+                    break;
+                case TokenType.StringOrNumber:
+                    if (value == null || value.Length == 0)
+                        s = "NULL";
+                    else s = value.ToString();
+                    break;
+                case TokenType.Equals:
+                    s = "=";
+                    break;
+                case TokenType.BracketOpen:
+                    s = "{";
+                    break;
+                case TokenType.BracketClose:
+                    s = "}";
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(type.ToString());
+            }
+
+            return $"{line}:{column} '{s}'";
+        }
+    }
+
+    protected class UnexpectedTokenException : Exception
+    {
+        public UnexpectedTokenException(Token token) :
+            base($"Unexpected token: {token}")
+        {
+        }
+    }
 }

ParadoxSaveParser.Lib/SearchExpression.cs

@@ -1,139 +1,169 @@
-using System.Diagnostics;
+namespace ParadoxSaveParser.Lib;
-using System.Linq;
 
-namespace ParadoxSaveParser.Lib;
+public readonly record struct SearchArgs
+{
+    public readonly string KeyStr;
+    public readonly StringBuilder? KeySB;
+    public readonly int LocalIndex;
 
-public record SearchArgs(string key, int currentDepth, int localIndex);
+    public SearchArgs(int localIndex, string keyStr)
+    {
+        KeyStr = keyStr;
+        KeySB = null;
+        LocalIndex = localIndex;
+    }
+    
+    public SearchArgs(int localIndex, StringBuilder keySb)
+    {
+        KeyStr = string.Empty;
+        KeySB = keySb;
+        LocalIndex = localIndex;
+    }
+}
 
 public interface ISearchExpression
 {
-    bool DoesMatch(SearchArgs args);
+    bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression);
 }
 
-public class SearchExpression : ISearchExpression
+public static class SearchExpressionCompiler
 {
-    private List<ISearchExpression> _compiledExpression;
+    private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan<char> chars, int i)
-    private int _expressionDepth;
+        => chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\');
 
-    private SearchExpression(List<ISearchExpression> compiledExpression, int expressionDepth)
+    public static ISearchExpression Compile(ReadOnlySpan<char> query)
     {
-        _compiledExpression = compiledExpression;
+        if (query.IsEmpty)
-        _expressionDepth = expressionDepth;
+            throw new ArgumentNullException(nameof(query));
-    }
 
-    
+        if (query[0] is '(')
-    public bool DoesMatch(SearchArgs args)
-    {
-        int index = args.currentDepth - _expressionDepth;
-        if (index < 0 || index >= _compiledExpression.Count)
-            return true;
-
-        return _compiledExpression[index].DoesMatch(args);
-    }
-
-    
-    private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan<char> chars, int i) => 
-        chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\');
-
-    public static SearchExpression Parse(string query) => ParseInternal(query, 0);
-    
-    private static SearchExpression ParseInternal(ReadOnlySpan<char> query, int expressionDepth)
-    {
-        var compiledExpression = new List<ISearchExpression>();
-        ISearchExpression exprPart;
-        int partBegin = 0;
-        int bracketBalance = 0;
-        int expressionDepthIncrement = 0;
-        
-        for (int i = 0; i < query.Length; i++)
-        {
-            if (CharEqualsAndNotEscaped('(', query, i))
-                bracketBalance++;
-            else if (CharEqualsAndNotEscaped(')', query, i))
-                bracketBalance--;
-            else if (bracketBalance == 0 && CharEqualsAndNotEscaped('.', query, i))
-            {
-                var part = query.Slice(partBegin, i - partBegin);
-                expressionDepthIncrement++;
-                exprPart = ParsePart(part, query, partBegin, 
-                    expressionDepth + expressionDepthIncrement);
-                compiledExpression.Add(exprPart);
-                partBegin = i + 1;
-            }
-        }
-
-        exprPart = ParsePart(query.Slice(partBegin), query, partBegin, 
-            expressionDepth + expressionDepthIncrement);
-        compiledExpression.Add(exprPart);
-        
-        return new SearchExpression(compiledExpression, expressionDepth);
-    }
-
-    private static ISearchExpression ParsePart(ReadOnlySpan<char> part, 
-        ReadOnlySpan<char> query, int partBegin, int expressionDepth)
-    {
-        if (part is "*")
-        {
-            return new AnyMatchExpression();
-        }
-        
-        if (CharEqualsAndNotEscaped('[', query, partBegin))
-        {
-            part = part.Slice(1, part.Length - 2);
-            return new IndexMatchExpression(int.Parse(part));
-        }
-        
-        if(part[0] is '(')
         {
             var subExprs = new List<ISearchExpression>();
-            ISearchExpression subExpr;
+            int supExprBegin = 1;
-            part = part.Slice(1, part.Length - 2);
+            int bracketBalance = 1;
-            int supExprBegin = 0;
+            int i = supExprBegin;
-            for (int j = 0; j < part.Length; j++)
+            for (; i < query.Length && bracketBalance != 0; i++)
             {
-                if (CharEqualsAndNotEscaped('|', part, j))
+                if (CharEqualsAndNotEscaped('(', query, i))
                 {
-                    subExpr = ParseInternal(part.Slice(supExprBegin, j - supExprBegin), 
+                    bracketBalance++;
-                        expressionDepth);
+                }
+                else if (CharEqualsAndNotEscaped(')', query, i))
+                {
+                    bracketBalance--;
+                }
+                else if (bracketBalance == 1 && CharEqualsAndNotEscaped('|', query, i))
+                {
+                    var subPart = query.Slice(supExprBegin, i - supExprBegin);
+                    var subExpr = Compile(subPart);
                     subExprs.Add(subExpr);
-                    supExprBegin = j + 1;
+                    supExprBegin = i + 1;
                 }
             }
 
-            subExpr = ParseInternal(part.Slice(supExprBegin), expressionDepth);
+            if (i != query.Length)
-            subExprs.Add(subExpr);
+                throw new NotImplementedException("Expressions after ')' are not supported");
+
+            if (bracketBalance > 0)
+                throw new Exception("Too many opening brackets");
+            if (bracketBalance < 0)
+                throw new Exception("Too many closing brackets");
+
+            var subPartLast = query.Slice(supExprBegin, i - 1 - supExprBegin);
+            var subExprLast = Compile(subPartLast);
+            subExprs.Add(subExprLast);
             return new MultipleMatchExpression(subExprs);
         }
 
-        return new ExactMatchExpression(part.ToString());
+        int partBeforePointLength = 0;
+        while (partBeforePointLength < query.Length)
+        {
+            if (CharEqualsAndNotEscaped('.', query, partBeforePointLength))
+                break;
+            partBeforePointLength++;
         }
 
-    private record AnyMatchExpression : ISearchExpression
+        var part = query.Slice(0, partBeforePointLength);
+        ReadOnlySpan<char> remaining = default;
+        if (partBeforePointLength < query.Length)
+            remaining = query.Slice(partBeforePointLength + 1);
+        if (part is "*") 
+            return new AnyMatchExpression(remaining.IsEmpty ? null : Compile(remaining));
+        if (part is "~")
+            return new NoMatchExpression();
+        
+        for (int j = 0; j < part.Length; j++)
+            if (CharEqualsAndNotEscaped('*', part, j))
+                throw new NotImplementedException("pattern matching other than '*' is not implemented yet");
+
+        if (part[0] is '[')
         {
-        public bool DoesMatch(SearchArgs args) => true;
+            part = part.Slice(1, part.Length - 2);
+            return new IndexMatchExpression(int.Parse(part), remaining.IsEmpty ? null : Compile(remaining));
         }
 
-    private record MultipleMatchExpression(List<ISearchExpression> subExprs) : ISearchExpression
+        return new ExactMatchExpression(part.ToString(), remaining.IsEmpty ? null : Compile(remaining));
+    }
+
+
+    private record AnyMatchExpression(ISearchExpression? next) : ISearchExpression
     {
-        public bool DoesMatch(SearchArgs args)
+        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
         {
-            foreach (var e in subExprs)
+            nextSearchExpression = next;
-            {
-                if(e.DoesMatch(args))
             return true;
         }
+    }
+
+    private record NoMatchExpression : ISearchExpression
+    {
+        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
+        {
+            nextSearchExpression = null;
             return false;
         }
     }
 
-    private record IndexMatchExpression(int index) : ISearchExpression
+    private record MultipleMatchExpression(List<ISearchExpression> subExprs) : ISearchExpression
     {
-        public bool DoesMatch(SearchArgs args) => args.localIndex == index;
+        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
+        {
+            foreach (var e in subExprs)
+                if (e.DoesMatch(args, out nextSearchExpression))
+                    return true;
+
+            nextSearchExpression = null;
+            return false;
+        }
     }
 
-    private record ExactMatchExpression(string key) : ISearchExpression
+    private record IndexMatchExpression(int index, ISearchExpression? next) : ISearchExpression
     {
-        public bool DoesMatch(SearchArgs args) => args.key == key;
+        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
+        {
+            if (args.LocalIndex == index)
+            {
+                nextSearchExpression = next;
+                return true;
             }
 
+            nextSearchExpression = null;
+            return false;
+        }
+    }
+
+    private record ExactMatchExpression(string key, ISearchExpression? next) : ISearchExpression
+    {
+        public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
+        {
+            if ((args.KeySB != null && args.KeySB.Equals(key)) || args.KeyStr == key)
+            {
+                nextSearchExpression = next;
+                return true;
+            }
+
+            nextSearchExpression = null;
+            return false;
+        }
+    }
 }

ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs

@@ -0,0 +1,25 @@
+using ParadoxSaveParser.WebAPI.Database;
+using ParadoxSaveParser.WebAPI.SaveDataFilters;
+
+namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
+
+public class BackgroundJobManager
+{
+    private readonly ILogger _parentLogger;
+    private long _lastJobId;
+
+    public BackgroundJobManager(ILogger logger)
+    {
+        _parentLogger = logger;
+    }
+    
+    public SaveParsingOperation StartNewParsingOperation(
+        SaveFileMetadata meta, ISaveDataFilter filter, CancellationToken ct)
+    {
+        long nextId = Interlocked.Increment(ref _lastJobId);
+        var contextLogger = new ContextLogger($"BackgroundJob-{nextId}", _parentLogger);
+        var op = new SaveParsingOperation(nextId, meta, filter, contextLogger, ct);
+        op.StartAsync();
+        return op;
+    }
+}

ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs

@@ -0,0 +1,102 @@
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Text.Encodings.Web;
+using System.Text.Json.Serialization;
+using ParadoxSaveParser.WebAPI.Database;
+using ParadoxSaveParser.WebAPI.SaveDataFilters;
+
+namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
+
+public class SaveParsingOperation
+{
+    public readonly long OperationId;
+    
+    private readonly SaveFileMetadata _meta;
+    private readonly ISaveDataFilter _filter;
+    private readonly ContextLogger _logger;
+    private readonly CancellationToken _ct;
+
+    private static readonly JsonSerializerOptions _jsonOptions = new()
+    {
+        WriteIndented = false,
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        MaxDepth = 1024,
+        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+    };
+
+    public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISaveDataFilter filter,
+        ContextLogger logger, CancellationToken ct)
+    {
+        OperationId = operationId;
+        _meta = meta;
+        _filter = filter;
+        _logger = logger;
+        _ct = ct;
+    }
+
+    public async void StartAsync()
+    {
+        try
+        {
+            _logger.LogInfo($"Starting background parsing operation of {_meta.game} save {_meta.id}");
+            switch (_meta.game)
+            {
+                case Game.EU4:
+                    await ParseSaveEU4();
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(_meta.game.ToString());
+            }
+            _logger.LogInfo($"Finished parsing operation of {_meta.game} save {_meta.id}");
+        }
+        catch (Exception ex)
+        {
+            string errorMesage = ex.ToStringDemystified();
+            _logger.LogError(errorMesage);
+            try
+            {
+                await Program.DB.SetMetadataError(_meta, errorMesage);
+            }
+            catch (Exception ex2)
+            {
+                _logger.LogError(ex2);
+            }
+        }
+        finally
+        {
+            GC.Collect(GC.MaxGeneration, GCCollectionMode.Aggressive, true, true);
+        }
+    }
+    
+    private async Task ParseSaveEU4()
+    {
+        // wait for save file closing
+        await Task.Delay(200, _ct);
+        Dictionary<string, object> parsedData;
+        await using (var gamestateStream = PathHelper.CreateTempFile())
+        {
+            using (var zipArchive = ZipFile.Open(_meta.GetSaveFilePath().Str, ZipArchiveMode.Read))
+            {
+                var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
+                if (zipEntry is null)
+                    throw new Exception("Invalid save format: no 'gamestate' file found");
+
+                await zipEntry.Open().CopyToAsync(gamestateStream, _ct);
+                gamestateStream.Seek(0, SeekOrigin.Begin);
+            }
+            
+            await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.Parsing);
+            var parser = new SaveParserEU4(gamestateStream, _filter.SearchExpression);
+            parsedData = parser.Parse();
+        }
+
+        _filter.Apply(parsedData);
+
+        await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.SavingResults);
+        var resultFilePath = _meta.GetParsedDataPath();
+        await using (var resultFile = File.OpenWrite(resultFilePath))
+            await JsonSerializer.SerializeAsync(resultFile, parsedData, _jsonOptions, _ct);
+        await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.Done);
+    }
+}

ParadoxSaveParser.WebAPI/Config.cs

@@ -5,9 +5,9 @@ namespace ParadoxSaveParser.WebAPI;
 public class Config
 {
     public const int ActualVersion = 1;
+    public string BaseUrl = "http://127.0.0.1:5226/";
 
     public int Version = ActualVersion;
-    public string BaseUrl = "http://127.0.0.1:5226/";
 
     public static Config FromDtsod(DtsodV23 d)
     {
@@ -23,11 +23,11 @@ public class Config
         return cfg;
     }
 
-    public DtsodV23 ToDtsod() =>
+    public DtsodV23 ToDtsod()
-        new()
+        => new()
         {
             { "version", Version },
-            { "baseUrl", BaseUrl },
+            { "baseUrl", BaseUrl }
         };
 
     public override string ToString() => ToDtsod().ToString();

ParadoxSaveParser.WebAPI/Database/DatabaseConnector.cs

@@ -0,0 +1,106 @@
+using System.Linq;
+using SQLite;
+
+namespace ParadoxSaveParser.WebAPI.Database;
+
+public class DatabaseConnector
+{
+    private readonly int _uploadsLifetimeDays;
+    private readonly SQLiteAsyncConnection _db;
+    private readonly ContextLogger _logger;
+
+    public DatabaseConnector(string databasePath, int uploadsLifetimeDays, ILogger logger)
+    {
+        _uploadsLifetimeDays = uploadsLifetimeDays;
+        _db = new SQLiteAsyncConnection(databasePath);
+        _logger = new ContextLogger("Database", logger);
+    }
+
+    public async Task InitializeAsync()
+    {
+        await _db.CreateTableAsync<SaveFileMetadata>();
+    }
+    
+    public async Task<SaveFileMetadata?> GetMetadata(string id)
+    {
+        return (await _db.QueryAsync<SaveFileMetadata>(
+            "select * from SaveFileMetadata where id = ?", id))
+            .FirstOrDefault();
+    }
+    
+    public async Task<SaveFileMetadata> CreateMetadata(Game game)
+    {
+        var meta = new SaveFileMetadata
+        {
+            id = Guid.NewGuid().ToString(),
+            game = game,
+            status = SaveFileProcessingStatus.Initialized,
+            uploadDateTime = DateTime.Now
+        };
+        await _db.InsertAsync(meta);
+        return meta;
+    }
+
+    public async Task UpdateMetadataStatus(SaveFileMetadata meta, SaveFileProcessingStatus status)
+    {
+        meta.status = status;
+        await _db.UpdateAsync(meta);
+    }
+
+    public async Task SetMetadataError(SaveFileMetadata meta, string errorMessage)
+    {
+        meta.errorMessage = errorMessage;
+        await UpdateMetadataStatus(meta, SaveFileProcessingStatus.Error);
+        TryDeleteAssociatedFiles(meta);
+    }
+
+    public async Task DeleteMetadata(SaveFileMetadata meta, string reason)
+    {
+        _logger.LogDebug($"Deleting save file (id: {meta.id} reason: {reason}" +
+                         $"uploadDate: {meta.uploadDateTime:yyyy/MM/dd})");
+        await _db.DeleteAsync(meta);
+        TryDeleteAssociatedFiles(meta);
+    }
+
+    /// <summary>
+    /// WARNING: Call this method only when no parsing operations are running.
+    /// </summary>
+    public async Task DeleteInvalidData()
+    {
+        _logger.LogInfo("Deleting invalid data...");
+        
+        var expirationDate = DateTime.Now.AddDays(-_uploadsLifetimeDays);
+        var metadataTable = _db.Table<SaveFileMetadata>();
+        var metadataList = await metadataTable.ToListAsync();
+        int deleteCount = 0;
+        foreach (var meta in metadataList)
+        {
+            string deletionReason;
+            if(meta.status != SaveFileProcessingStatus.Done)
+                deletionReason = $"invalid status ({meta.status})";
+            else if (meta.uploadDateTime < expirationDate)
+               deletionReason = "expired";
+            else if(!meta.AssociatedFilesExist())
+                deletionReason = "files not found";
+            else continue;
+            
+            deleteCount++;
+            await DeleteMetadata(meta, deletionReason);
+        }
+
+        _logger.LogInfo($"Deleted {deleteCount} invalid records");
+    }
+
+    private void TryDeleteFile(IOPath file)
+    {
+        if (!File.Exists(file)) return;
+        _logger.LogDebug($"Deleting file '{file}'");
+        File.Delete(file);
+    }
+
+    private void TryDeleteAssociatedFiles(SaveFileMetadata meta)
+    {
+        TryDeleteFile(meta.GetSaveFilePath());
+        TryDeleteFile(meta.GetParsedDataPath());
+    }
+}

ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs

@@ -0,0 +1,45 @@
+using System.Text.Json.Serialization;
+
+namespace ParadoxSaveParser.WebAPI.Database;
+
+public enum SaveFileProcessingStatus
+{
+    Initialized,
+    Uploaded,
+    Parsing,
+    SavingResults,
+    Done,
+    Error,
+}
+
+public class SaveFileMetadata
+{
+    [SQLite.PrimaryKey]
+    [SQLite.NotNull]
+    public string id { get; init; } = null!;
+
+    [JsonConverter(typeof(JsonStringEnumConverter))]
+    [SQLite.NotNull]
+    public Game game { get; init; }
+
+    [JsonConverter(typeof(JsonStringEnumConverter))]
+    [SQLite.NotNull]
+    public SaveFileProcessingStatus status { get; set; }
+    
+    [SQLite.NotNull]
+    public DateTime uploadDateTime { get; set; }
+    
+    // if error occured during parsing, it's message is saved here
+    // status stays the same as when error occured
+    public string? errorMessage { get; set; }
+    
+    
+    public IOPath GetSaveFilePath() => Path.Concat(PathHelper.SAVES_DIR, id + ".eu4");
+
+    public IOPath GetParsedDataPath() => Path.Concat(PathHelper.PARSED_DIR, id + ".parsed.json");
+
+    public bool AssociatedFilesExist()
+    {
+        return File.Exists(GetSaveFilePath()) && File.Exists(GetParsedDataPath());
+    }
+}

ParadoxSaveParser.WebAPI/Game.cs

@@ -0,0 +1,7 @@
+namespace ParadoxSaveParser.WebAPI;
+
+public enum Game
+{
+    Unknown,
+    EU4
+}

ParadoxSaveParser.WebAPI/HttpHelpers/ErrorMessage.cs

@@ -0,0 +1,17 @@
+using System.Net;
+using System.Text.Json.Serialization;
+
+namespace ParadoxSaveParser.WebAPI.HttpHelpers;
+
+public record ErrorMessage
+{
+    public ErrorMessage(HttpStatusCode statusCode, string message)
+    {
+        StatusCode = statusCode;
+        Message = message;
+    }
+
+    [JsonIgnore] public HttpStatusCode StatusCode { get; }
+
+    [JsonPropertyName("errorMessage")] public string Message { get; }
+}

ParadoxSaveParser.WebAPI/HttpHelpers/RequestHelper.cs

@@ -0,0 +1,17 @@
+using System.Linq;
+using System.Net;
+
+namespace ParadoxSaveParser.WebAPI.HttpHelpers;
+
+public class RequestHelper
+{
+    internal static ValueOrError<string> GetQueryValue(HttpListenerContext ctx, string paramName)
+    {
+        string[]? values = ctx.Request.QueryString.GetValues(paramName);
+        string? value = values?.FirstOrDefault();
+        if (string.IsNullOrEmpty(value))
+            return new ErrorMessage(HttpStatusCode.BadRequest,
+                $"No request parameter '{paramName}' provided");
+        return value;
+    }
+}

ParadoxSaveParser.WebAPI/HttpHelpers/ReturnHelper.cs

@@ -0,0 +1,96 @@
+using System.IO;
+using System.Net;
+using System.Text.Encodings.Web;
+using System.Text.Json.Serialization;
+using DTLib.Extensions;
+
+namespace ParadoxSaveParser.WebAPI.HttpHelpers;
+
+public class ReturnHelper
+{
+    private static readonly JsonSerializerOptions _responseJsonSerializerOptions = new()
+    {
+        WriteIndented = false,
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        MaxDepth = 1024,
+        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+    };
+
+    private static async Task ResponseShort(HttpListenerContext ctx,
+        ContextLogger logger,
+        CancellationToken ct,
+        byte[] value,
+        HttpStatusCode statusCode,
+        string contentType)
+    {
+        ctx.Response.StatusCode = (int)statusCode;
+        ctx.Response.ContentType = contentType;
+        logger.LogDebug($"short response (length: {value.Length} type: {contentType})");
+        if (value.Length > 4000)
+        {
+            logger.LogWarn($"response (length: {value.Length} type: {contentType})\n"
+                + $"Content is too big for {nameof(ResponseShort)}."
+                + $"You should send stream instead of byte array.");
+        }
+        await ctx.Response.OutputStream.WriteAsync(value, ct);
+    }
+    
+    public static async Task<HttpStatusCode> ResponseString(
+        HttpListenerContext ctx,
+        ContextLogger logger,
+        CancellationToken ct,
+        string value, 
+        HttpStatusCode statusCode = HttpStatusCode.OK,
+        string contentType = "text/plain")
+    {
+        await ResponseShort(ctx, logger, ct, value.ToBytes(), statusCode, contentType);
+        logger.LogDebug(value);
+        return statusCode;
+    }
+
+    public static async Task<HttpStatusCode> ResponseJson(
+        HttpListenerContext ctx,
+        ContextLogger logger,
+        CancellationToken ct,
+        object value,
+        HttpStatusCode statusCode = HttpStatusCode.OK)
+    {
+        string json = JsonSerializer.Serialize(
+            value, 
+            value.GetType(),
+            _responseJsonSerializerOptions);
+        return await ResponseString(ctx, logger, ct, json, statusCode, "application/json");
+    }
+
+    public static async Task<HttpStatusCode> ResponseError(
+        HttpListenerContext ctx, 
+        ContextLogger logger,
+        CancellationToken ct, 
+        ErrorMessage error)
+    {
+        return await ResponseJson(ctx, logger, ct, error, error.StatusCode);
+    }
+
+    public static async Task<HttpStatusCode> ResponseStream(HttpListenerContext ctx,
+        ContextLogger logger,
+        CancellationToken ct,
+        Stream valueStream,
+        HttpStatusCode statusCode = HttpStatusCode.OK,
+        string contentType = "application/octet-stream")
+    {
+        try
+        {
+            ctx.Response.StatusCode = (int)statusCode;
+            ctx.Response.ContentType = contentType;
+            logger.LogDebug($"stream response (type: {contentType})");
+            await valueStream.CopyToAsync(ctx.Response.OutputStream, ct);
+            return statusCode;
+        }
+        catch (Exception ex)
+        {
+            return await ResponseError(ctx, logger, ct, 
+                new ErrorMessage(HttpStatusCode.InternalServerError, 
+                ex.ToStringDemystified()));
+        }
+    }
+}

ParadoxSaveParser.WebAPI/HttpHelpers/ValueOrError.cs

@@ -0,0 +1,19 @@
+namespace ParadoxSaveParser.WebAPI.HttpHelpers;
+
+public class ValueOrError<T>
+    {
+        public readonly ErrorMessage? Error;
+        public readonly T? Value;
+
+        private ValueOrError(T? value, ErrorMessage? error)
+        {
+            Value = value;
+            Error = error;
+        }
+
+        public bool HasError => Error is not null;
+
+        public static implicit operator ValueOrError<T>(T v) => new(v, null);
+
+        public static implicit operator ValueOrError<T>(ErrorMessage e) => new(default, e);
+    }

ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.csproj

@@ -13,6 +13,16 @@
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="DTLib.Web" Version="1.2.2" />
+        <PackageReference Include="DTLib.Web" Version="1.4.0" />
+<!--        <PackageReference Include="Google.Protobuf" Version="3.31.0" />-->
+        <PackageReference Include="sqlite-net-pcl" Version="1.9.172" />
     </ItemGroup>
+
+<!--    <ItemGroup>-->
+<!--        <Compile Include="obj\Protobuf\*.g.cs" />-->
+<!--    </ItemGroup>-->
+
+<!--    <Target Name="PreBuild" BeforeTargets="PreBuildEvent">-->
+<!--        <Exec Command="sh -c &quot;mkdir -p obj/Protobuf &amp;&amp; protoc Protobuf/*.proto &#45;&#45;csharp_out=obj/Protobuf &#45;&#45;csharp_opt=file_extension=.g.cs&quot;" />-->
+<!--    </Target>-->
 </Project>

ParadoxSaveParser.WebAPI/PathHelper.cs

@@ -1,10 +1,37 @@
-namespace ParadoxSaveParser.WebAPI;
+using System.IO;
+
+namespace ParadoxSaveParser.WebAPI;
 
 public static class PathHelper
 {
-    public static readonly IOPath DATA_DIR = "data";
+    public static readonly IOPath PUBLIC_DIR = "public";
+    public static readonly IOPath DATA_DIR = Path.Concat(PUBLIC_DIR, "data");
     public static readonly IOPath SAVES_DIR = Path.Concat(DATA_DIR, "saves");
-    public static IOPath GetMetaFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".meta.json");
+    public static readonly IOPath PARSED_DIR = Path.Concat(DATA_DIR, "parsed");
-    public static IOPath GetSaveFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".eu4");
+    public static readonly IOPath TEMP_DIR = "temp";
-    public static IOPath GetParsedSaveFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".parsed.json");
+
+    public static void CreateProgramDirectories()
+    {
+        Directory.Create(DATA_DIR);
+        Directory.Create(SAVES_DIR);
+        Directory.Create(PARSED_DIR);
+        Directory.Create(TEMP_DIR);
+    }
+
+    public static void CleanTempDirectory()
+    {
+        Directory.Delete(TEMP_DIR);
+        Directory.Create(TEMP_DIR);
+    }
+
+    public static Stream CreateTempFile()
+    {
+        string fileName = Guid.NewGuid().ToString();
+        IOPath filePath = Path.Concat(TEMP_DIR, fileName);
+        var stream = System.IO.File.Open(filePath.Str, FileMode.CreateNew, FileAccess.ReadWrite,
+            FileShare.Read | FileShare.Delete);
+        // file stays as a ghost until it is closed
+        File.Delete(filePath);
+        return stream;
+    }
 }

ParadoxSaveParser.WebAPI/Program.Responses.cs

@@ -1,125 +0,0 @@
-using System.IO.Compression;
-using System.Linq;
-using System.Net;
-using DTLib.Extensions;
-
-namespace ParadoxSaveParser.WebAPI;
-
-public partial class Program
-{
-    // ReSharper disable once NotAccessedPositionalProperty.Global
-    public record ErrorMessage(string errorMessage);
-
-    private static async Task<HttpStatusCode> ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode,
-        object response)
-    {
-        await JsonSerializer.SerializeAsync(ctx.Response.OutputStream, response, response.GetType(),
-            JsonSerializerOptions.Default, _mainCancel.Token);
-        ctx.Response.StatusCode = (int)statusCode;
-        return statusCode;
-    }
-
-    private static async Task<HttpStatusCode> ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode,
-        string response)
-    {
-        await ctx.Response.OutputStream.WriteAsync(response.ToBytes(), _mainCancel.Token);
-        ctx.Response.StatusCode = (int)statusCode;
-        return statusCode;
-    }
-
-    private static async Task<HttpStatusCode> UploadSaveHandler(HttpListenerContext ctx)
-    {
-        string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault();
-        if (contentType != "application/octet-stream")
-            return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
-                new ErrorMessage($"Invalid request Content-Type: '{contentType}'"));
-
-        string saveId = Guid.NewGuid().ToString();
-        IOPath metaFilePath = PathHelper.GetMetaFilePath(saveId);
-        if (File.Exists(metaFilePath))
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError,
-                new ErrorMessage($"Guid collision! file' {metaFilePath}' already exists."));
-
-        var meta = new SaveFileMetadata
-            { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized, };
-        if (!_saveMetadataStorage.TryAdd(saveId, meta))
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError,
-                new ErrorMessage($"Guid collision! Can't create metadata with id {saveId}"));
-
-        meta.status = SaveFileProcessingStatus.Uploading;
-        IOPath saveFilePath = PathHelper.GetSaveFilePath(meta.id);
-        await using var saveFile = File.OpenWrite(saveFilePath);
-        await using var remoteStream = ctx.Request.InputStream;
-        await remoteStream.CopyToAsync(saveFile, _mainCancel.Token);
-        meta.status = SaveFileProcessingStatus.Uploaded;
-
-        return await ReturnResponse(ctx, HttpStatusCode.OK, saveId);
-    }
-
-    private static (SaveFileMetadata? meta, ErrorMessage? errorMesage) GetMetaFromRequestId(HttpListenerContext ctx,
-        string requestParamName)
-    {
-        var ids = ctx.Request.QueryString.GetValues(requestParamName);
-        string? id = ids?.FirstOrDefault();
-        if (string.IsNullOrEmpty(id))
-            return (null, new ErrorMessage($"No request parameter '{requestParamName}' provided"));
-
-        if (!_saveMetadataStorage.TryGetValue(id, out var meta))
-            return (null, new ErrorMessage($"Save with {id} not found"));
-
-        return (meta, null);
-    }
-
-    private static async Task<HttpStatusCode> GetSaveStatusHandler(HttpListenerContext ctx)
-    {
-        var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id");
-        if (errorMessage is not null)
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage);
-
-        return await ReturnResponse(ctx, HttpStatusCode.OK, meta!);
-    }
-
-    private static async Task<HttpStatusCode> ParseSaveEU4Handler(HttpListenerContext ctx)
-    {
-        var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id");
-        if (errorMessage is not null)
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage);
-
-        //TODO: get actual query
-        string searchQuery = "";
-        
-        try
-        {
-            using var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(meta!.id).Str, ZipArchiveMode.Read);
-            var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
-            if (zipEntry is null)
-                return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
-                    new ErrorMessage("Invalid save format: no 'gamestate' file found"));
-
-            string extractedGamestatePath = PathHelper.GetSaveFilePath(meta.id) + ".gamestate";
-            zipEntry.ExtractToFile(extractedGamestatePath, true);
-            var gamestateStream = File.OpenRead(extractedGamestatePath);
-
-            meta.status = SaveFileProcessingStatus.Parsing;
-            var se = SearchExpression.Parse(searchQuery);
-            var parser = new SaveParserEU4(gamestateStream, se);
-            var result = parser.Parse();
-
-            meta.status = SaveFileProcessingStatus.SavingResults;
-            IOPath resultFilePath = PathHelper.GetParsedSaveFilePath(meta.id);
-            await using var resultFile = File.OpenWrite(resultFilePath);
-            await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions, _mainCancel.Token);
-            meta.status = SaveFileProcessingStatus.Done;
-            meta.SaveToFile();
-        }
-        catch (Exception ex)
-        {
-            string errorMesage = ex.ToStringDemystified();
-            _loggerRoot.LogWarn(nameof(ParseSaveEU4Handler), errorMesage);
-            return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
-                new ErrorMessage(errorMesage));
-        }
-
-        return await ReturnResponse(ctx, HttpStatusCode.OK, meta);
-    }
-}

ParadoxSaveParser.WebAPI/Program.cs

@@ -11,157 +11,110 @@ global using ParadoxSaveParser.Lib;
 global using Directory = DTLib.Filesystem.Directory;
 global using File = DTLib.Filesystem.File;
 global using Path = DTLib.Filesystem.Path;
-using System.Collections.Concurrent;
+using DTLib.Console;
-using System.IO;
-using System.Text.Encodings.Web;
 using DTLib.Dtsod;
-using DTLib.Extensions;
 using DTLib.Web;
 using DTLib.Web.Routes;
+using ParadoxSaveParser.WebAPI.BackgroundTasks;
+using ParadoxSaveParser.WebAPI.Database;
+using ParadoxSaveParser.WebAPI.Routes;
+using ParadoxSaveParser.WebAPI.SaveDataFilters;
+// ReSharper disable MethodHasAsyncOverload
 
 namespace ParadoxSaveParser.WebAPI;
 
-public partial class Program
+public static class Program
 {
-    private static readonly IOPath _configPath = "./config.dtsod";
+    internal static bool IsDebug = true;
-    private static Config _config = new();
+    internal static DatabaseConnector DB = null!;
 
-    private static readonly ILogger _loggerRoot = new CompositeLogger(
-        new ConsoleLogger(),
-        new FileLogger("logs", "ParadoxSaveParser.WebAPI"));
 
-    private static readonly CancellationTokenSource _mainCancel = new();
+    public static async Task Main(string[] args)
-    private static ConcurrentDictionary<string, SaveFileMetadata> _saveMetadataStorage = new();
-    
-    private static JsonSerializerOptions _saveSerializerOptions = new()
     {
-        WriteIndented = false,
-        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
-        MaxDepth = 1024,
-    };
-
-    static void TestSearchExpression(Stream saveStream, TestCase tc)
-    {
-        saveStream.Seek(0, SeekOrigin.Begin);
-        var se = SearchExpression.Parse(tc.q);
-        var parser = new SaveParserEU4(saveStream, se);
-        var rootNode = parser.Parse();
-        string json = JsonSerializer.Serialize(rootNode, _saveSerializerOptions);
-        string pdx = json.Substring(1, json.Length - 2)
-            .Replace(",", " ").Replace("{", "{ ").Replace("}", " }")
-            .Replace("\"", "").Replace("[", "").Replace("]", "").Replace(":", "=");
-        if(pdx == tc.a)
-        {
-            Console.WriteLine($"[OK] q:'{tc.q}' a:'{tc.a}'");
-        }
-        else
-        {
-            Console.WriteLine($"[Error] q:'{tc.q}' a:'{tc.a}' r:'{pdx}'");
-        }
-    }
-
-    record TestCase(string q, string a);
-    
-    public static void Main(string[] args)
-    {
-        
-        using var saveStream = new MemoryStream(
-            "EU4txt a={ b={ c=0 d=1 e=2 } f=3 }".ToBytes(), 
-            false);
-
-        TestCase[] testCases = [
-            new("a", 
-                "a={ b={ c=0 d=1 e=2 } f=3 }"),
-            
-            new("a.*", 
-                "a={ b={ c=0 d=1 e=2 } f=3 }"),
-            
-            new("a.b", 
-                "a={ b={ c=0 d=1 e=2 } }"),
-            
-            new("a.[0].c",
-                "a={ b={ c=0 } }"),
-            
-            new("a.[1]",
-                "a={ f=3 }"),
-            
-            new("a.b.(c|d)", 
-                "a={ b={ c=0 d=1 } }"),
-            
-            new("a.(b.e|f)",
-                "a={ b={ e=2 } f=3 }"),
-        ];
-
-        foreach (var test in testCases)
-        {
-            TestSearchExpression(saveStream, test);
-        }
-
-/*
         Console.InputEncoding = Encoding.UTF8;
         Console.OutputEncoding = Encoding.UTF8;
         Console.CursorVisible = false;
-        ContextLogger logger = new ContextLogger(nameof(Main), _loggerRoot);
+        
+        #if DEBUG
+        IsDebug = true;
+        #endif
+        
+        new LaunchArgumentParser(
+                new LaunchArgument(["-d", "--debug"], 
+                    "enables debug log output to console",
+                    () => IsDebug = true)
+            ).AllowNoArguments().ParseAndHandle(args);
+        
+        var loggerRoot = new CompositeLogger(
+            new ConsoleLogger
+            {
+                DebugLogEnabled = IsDebug
+            },
+            new FileLogger("logs", "ParadoxSaveParser.WebAPI")
+            {
+                DebugLogEnabled = true
+            });
+        var loggerMain = new ContextLogger(nameof(Main), loggerRoot);
+        loggerMain.LogDebug("Debug log is enabled");
+
+        CancellationTokenSource mainCancel = new();
         Console.CancelKeyPress += (_, e) =>
         {
             e.Cancel = true;
-            logger.LogInfo("Ctrl+C Pressed");
+            loggerMain.LogInfo("Ctrl+C Pressed");
-            _mainCancel.Cancel();
+            mainCancel.Cancel();
         };
 
         try
         {
             // config
-            if (!File.Exists(_configPath))
+            IOPath configPath = "./config.dtsod";
+            Config config;
+            if (File.Exists(configPath))
             {
-                logger.LogWarn("config file not found.");
+                config = Config.FromDtsod(new DtsodV23(File.ReadAllText(configPath)));
-                File.WriteAllText(_configPath, _config.ToString());
+            }
-                logger.LogWarn($"created default at {_configPath}.");
+            else
+            {
+                loggerMain.LogWarn("config file not found.");
+                config = new();
+                File.WriteAllText(configPath, config.ToString());
+                loggerMain.LogWarn($"created default at {configPath}.");
             }
-            else _config = Config.FromDtsod(new DtsodV23(File.ReadAllText(_configPath)));
             
-            PrepareLocalFiles();
+            PathHelper.CreateProgramDirectories();
+            PathHelper.CleanTempDirectory();
+            DB = new("database.sqlite", 30, loggerRoot);
+            await DB.InitializeAsync();
+            await DB.DeleteInvalidData();
+
+            var bgJobManager = new BackgroundJobManager(loggerRoot);
+            var saveFilters = new Dictionary<Game, ISaveDataFilter>
+            {
+                { Game.EU4, new SaveDataFilterEU4() },
+            };
             
             // http server
-            var router = new SimpleRouter(_loggerRoot);
+            var router = new SimpleRouter(loggerRoot);
-            router.DefaultRoute = new ServeFilesRouteHandler("public");
+            router.DefaultRoute = new SimpleRouter.RouteWithMethod(HttpMethod.GET, 
-            router.MapRoute("/getSaveStatus", HttpMethod.GET, GetSaveStatusHandler);
+                new ServeFilesRouteHandler(PathHelper.PUBLIC_DIR));
-            router.MapRoute("/uploadSave/eu4", HttpMethod.POST, UploadSaveHandler);
+            router.MapRoute("/uploadSave", HttpMethod.POST,
-            router.MapRoute("/parseSave/eu4", HttpMethod.POST, ParseSaveEU4Handler);
+                new UploadSaveHandler(mainCancel.Token, bgJobManager, saveFilters));
+            router.MapRoute("/getSaveStatus", HttpMethod.GET,
+                new GetSaveStatusHandler(mainCancel.Token));
+            router.MapRoute("/getSaveData", HttpMethod.GET,
+                new GetSaveDataHandler(mainCancel.Token));
 
-            var app = new WebApp(_config.BaseUrl, _loggerRoot, router, _mainCancel.Token);
+            var app = new WebApp(config.BaseUrl, loggerRoot, router, mainCancel.Token);
-            app.Run().GetAwaiter().GetResult();
+            await app.Run();
         }
         catch (OperationCanceledException ex)
         {
-            logger.LogWarn($"catched OperationCanceledException from {ex.Source}");
+            loggerMain.LogWarn($"catched OperationCanceledException from {ex.Source}");
         }
         catch (Exception ex)
         {
-            logger.LogError(ex.ToStringDemystified());
+            loggerMain.LogError(ex.ToStringDemystified());
-        }
-*/
-    }
-
-    public static void PrepareLocalFiles()
-    {
-        Directory.Create(PathHelper.DATA_DIR);
-        Directory.Create(PathHelper.SAVES_DIR);
-        foreach (var metaFilePath in System.IO.Directory.GetFiles(
-                     PathHelper.SAVES_DIR.Str, "*.meta.json",
-                     SearchOption.TopDirectoryOnly))
-        {
-            using var metaFile = File.OpenRead(metaFilePath);
-            var meta = JsonSerializer.Deserialize<SaveFileMetadata>(metaFile) ?? 
-                       throw new NullReferenceException(metaFilePath);
-            if (meta.status != SaveFileProcessingStatus.Done)
-            {
-                _loggerRoot.LogWarn(nameof(PrepareLocalFiles), $"metadata file '{metaFilePath}' status has invalid status {meta.status}");
-            }
-
-            if(!_saveMetadataStorage.TryAdd(meta.id, meta))
-                throw new Exception("Guid collision!");
         }
     }
-    
 }

ParadoxSaveParser.WebAPI/Protobuf/ParsedData.proto

@@ -0,0 +1,21 @@
+syntax = "proto3";
+option csharp_namespace = "ParadoxSaveParser.WebAPI.MyProtobuf";
+
+message Item {
+  oneof value {
+    bool b = 1;
+    int64 i64 = 2;
+    double f64 = 3;
+    string str = 4;
+    ItemList list = 5;
+    ItemListMap map = 6;
+  }
+}
+
+message ItemList {
+  repeated Item items = 1;
+}
+
+message ItemListMap {
+  map<string, ItemList> items_map = 1;
+}

ParadoxSaveParser.WebAPI/README.md

@@ -0,0 +1,44 @@
+# WebAPI
+Simple web application created using DTLib.Web.
+
+
+## Important
+Restart web application once per day to delete outdated data and clean RAM.
+
+## Routes
+### POST `/uploadSave`
+- **Query Params:**
+  - `game` - short name of the game (see [../README.md](../README.md))
+- **Request Body:** `application/octet-stream` - .eu4 file
+- **Response:**
+    ```json
+    { "saveId": "string" }
+    ```
+    or `{ "errorMessage": "string" }`
+
+### GET `/getSaveStatus`
+- **Query Params:**
+    - `id` - id of uploaded save file
+- **Response:** [SaveFileMetadata](./SaveFileMetadata.cs)
+    ```json
+    {
+        "id": "string",
+        "game": "string",
+        "status": "string",
+        "errorMessage": "string?"
+    }
+    ```
+
+### GET `/getSaveData`
+- **Query Params:**
+    - `id` - id of uploaded save file
+- **Response:**
+    ```json5
+    {
+        "key0": [ "objects" ],
+        "key1": [ "objects" ],
+        //...
+    }
+    ```
+    or `{ "errorMessage": "string" }`
+

ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs

@@ -0,0 +1,44 @@
+using System.Net;
+using ParadoxSaveParser.WebAPI.Database;
+using ParadoxSaveParser.WebAPI.HttpHelpers;
+
+namespace ParadoxSaveParser.WebAPI.Routes;
+
+internal class GetSaveDataHandler : RouteHandlerBase
+{
+    public GetSaveDataHandler(CancellationToken cancelAllToken) 
+        : base(cancelAllToken)
+    {
+    }
+
+    public override async Task<HttpStatusCode> HandleRequest(
+        HttpListenerContext ctx, ContextLogger requestLogger)
+    {
+        var idOrError = RequestHelper.GetQueryValue(ctx, "id");
+        if (idOrError.HasError)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, idOrError.Error!);
+        string id = idOrError.Value!;
+
+        var meta = await Program.DB.GetMetadata(id);
+        if(meta is null)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, 
+                new ErrorMessage(HttpStatusCode.NotFound,
+                    $"Save with id {id} not found"));
+        if(meta.status != SaveFileProcessingStatus.Done)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, 
+                new ErrorMessage(HttpStatusCode.BadRequest,
+                    $"Save with id {id} has status {meta.status}"));
+        
+        IOPath dataFilePath = meta.GetParsedDataPath();
+        if (!File.Exists(dataFilePath))
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                new ErrorMessage(HttpStatusCode.InternalServerError, 
+                    $"Save with id {id} not found")
+            );
+
+        await using var dataFile = File.OpenRead(dataFilePath);
+        
+        return await ReturnHelper.ResponseStream(ctx, requestLogger, _cancelAllToken,
+            dataFile, contentType: "application/json");
+    }
+}

ParadoxSaveParser.WebAPI/Routes/GetSaveStatusHandler.cs

@@ -0,0 +1,30 @@
+using System.Net;
+using ParadoxSaveParser.WebAPI.HttpHelpers;
+
+namespace ParadoxSaveParser.WebAPI.Routes;
+
+internal class GetSaveStatusHandler : RouteHandlerBase
+{
+    public GetSaveStatusHandler(CancellationToken cancelAllToken) 
+        : base(cancelAllToken)
+    {
+    }
+
+    public override async Task<HttpStatusCode> HandleRequest(
+        HttpListenerContext ctx, ContextLogger requestLogger)
+    {
+        var idOrError = RequestHelper.GetQueryValue(ctx, "id");
+        if (idOrError.HasError)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, idOrError.Error!);
+        string id = idOrError.Value!;
+
+        var meta = await Program.DB.GetMetadata(id);
+        if (meta is null)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                new ErrorMessage(HttpStatusCode.InternalServerError, 
+                    $"Save with id {id} not found")
+                );
+
+        return await ReturnHelper.ResponseJson(ctx, requestLogger, _cancelAllToken, meta);
+    }
+}

ParadoxSaveParser.WebAPI/Routes/RouteHandlerBase.cs

@@ -0,0 +1,16 @@
+using System.Net;
+using DTLib.Web.Routes;
+
+namespace ParadoxSaveParser.WebAPI.Routes;
+
+public abstract class RouteHandlerBase : IRouteHandler
+{
+    protected readonly CancellationToken _cancelAllToken;
+    
+    protected RouteHandlerBase(CancellationToken cancelAllToken)
+    {
+        _cancelAllToken = cancelAllToken;
+    }
+
+    public abstract Task<HttpStatusCode> HandleRequest(HttpListenerContext ctx, ContextLogger requestLogger);
+}

ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs

@@ -0,0 +1,56 @@
+using System.Linq;
+using System.Net;
+using ParadoxSaveParser.WebAPI.BackgroundTasks;
+using ParadoxSaveParser.WebAPI.Database;
+using ParadoxSaveParser.WebAPI.HttpHelpers;
+using ParadoxSaveParser.WebAPI.SaveDataFilters;
+
+namespace ParadoxSaveParser.WebAPI.Routes;
+
+public class UploadSaveHandler : RouteHandlerBase
+{
+    private readonly BackgroundJobManager _bgJobManager;
+    private readonly Dictionary<Game, ISaveDataFilter> _saveFilters;
+
+    public UploadSaveHandler(
+        CancellationToken cancelAllToken, 
+        BackgroundJobManager bgJobManager,
+        Dictionary<Game, ISaveDataFilter> saveFilters) 
+        : base(cancelAllToken)
+    {
+        _bgJobManager = bgJobManager;
+        _saveFilters = saveFilters;
+    }
+
+    public override async Task<HttpStatusCode> HandleRequest(
+        HttpListenerContext ctx, ContextLogger requestLogger)
+    {
+        string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault();
+        if (contentType != "application/octet-stream")
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                    new ErrorMessage(HttpStatusCode.BadRequest,
+                $"Invalid request Content-Type: '{contentType}'"));
+
+        var gameStrOrError = RequestHelper.GetQueryValue(ctx, "game");
+        if (gameStrOrError.HasError)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, gameStrOrError.Error!);
+
+        if (!Enum.TryParse(gameStrOrError.Value, ignoreCase: true, out Game game))
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                new ErrorMessage(HttpStatusCode.BadRequest,
+                    $"Invalid requested game: '{gameStrOrError.Value}'"));  
+        var meta = await Program.DB.CreateMetadata(game);
+
+        var saveFilePath = meta.GetSaveFilePath();
+        await using (var saveFile = File.OpenWrite(saveFilePath))
+        {
+            await using (var remoteStream = ctx.Request.InputStream)
+                await remoteStream.CopyToAsync(saveFile, _cancelAllToken);
+        }
+        await Program.DB.UpdateMetadataStatus(meta, SaveFileProcessingStatus.Uploaded);
+
+        _bgJobManager.StartNewParsingOperation(meta, _saveFilters[meta.game], _cancelAllToken);
+        dynamic responseData = new { meta.id };
+        return await ReturnHelper.ResponseJson(ctx, requestLogger, _cancelAllToken, responseData);
+    }
+}

ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs

@@ -0,0 +1,9 @@
+namespace ParadoxSaveParser.WebAPI.SaveDataFilters;
+
+public interface ISaveDataFilter
+{
+    public string SearchString { get; }
+    public ISearchExpression SearchExpression { get; }
+    
+    public void Apply(Dictionary<string, object> data);
+}

ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs

@@ -0,0 +1,48 @@
+using System.Linq;
+using ParsedDict = System.Collections.Generic.Dictionary<string, object>;
+
+namespace ParadoxSaveParser.WebAPI.SaveDataFilters;
+
+public class SaveDataFilterEU4 : ISaveDataFilter
+{
+    public string SearchString { get; }
+    public ISearchExpression SearchExpression { get; }
+    
+    public SaveDataFilterEU4()
+    {
+        SearchString =
+            """
+            (
+                active_war|
+                previous_war|
+                income_statistics|
+                nation_size_statistics|
+                inflation_statistics|
+                countries.(
+                    ---.~|REB.~|PIR.~|NAT.~|
+                    *.(
+                        flags.~|hidden_flags.~|variables.~|estate.~|active_agenda.~|
+                        power_projection.~|ai.~|history.~|navy.~|army.~|mercenary_company.~|
+                        active_relations.~|border_pct.~|border_sit.~|border_provinces.~|
+                        neighbours.~|home_neighbours.~|core_neighbours.~|inflation_history.~|
+                        opinion_cache.~|owned_provinces.~|controlled_provinces.~|core_provinces.~|
+                        claim_provinces.~|leader.~|*
+                    )
+                )
+            )
+            """
+            .Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", "");
+        SearchExpression = SearchExpressionCompiler.Compile(SearchString);
+    }
+
+    
+    public void Apply(ParsedDict data)
+    {
+        var countries = (ParsedDict)data["countries"];
+        var countries_filtered = countries.Where(pair 
+            => ((ParsedDict)pair.Value).TryGetValue("raw_development", out var raw_development)
+               && (double)raw_development > 0
+            ).ToDictionary();
+        data["countries"] = countries_filtered;
+    }
+}

ParadoxSaveParser.WebAPI/SaveDataFilters/data_filtering_EU4.txt

@@ -0,0 +1,43 @@
+countries
+	filter:
+		always: exists (has raw_development && raw_development != 0)
+		optional: is player (was_player == yes)
+	exclude:
+		flags
+		hidden_flags
+		variables
+		estate
+		active_agenda
+		power_projection
+		ai
+		history
+		navy
+		army
+		mercenary_company
+		active_relations
+		border_pct
+		border_sit
+		border_provinces
+		neighbours
+		home_neighbours
+		core_neighbours
+		inflation_history
+		opinion_cache
+		owned_provinces
+		controlled_provinces
+		core_provinces
+		claim_provinces
+		leader
+	query:
+		countries.(---.~|REB.~|PIR.~|NAT.~|*.(flags.~|hidden_flags.~|variables.~|estate.~|active_agenda.~|power_projection.~|ai.~|history.~|navy.~|army.~|mercenary_company.~|active_relations.~|border_pct.~|border_sit.~|border_provinces.~|neighbours.~|home_neighbours.~|core_neighbours.~|inflation_history.~|opinion_cache.~|owned_provinces.~|controlled_provinces.~|core_provinces.~|claim_provinces.~|leader.~|*))
+
+active_war
+	filter:
+		optional: only player wars
+previous_war
+	filter:
+		optional: only player wars
+		always: not fictive (has losses and lasts long)
+income_statistics
+nation_size_statistics
+inflation_statistics

ParadoxSaveParser.WebAPI/SaveFileMetadata.cs

@@ -1,32 +0,0 @@
-using System.Text.Json.Serialization;
-
-namespace ParadoxSaveParser.WebAPI;
-
-public enum SaveFileProcessingStatus
-{
-    Initialized, Uploading, Uploaded, Parsing, SavingResults, Done
-}
-
-public enum Game
-{
-    Unknown, EU4
-}
-
-public class SaveFileMetadata
-{
-    public required string id { get; init; }
-    
-    [JsonConverter(typeof(JsonStringEnumConverter))]
-    public required Game game { get; init; }
-    
-    [JsonConverter(typeof(JsonStringEnumConverter))]
-    public required SaveFileProcessingStatus status { get; set; }
-    
-    
-    private static readonly JsonSerializerOptions _jsonOptions = new() { WriteIndented = true };
-    public void SaveToFile()
-    {
-        using var metaFile = File.OpenWrite(PathHelper.GetMetaFilePath(id));
-        JsonSerializer.Serialize(metaFile, this, _jsonOptions);
-    }
-}

ParadoxSaveParser.sln

@@ -7,9 +7,14 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SolutionFolder", "SolutionFolder", "{F1D312F1-0620-4E35-8D78-9A2808CDE12C}"
 	ProjectSection(SolutionItems) = preProject
 		.gitignore = .gitignore
-		TODO.txt = TODO.txt
+		TODO.md = TODO.md
+		README.md = README.md
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParadoxSaveParser.Lib.Tests", "ParadoxSaveParser.Lib.Tests\ParadoxSaveParser.Lib.Tests.csproj", "{23F4BE1B-3043-4821-9F65-74FF5F57FA59}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParadoxSaveParser.CLI", "ParadoxSaveParser.CLI\ParadoxSaveParser.CLI.csproj", "{2D4448A6-390D-47F3-9BB7-6266669719DE}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -24,5 +29,13 @@ Global
 		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{53ED0135-9513-4DE2-9187-CF2899F179B3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{23F4BE1B-3043-4821-9F65-74FF5F57FA59}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2D4448A6-390D-47F3-9BB7-6266669719DE}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

README.md

@@ -0,0 +1,18 @@
+# Paradox Save Parser
+Yet another save files parser. 
+
+## Supported games:
+- EU4
+
+## Project structure
+- **[ParadoxSaveParser.Lib](./ParadoxSaveParser.Lib)** - 
+    Parser itself
+- **[ParadoxSaveParser.Lib.Tests](./ParadoxSaveParser.Lib.Tests)** - 
+    Tests for parser
+- **[ParadoxSaveParser.CLI](./ParadoxSaveParser.CLI)** - 
+    Command line tool to parse save files. Can be used in interactive mode.
+- **[ParadoxSaveParser.WebAPI](./ParadoxSaveParser.WebAPI)** - 
+    Backend for my save file analytics website (TODO: add repo link).
+
+## Building
+1. Install protobuf compiler https://protobuf.dev/installation/

TODO.md

@@ -0,0 +1,4 @@
+## WebAPI.SaveParsingOperation:
+	Save parsed data in protobuf
+	Re-parse if saved data was parsed with another query
+    Implement automatic database cleanup

TODO.txt

@@ -1,9 +0,0 @@
-Main:
-	Add temporary files deletion
-
-ParseSaveHandler:
-	Make this method run as background task instead of POST query
-	Add debug log
-	
-Parser:
-	Add query support to parse only needed information