ParadoxSaveParser.CLI/Modes/Search.cs

@@ -38,7 +38,7 @@ internal static partial class Modes
             var parser = new SaveParserEU4(inputStream, searchExpression);
             var parsedValue = parser.Parse();
             JsonSerializer.Serialize(outputStream, parsedValue,
-                ParsedValueJsonContext.Default.DictionaryStringObject);
+                ParsedValueJsonContext.Default.DictionaryStringListObject);
             outputStream.WriteByte((byte)'\n');
         }
         finally

ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj

@@ -8,6 +8,6 @@
     </PropertyGroup>
     
     <ItemGroup>
-      <PackageReference Include="Microsoft.Extensions.ObjectPool" Version="9.0.4" />
+      <PackageReference Include="Microsoft.Extensions.ObjectPool" Version="9.0.3" />
     </ItemGroup>
 </Project>

ParadoxSaveParser.Lib/ParsedValueJsonContext.cs

@@ -3,12 +3,11 @@
 namespace ParadoxSaveParser.Lib;
 
 [JsonSourceGenerationOptions(MaxDepth = 1024, WriteIndented = true)]
-[JsonSerializable(typeof(Dictionary<string, object>))]
+[JsonSerializable(typeof(Dictionary<string, List<object>>))]
 [JsonSerializable(typeof(List<object>))]
 [JsonSerializable(typeof(string))]
 [JsonSerializable(typeof(long))]
 [JsonSerializable(typeof(double))]
-[JsonSerializable(typeof(bool))]
 public partial class ParsedValueJsonContext : JsonSerializerContext
 {
 }

ParadoxSaveParser.Lib/SaveParserEU4.cs

@@ -154,17 +154,12 @@ public class SaveParserEU4
         switch (tok.type)
         {
             case TokenType.StringOrNumber:
-                try
-                {
                 // string values can be empty
                 if(tok.value!.Length == 0)
                     return string.Empty;
-                    if (tok.value.Equals("yes"))
-                        return true;
-                    if (tok.value.Equals("no"))
-                        return false;
                 
                 string tokStr = tok.value.ToString();
+                _stringBuilderPool.Return(tok.value);
                 if (tokStr[0] != '-' && !char.IsDigit(tokStr[0]))
                     return tokStr;
                 if (tokStr.Contains('.') && double.TryParse(tokStr, out double d))
@@ -172,11 +167,6 @@ public class SaveParserEU4
                 if (long.TryParse(tokStr, out long l))
                     return l;
                 return tokStr;
-                }
-                finally
-                {
-                    _stringBuilderPool.Return(tok.value!);
-                }
             case TokenType.BracketOpen:
                 object obj = ParseListOrDict();
                 return obj;
@@ -226,7 +216,7 @@ public class SaveParserEU4
     }
 
     private static bool IsEmptyCollection(object value)
-        => value is Dictionary<string, object> { Count: 0 } or List<object> { Count: 0 };
+        => value is Dictionary<string, List<object>> { Count: 0 } or List<object> { Count: 0 };
 
     // doesn't move next
     private object ParseListOrDict()
@@ -274,9 +264,9 @@ public class SaveParserEU4
     }
 
     // moves next
-    private Dictionary<string, object> ParseDict()
+    private Dictionary<string, List<object>> ParseDict()
     {
-        Dictionary<string, object> dict = new();
+        Dictionary<string, List<object>> dict = new();
 
         // root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count
         for (int localIndex = 0; _tokens.MoveNext(); localIndex++)
@@ -337,32 +327,23 @@ public class SaveParserEU4
 
             string keyStr = keySB.ToString();
             _stringBuilderPool.Return(keySB);
-            
+            if (!dict.TryGetValue(keyStr, out var list))
-            // Paradox save format has another way of defining list:
-            // a = 1
-            // a = 2
-            // It means `a = { 1 2 }`
-            if (dict.TryGetValue(keyStr, out var firstValue))
             {
-                // Do dot add empty collections into list.
+                list = new List<object>();
-                // `key:{}` is okay, but i don't want to see `key:[{},{},{},{},{},{}]`
+                dict.Add(keyStr, list);
+            }
+
+            
+            // do dot add empty collections into list
             if (IsEmptyCollection(value))
                 continue;
-                
+            list.Add(value);
-                if (firstValue is List<object> existingList)
-                    existingList.Add(value);
-                else dict[keyStr] = new List<object> { firstValue, value };
-            }
-            else
-            {
-                dict.Add(keyStr, value);
-            }
         }
 
         return dict;
     }
 
-    public Dictionary<string, object> Parse()
+    public Dictionary<string, List<object>> Parse()
     {
         var root = ParseDict();
         return root;

ParadoxSaveParser.Lib/SearchExpression.cs

@@ -1,6 +1,6 @@
 namespace ParadoxSaveParser.Lib;
 
-public readonly record struct SearchArgs
+public record SearchArgs
 {
     public readonly string KeyStr;
     public readonly StringBuilder? KeySB;

ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs

@@ -1,7 +1,4 @@
-using ParadoxSaveParser.WebAPI.Database;
+namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
-using ParadoxSaveParser.WebAPI.SaveDataFilters;
-
-namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
 
 public class BackgroundJobManager
 {
@@ -14,11 +11,11 @@ public class BackgroundJobManager
     }
     
     public SaveParsingOperation StartNewParsingOperation(
-        SaveFileMetadata meta, ISaveDataFilter filter, CancellationToken ct)
+        SaveFileMetadata meta, ISearchExpression searchQuery, CancellationToken ct)
     {
         long nextId = Interlocked.Increment(ref _lastJobId);
         var contextLogger = new ContextLogger($"BackgroundJob-{nextId}", _parentLogger);
-        var op = new SaveParsingOperation(nextId, meta, filter, contextLogger, ct);
+        var op = new SaveParsingOperation(nextId, meta, searchQuery, contextLogger, ct);
         op.StartAsync();
         return op;
     }

ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs

@@ -1,23 +1,20 @@
-using System.IO;
+using System.IO.Compression;
-using System.IO.Compression;
 using System.Linq;
 using System.Text.Encodings.Web;
 using System.Text.Json.Serialization;
-using ParadoxSaveParser.WebAPI.Database;
-using ParadoxSaveParser.WebAPI.SaveDataFilters;
 
 namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
 
 public class SaveParsingOperation
 {
     public readonly long OperationId;
+    public readonly SaveFileMetadata Meta;
+    public readonly ISearchExpression SearchQuery;
     
-    private readonly SaveFileMetadata _meta;
-    private readonly ISaveDataFilter _filter;
     private readonly ContextLogger _logger;
-    private readonly CancellationToken _ct;
+    private readonly CancellationToken _cancelToken;
 
-    private static readonly JsonSerializerOptions _jsonOptions = new()
+    private static readonly JsonSerializerOptions _saveSerializerOptions = new()
     {
         WriteIndented = false,
         Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
@@ -25,43 +22,36 @@ public class SaveParsingOperation
         DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
     };
 
-    public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISaveDataFilter filter,
+    public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISearchExpression searchQuery,
-        ContextLogger logger, CancellationToken ct)
+        ContextLogger logger, CancellationToken cancelToken)
     {
         OperationId = operationId;
-        _meta = meta;
+        Meta = meta;
-        _filter = filter;
+        SearchQuery = searchQuery;
         _logger = logger;
-        _ct = ct;
+        _cancelToken = cancelToken;
     }
 
     public async void StartAsync()
     {
         try
         {
-            _logger.LogInfo($"Starting background parsing operation of {_meta.game} save {_meta.id}");
+            _logger.LogInfo($"Starting background parsing operation of {Meta.game} save {Meta.id}");
-            switch (_meta.game)
+            switch (Meta.game)
             {
                 case Game.EU4:
                     await ParseSaveEU4();
                     break;
                 default:
-                    throw new ArgumentOutOfRangeException(_meta.game.ToString());
+                    throw new ArgumentOutOfRangeException(Meta.game.ToString());
             }
-            _logger.LogInfo($"Finished parsing operation of {_meta.game} save {_meta.id}");
+            _logger.LogInfo($"Finished parsing operation of {Meta.game} save {Meta.id}");
         }
         catch (Exception ex)
         {
             string errorMesage = ex.ToStringDemystified();
             _logger.LogError(errorMesage);
-            try
+            Meta.errorMessage = errorMesage;
-            {
-                await Program.DB.SetMetadataError(_meta, errorMesage);
-            }
-            catch (Exception ex2)
-            {
-                _logger.LogError(ex2);
-            }
         }
         finally
         {
@@ -72,31 +62,29 @@ public class SaveParsingOperation
     private async Task ParseSaveEU4()
     {
         // wait for save file closing
-        await Task.Delay(200, _ct);
+        await Task.Delay(200, _cancelToken);
-        Dictionary<string, object> parsedData;
+        string extractedGamestatePath = PathHelper.GetSaveFilePath(Meta.id) + ".gamestate";
-        await using (var gamestateStream = PathHelper.CreateTempFile())
+        using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(Meta.id).Str, ZipArchiveMode.Read))
-        {
-            using (var zipArchive = ZipFile.Open(_meta.GetSaveFilePath().Str, ZipArchiveMode.Read))
         {
             var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
             if (zipEntry is null)
                 throw new Exception("Invalid save format: no 'gamestate' file found");
 
-                await zipEntry.Open().CopyToAsync(gamestateStream, _ct);
+            zipEntry.ExtractToFile(extractedGamestatePath, true);
-                gamestateStream.Seek(0, SeekOrigin.Begin);
         }
 
-            await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.Parsing);
+        var gamestateStream = File.OpenRead(extractedGamestatePath);
-            var parser = new SaveParserEU4(gamestateStream, _filter.SearchExpression);
-            parsedData = parser.Parse();
-        }
 
-        _filter.Apply(parsedData);
+        Meta.status = SaveFileProcessingStatus.Parsing;
+        var parser = new SaveParserEU4(gamestateStream, SearchQuery);
+        var result = parser.Parse();
 
-        await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.SavingResults);
+        Meta.status = SaveFileProcessingStatus.SavingResults;
-        var resultFilePath = _meta.GetParsedDataPath();
+        var resultFilePath = PathHelper.GetParsedSaveFilePath(Meta.id);
-        await using (var resultFile = File.OpenWrite(resultFilePath))
+        await using var resultFile = File.OpenWrite(resultFilePath);
-            await JsonSerializer.SerializeAsync(resultFile, parsedData, _jsonOptions, _ct);
+        await JsonSerializer.SerializeAsync(resultFile, result, 
-        await Program.DB.UpdateMetadataStatus(_meta, SaveFileProcessingStatus.Done);
+            _saveSerializerOptions, _cancelToken);
+        Meta.status = SaveFileProcessingStatus.Done;
+        Meta.SaveToFile();
     }
 }

ParadoxSaveParser.WebAPI/Database/DatabaseConnector.cs

@@ -1,105 +0,0 @@
-using System.Linq;
-using SQLite;
-
-namespace ParadoxSaveParser.WebAPI.Database;
-
-public class DatabaseConnector
-{
-    private readonly int _uploadsLifetimeDays;
-    private readonly SQLiteAsyncConnection _db;
-    private readonly ContextLogger _logger;
-
-    public DatabaseConnector(string databasePath, int uploadsLifetimeDays, ILogger logger)
-    {
-        _uploadsLifetimeDays = uploadsLifetimeDays;
-        _db = new SQLiteAsyncConnection(databasePath);
-        _logger = new ContextLogger("Database", logger);
-    }
-
-    public async Task InitializeAsync()
-    {
-        await _db.CreateTableAsync<SaveFileMetadata>();
-    }
-    
-    public async Task<SaveFileMetadata?> GetMetadata(string id)
-    {
-        return (await _db.QueryAsync<SaveFileMetadata>(
-            "select * from SaveFileMetadata where id = ?", id))
-            .FirstOrDefault();
-    }
-    
-    public async Task<SaveFileMetadata> CreateMetadata(Game game)
-    {
-        var meta = new SaveFileMetadata
-        {
-            id = Guid.NewGuid().ToString(),
-            game = game,
-            status = SaveFileProcessingStatus.Initialized,
-            uploadDateTime = DateTime.Now
-        };
-        await _db.InsertAsync(meta);
-        return meta;
-    }
-
-    public async Task UpdateMetadataStatus(SaveFileMetadata meta, SaveFileProcessingStatus status)
-    {
-        meta.status = status;
-        await _db.UpdateAsync(meta);
-    }
-
-    public async Task SetMetadataError(SaveFileMetadata meta, string errorMessage)
-    {
-        meta.errorMessage = errorMessage;
-        await UpdateMetadataStatus(meta, SaveFileProcessingStatus.Error);
-        DeleteAssociatedFiles(meta);
-    }
-
-    public async Task DeleteMetadata(SaveFileMetadata meta)
-    {
-        _logger.LogDebug($"Deleting save file (id: {meta.id} status: {meta.status} " +
-                         $"uploadDate: {meta.uploadDateTime:yyyy/MM/dd})");
-        await _db.DeleteAsync(meta);
-        DeleteAssociatedFiles(meta);
-    }
-
-    /// <summary>
-    /// WARNING: Call this method only when no parsing operations are running.
-    /// </summary>
-    public async Task DeleteInvalidData()
-    {
-        _logger.LogInfo("Deleting invalid data...");
-        
-        var expirationDate = DateTime.Now.AddDays(-_uploadsLifetimeDays);
-        var metadataTable = _db.Table<SaveFileMetadata>();
-        int rowCount = await metadataTable.CountAsync();
-        int i = 0;
-        int deleteCount = 0;
-        while(i < rowCount)
-        {
-            var meta = await metadataTable.ElementAtAsync(i)!;
-            if(meta.status != SaveFileProcessingStatus.Done || 
-               meta.uploadDateTime < expirationDate ||
-               !meta.AssociatedFilesExist())
-            {
-                deleteCount++;
-                await DeleteMetadata(meta);
-            }
-            i++;
-        }
-
-        _logger.LogInfo($"Deleted {deleteCount} invalid records");
-    }
-
-    private void TryDeleteFile(IOPath file)
-    {
-        if (!File.Exists(file)) return;
-        _logger.LogDebug($"Deleting file '{file}'");
-        File.Delete(file);
-    }
-
-    private void DeleteAssociatedFiles(SaveFileMetadata meta)
-    {
-        TryDeleteFile(meta.GetSaveFilePath());
-        TryDeleteFile(meta.GetParsedDataPath());
-    }
-}

ParadoxSaveParser.WebAPI/Database/SaveFileMetadata.cs

@@ -1,45 +0,0 @@
-using System.Text.Json.Serialization;
-
-namespace ParadoxSaveParser.WebAPI.Database;
-
-public enum SaveFileProcessingStatus
-{
-    Initialized,
-    Uploaded,
-    Parsing,
-    SavingResults,
-    Done,
-    Error,
-}
-
-public class SaveFileMetadata
-{
-    [SQLite.PrimaryKey]
-    [SQLite.NotNull]
-    public string id { get; init; } = null!;
-
-    [JsonConverter(typeof(JsonStringEnumConverter))]
-    [SQLite.NotNull]
-    public Game game { get; init; }
-
-    [JsonConverter(typeof(JsonStringEnumConverter))]
-    [SQLite.NotNull]
-    public SaveFileProcessingStatus status { get; set; }
-    
-    [SQLite.NotNull]
-    public DateTime uploadDateTime { get; set; }
-    
-    // if error occured during parsing, it's message is saved here
-    // status stays the same as when error occured
-    public string? errorMessage { get; set; }
-    
-    
-    public IOPath GetSaveFilePath() => Path.Concat(PathHelper.SAVES_DIR, id + ".eu4");
-
-    public IOPath GetParsedDataPath() => Path.Concat(PathHelper.PARSED_DIR, id + ".parsed.json");
-
-    public bool AssociatedFilesExist()
-    {
-        return File.Exists(GetSaveFilePath()) && File.Exists(GetParsedDataPath());
-    }
-}

ParadoxSaveParser.WebAPI/Game.cs

@@ -1,7 +0,0 @@
-namespace ParadoxSaveParser.WebAPI;
-
-public enum Game
-{
-    Unknown,
-    EU4
-}

ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.csproj

@@ -15,7 +15,6 @@
     <ItemGroup>
         <PackageReference Include="DTLib.Web" Version="1.3.0"/>
         <PackageReference Include="Google.Protobuf" Version="3.30.2" />
-        <PackageReference Include="sqlite-net-pcl" Version="1.9.172" />
     </ItemGroup>
 
     <ItemGroup>

ParadoxSaveParser.WebAPI/PathHelper.cs

@@ -6,31 +6,16 @@ public static class PathHelper
 {
     public static readonly IOPath DATA_DIR = "data";
     public static readonly IOPath SAVES_DIR = Path.Concat(DATA_DIR, "saves");
-    public static readonly IOPath PARSED_DIR = Path.Concat(DATA_DIR, "parsed");
+
-    public static readonly IOPath TEMP_DIR = "temp";
+    public static IOPath GetMetaFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".meta.json");
+
+    public static IOPath GetSaveFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".eu4");
+
+    public static IOPath GetParsedSaveFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".parsed.json");
     
     public static void CreateProgramDirectories()
     {
         Directory.Create(DATA_DIR);
         Directory.Create(SAVES_DIR);
-        Directory.Create(PARSED_DIR);
-        Directory.Create(TEMP_DIR);
-    }
-
-    public static void CleanTempDirectory()
-    {
-        Directory.Delete(TEMP_DIR);
-        Directory.Create(TEMP_DIR);
-    }
-
-    public static Stream CreateTempFile()
-    {
-        string fileName = Guid.NewGuid().ToString();
-        IOPath filePath = Path.Concat(TEMP_DIR, fileName);
-        var stream = System.IO.File.Open(filePath.Str, FileMode.CreateNew, FileAccess.ReadWrite,
-            FileShare.Read | FileShare.Delete);
-        // file stays as a ghost until it is closed
-        File.Delete(filePath);
-        return stream;
     }
 }

ParadoxSaveParser.WebAPI/Program.cs

@@ -11,25 +11,27 @@ global using ParadoxSaveParser.Lib;
 global using Directory = DTLib.Filesystem.Directory;
 global using File = DTLib.Filesystem.File;
 global using Path = DTLib.Filesystem.Path;
+using System.Collections.Concurrent;
+using System.IO;
 using DTLib.Console;
 using DTLib.Dtsod;
 using DTLib.Web;
 using DTLib.Web.Routes;
 using ParadoxSaveParser.WebAPI.BackgroundTasks;
-using ParadoxSaveParser.WebAPI.Database;
 using ParadoxSaveParser.WebAPI.Routes;
-using ParadoxSaveParser.WebAPI.SaveDataFilters;
-// ReSharper disable MethodHasAsyncOverload
 
 namespace ParadoxSaveParser.WebAPI;
 
 public static class Program
 {
-    internal static bool IsDebug = true;
+    private static readonly IOPath _configPath = "./config.dtsod";
-    internal static DatabaseConnector DB = null!;
+    private static Config _config = new();
+    private static bool IsDebug = true;
+    private static readonly CancellationTokenSource _mainCancel = new();
+    internal static readonly ConcurrentDictionary<string, SaveFileMetadata> _saveMetadataStorage = new();
 
 
-    public static async Task Main(string[] args)
+    public static void Main(string[] args)
     {
         Console.InputEncoding = Encoding.UTF8;
         Console.OutputEncoding = Encoding.UTF8;
@@ -57,53 +59,61 @@ public static class Program
         var loggerMain = new ContextLogger(nameof(Main), loggerRoot);
         loggerMain.LogDebug("Debug log is enabled");
         
-        CancellationTokenSource mainCancel = new();
         Console.CancelKeyPress += (_, e) =>
         {
             e.Cancel = true;
             loggerMain.LogInfo("Ctrl+C Pressed");
-            mainCancel.Cancel();
+            _mainCancel.Cancel();
         };
 
         try
         {
             // config
-            IOPath configPath = "./config.dtsod";
+            if (!File.Exists(_configPath))
-            Config config;
-            if (File.Exists(configPath))
             {
-                config = Config.FromDtsod(new DtsodV23(File.ReadAllText(configPath)));
+                loggerMain.LogWarn("config file not found.");
+                File.WriteAllText(_configPath, _config.ToString());
+                loggerMain.LogWarn($"created default at {_configPath}.");
             }
             else
             {
-                loggerMain.LogWarn("config file not found.");
+                _config = Config.FromDtsod(new DtsodV23(File.ReadAllText(_configPath)));
-                config = new();
-                File.WriteAllText(configPath, config.ToString());
-                loggerMain.LogWarn($"created default at {configPath}.");
             }
 
             PathHelper.CreateProgramDirectories();
-            PathHelper.CleanTempDirectory();
+
-            DB = new("database.sqlite", 30, loggerRoot);
+            var metaFiles = System.IO.Directory.GetFiles(
-            await DB.InitializeAsync();
+                PathHelper.SAVES_DIR.Str, "*.meta.json",
-            await DB.DeleteInvalidData();
+                SearchOption.TopDirectoryOnly);
+            foreach (string metaFilePath in metaFiles)
+            {
+                using var metaFile = File.OpenRead(metaFilePath);
+                var meta = JsonSerializer.Deserialize<SaveFileMetadata>(metaFile) ??
+                           throw new NullReferenceException(metaFilePath);
+                if (meta.status != SaveFileProcessingStatus.Done)
+                    loggerMain.LogWarn(nameof(Main),
+                        $"metadata file '{metaFilePath}' status has invalid status {meta.status}");
+
+                if (!_saveMetadataStorage.TryAdd(meta.id, meta))
+                    throw new Exception("Guid collision!");
+            }
 
             var bgJobManager = new BackgroundJobManager(loggerRoot);
-            var saveFilters = new Dictionary<Game, ISaveDataFilter>
+            var saveParsingSearchExpressions = new Dictionary<Game, ISearchExpression>
             {
-                { Game.EU4, new SaveDataFilterEU4() },
+                { Game.EU4, SearchExpressionCompiler.Compile("*.~") },
             };
             
             // http server
             var router = new SimpleRouter(loggerRoot);
             router.DefaultRoute = new ServeFilesRouteHandler("public");
-            router.MapRoute("/getSaveStatus", HttpMethod.GET, new GetSaveStatusHandler(mainCancel.Token));
+            router.MapRoute("/getSaveStatus", HttpMethod.GET, new GetSaveStatusHandler(_mainCancel.Token));
-            router.MapRoute("/uploadSave", HttpMethod.POST, new UploadSaveHandler(mainCancel.Token, 
+            router.MapRoute("/uploadSave/eu4", HttpMethod.POST, new UploadSaveHandler(_mainCancel.Token, 
-                bgJobManager, saveFilters));
+                bgJobManager, saveParsingSearchExpressions));
-            router.MapRoute("/getSaveData", HttpMethod.GET, new GetSaveDataHandler(mainCancel.Token));
+            router.MapRoute("/getSaveData", HttpMethod.GET, new GetSaveDataHandler(_mainCancel.Token));
 
-            var app = new WebApp(config.BaseUrl, loggerRoot, router, mainCancel.Token);
+            var app = new WebApp(_config.BaseUrl, loggerRoot, router, _mainCancel.Token);
-            await app.Run();
+            app.Run().GetAwaiter().GetResult();
         }
         catch (OperationCanceledException ex)
         {
@@ -114,4 +124,6 @@ public static class Program
             loggerMain.LogError(ex.ToStringDemystified());
         }
     }
+
+    
 }

ParadoxSaveParser.WebAPI/Protobuf/ParsedData.proto

@@ -3,12 +3,11 @@ option csharp_namespace = "ParadoxSaveParser.WebAPI.MyProtobuf";
 
 message Item {
   oneof value {
-    bool b = 1;
+    int64 i64 = 1;
-    int64 i64 = 2;
+    double f64 = 2;
-    double f64 = 3;
+    string str = 3;
-    string str = 4;
+    ItemList list = 4;
-    ItemList list = 5;
+    ItemListMap map = 5;
-    ItemListMap map = 6;
   }
 }
 

ParadoxSaveParser.WebAPI/README.md

@@ -1,15 +1,10 @@
 # WebAPI
 Simple web application created using DTLib.Web.
 
+# Routes
 
-## Important
+### POST `/uploadSave/eu4`
-Restart web application once per day to delete outdated data and clean RAM.
+- **Request:** `application/octet-stream` - .eu4 file
-
-## Routes
-### POST `/uploadSave`
-- **Query Params:**
-  - `game` - short name of the game (see [../README.md](../README.md))
-- **Request Body:** `application/octet-stream` - .eu4 file
 - **Response:**
     ```json
     { "saveId": "string" }

ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs

@@ -1,5 +1,4 @@
 using System.Net;
-using ParadoxSaveParser.WebAPI.Database;
 using ParadoxSaveParser.WebAPI.HttpHelpers;
 
 namespace ParadoxSaveParser.WebAPI.Routes;
@@ -19,17 +18,7 @@ internal class GetSaveDataHandler : RouteHandlerBase
             return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, idOrError.Error!);
         string id = idOrError.Value!;
 
-        var meta = await Program.DB.GetMetadata(id);
+        IOPath dataFilePath = PathHelper.GetParsedSaveFilePath(id);
-        if(meta is null)
-            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, 
-                new ErrorMessage(HttpStatusCode.NotFound,
-                    $"Save with id {id} not found"));
-        if(meta.status != SaveFileProcessingStatus.Done)
-            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, 
-                new ErrorMessage(HttpStatusCode.BadRequest,
-                    $"Save with id {id} has status {meta.status}"));
-        
-        IOPath dataFilePath = meta.GetParsedDataPath();
         if (!File.Exists(dataFilePath))
             return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
                 new ErrorMessage(HttpStatusCode.InternalServerError, 

ParadoxSaveParser.WebAPI/Routes/GetSaveStatusHandler.cs

@@ -18,8 +18,7 @@ internal class GetSaveStatusHandler : RouteHandlerBase
             return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, idOrError.Error!);
         string id = idOrError.Value!;
         
-        var meta = await Program.DB.GetMetadata(id);
+        if (!Program._saveMetadataStorage.TryGetValue(id, out var meta))
-        if (meta is null)
             return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
                 new ErrorMessage(HttpStatusCode.InternalServerError, 
                     $"Save with id {id} not found")

ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs

@@ -1,26 +1,23 @@
 using System.Linq;
 using System.Net;
-using System.Reflection.Metadata;
 using ParadoxSaveParser.WebAPI.BackgroundTasks;
-using ParadoxSaveParser.WebAPI.Database;
 using ParadoxSaveParser.WebAPI.HttpHelpers;
-using ParadoxSaveParser.WebAPI.SaveDataFilters;
 
 namespace ParadoxSaveParser.WebAPI.Routes;
 
 public class UploadSaveHandler : RouteHandlerBase
 {
     private readonly BackgroundJobManager _bgJobManager;
-    private readonly Dictionary<Game, ISaveDataFilter> _saveFilters;
+    private readonly Dictionary<Game, ISearchExpression> _searchQueries;
 
     public UploadSaveHandler(
         CancellationToken cancelAllToken, 
         BackgroundJobManager bgJobManager,
-        Dictionary<Game, ISaveDataFilter> saveFilters) 
+        Dictionary<Game, ISearchExpression> searchQueries) 
         : base(cancelAllToken)
     {
         _bgJobManager = bgJobManager;
-        _saveFilters = saveFilters;
+        _searchQueries = searchQueries;
     }
 
     public override async Task<HttpStatusCode> HandleRequest(
@@ -30,28 +27,34 @@ public class UploadSaveHandler : RouteHandlerBase
         if (contentType != "application/octet-stream")
             return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
                     new ErrorMessage(HttpStatusCode.BadRequest,
-                $"Invalid request Content-Type: '{contentType}'"));
+                $"Invalid request Content-Type: '{contentType}'")
+                    );
 
-        var gameStrOrError = RequestHelper.GetQueryValue(ctx, "game");
+        string saveId = Guid.NewGuid().ToString();
-        if (gameStrOrError.HasError)
+        var metaFilePath = PathHelper.GetMetaFilePath(saveId);
-            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, gameStrOrError.Error!);
+        if (File.Exists(metaFilePath))
-
-        if (!Enum.TryParse(gameStrOrError.Value, ignoreCase: true, out Game game))
             return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
-                new ErrorMessage(HttpStatusCode.BadRequest,
+                    new ErrorMessage(HttpStatusCode.InternalServerError,
-                    $"Invalid requested game: '{gameStrOrError.Value}'"));  
+                $"Guid collision! file' {metaFilePath}' already exists.")
-        var meta = await Program.DB.CreateMetadata(game);
+                    );
 
-        var saveFilePath = meta.GetSaveFilePath();
+        var meta = new SaveFileMetadata
-        await using (var saveFile = File.OpenWrite(saveFilePath))
+            { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized };
-        {
+        if (!Program._saveMetadataStorage.TryAdd(saveId, meta))
-            await using (var remoteStream = ctx.Request.InputStream)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
+                    new ErrorMessage(HttpStatusCode.InternalServerError,
+                $"Guid collision! Can't create metadata with id {saveId}")
+                    );
+
+        meta.status = SaveFileProcessingStatus.Uploading;
+        var saveFilePath = PathHelper.GetSaveFilePath(meta.id);
+        await using var saveFile = File.OpenWrite(saveFilePath);
+        await using var remoteStream = ctx.Request.InputStream;
         await remoteStream.CopyToAsync(saveFile, _cancelAllToken);
-        }
+        meta.status = SaveFileProcessingStatus.Uploaded;
-        await Program.DB.UpdateMetadataStatus(meta, SaveFileProcessingStatus.Uploaded);
 
-        _bgJobManager.StartNewParsingOperation(meta, _saveFilters[meta.game], _cancelAllToken);
+        _bgJobManager.StartNewParsingOperation(meta, _searchQueries[meta.game], _cancelAllToken);
-        dynamic responseData = new { meta.id };
+        dynamic responseData = new { saveId };
         return await ReturnHelper.ResponseJson(ctx, requestLogger, _cancelAllToken, responseData);
     }
 }

ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs

@@ -1,9 +0,0 @@
-namespace ParadoxSaveParser.WebAPI.SaveDataFilters;
-
-public interface ISaveDataFilter
-{
-    public string SearchString { get; }
-    public ISearchExpression SearchExpression { get; }
-    
-    public void Apply(Dictionary<string, object> data);
-}

ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs

@@ -1,48 +0,0 @@
-using System.Linq;
-using ParsedDict = System.Collections.Generic.Dictionary<string, object>;
-
-namespace ParadoxSaveParser.WebAPI.SaveDataFilters;
-
-public class SaveDataFilterEU4 : ISaveDataFilter
-{
-    public string SearchString { get; }
-    public ISearchExpression SearchExpression { get; }
-    
-    public SaveDataFilterEU4()
-    {
-        SearchString =
-            """
-            (
-                active_war|
-                previous_war|
-                income_statistics|
-                nation_size_statistics|
-                inflation_statistics|
-                countries.(
-                    ---.~|REB.~|PIR.~|NAT.~|
-                    *.(
-                        flags.~|hidden_flags.~|variables.~|estate.~|active_agenda.~|
-                        power_projection.~|ai.~|history.~|navy.~|army.~|mercenary_company.~|
-                        active_relations.~|border_pct.~|border_sit.~|border_provinces.~|
-                        neighbours.~|home_neighbours.~|core_neighbours.~|inflation_history.~|
-                        opinion_cache.~|owned_provinces.~|controlled_provinces.~|core_provinces.~|
-                        claim_provinces.~|leader.~|*
-                    )
-                )
-            )
-            """
-            .Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", "");
-        SearchExpression = SearchExpressionCompiler.Compile(SearchString);
-    }
-
-    
-    public void Apply(ParsedDict data)
-    {
-        var countries = (ParsedDict)data["countries"];
-        var countries_filtered = countries.Where(pair 
-            => ((ParsedDict)pair.Value).TryGetValue("raw_development", out var raw_development)
-               && (double)raw_development > 0
-            ).ToDictionary();
-        data["countries"] = countries_filtered;
-    }
-}

ParadoxSaveParser.WebAPI/SaveFileMetadata.cs

@@ -0,0 +1,45 @@
+using System.Text.Json.Serialization;
+
+namespace ParadoxSaveParser.WebAPI;
+
+public enum SaveFileProcessingStatus
+{
+    Initialized,
+    Uploading,
+    Uploaded,
+    Parsing,
+    SavingResults,
+    Done
+}
+
+public enum Game
+{
+    Unknown,
+    EU4
+}
+
+public class SaveFileMetadata
+{
+    private static readonly JsonSerializerOptions _configSerializerOptions = new()
+    {
+        WriteIndented = true,
+        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+    };
+    public required string id { get; init; }
+
+    [JsonConverter(typeof(JsonStringEnumConverter))]
+    public required Game game { get; init; }
+
+    [JsonConverter(typeof(JsonStringEnumConverter))]
+    public required SaveFileProcessingStatus status { get; set; }
+    
+    // if error occured during parsing, it's message is saved here
+    // status stays the same as when error occured
+    public string? errorMessage { get; set; }
+
+    public void SaveToFile()
+    {
+        using var metaFile = File.OpenWrite(PathHelper.GetMetaFilePath(id));
+        JsonSerializer.Serialize(metaFile, this, _configSerializerOptions);
+    }
+}

TODO.md

@@ -1,3 +1,8 @@
+## WebAPI:
+	Temp files management system
+	Database to store metadata (SQLite-Net Extensions)
+
 ## WebAPI.SaveParsingOperation:
+	Add debug log
 	Save parsed data in protobuf
 	Re-parse if saved data was parsed with another query