SaveDataFilter

ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs

@@ -1,4 +1,6 @@
-namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
+using ParadoxSaveParser.WebAPI.SaveDataFilters;
+
+namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
 
 public class BackgroundJobManager
 {
@@ -11,11 +13,11 @@ public class BackgroundJobManager
     }
     
     public SaveParsingOperation StartNewParsingOperation(
-        SaveFileMetadata meta, ISearchExpression searchQuery, CancellationToken ct)
+        SaveFileMetadata meta, ISaveDataFilter filter, CancellationToken ct)
     {
         long nextId = Interlocked.Increment(ref _lastJobId);
         var contextLogger = new ContextLogger($"BackgroundJob-{nextId}", _parentLogger);
-        var op = new SaveParsingOperation(nextId, meta, searchQuery, contextLogger, ct);
+        var op = new SaveParsingOperation(nextId, meta, filter, contextLogger, ct);
         op.StartAsync();
         return op;
     }

ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs

@@ -2,15 +2,16 @@
 using System.Linq;
 using System.Text.Encodings.Web;
 using System.Text.Json.Serialization;
+using ParadoxSaveParser.WebAPI.SaveDataFilters;
 
 namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
 
 public class SaveParsingOperation
 {
     public readonly long OperationId;
-    public readonly SaveFileMetadata Meta;
-    public readonly ISearchExpression SearchQuery;
     
+    private readonly SaveFileMetadata _meta;
+    private readonly ISaveDataFilter _filter;
     private readonly ContextLogger _logger;
     private readonly CancellationToken _cancelToken;
 
@@ -22,12 +23,12 @@ public class SaveParsingOperation
         DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
     };
 
-    public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISearchExpression searchQuery,
+    public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISaveDataFilter filter,
         ContextLogger logger, CancellationToken cancelToken)
     {
         OperationId = operationId;
-        Meta = meta;
+        _meta = meta;
-        SearchQuery = searchQuery;
+        _filter = filter;
         _logger = logger;
         _cancelToken = cancelToken;
     }
@@ -36,22 +37,22 @@ public class SaveParsingOperation
     {
         try
         {
-            _logger.LogInfo($"Starting background parsing operation of {Meta.game} save {Meta.id}");
+            _logger.LogInfo($"Starting background parsing operation of {_meta.game} save {_meta.id}");
-            switch (Meta.game)
+            switch (_meta.game)
             {
                 case Game.EU4:
                     await ParseSaveEU4();
                     break;
                 default:
-                    throw new ArgumentOutOfRangeException(Meta.game.ToString());
+                    throw new ArgumentOutOfRangeException(_meta.game.ToString());
             }
-            _logger.LogInfo($"Finished parsing operation of {Meta.game} save {Meta.id}");
+            _logger.LogInfo($"Finished parsing operation of {_meta.game} save {_meta.id}");
         }
         catch (Exception ex)
         {
             string errorMesage = ex.ToStringDemystified();
             _logger.LogError(errorMesage);
-            Meta.errorMessage = errorMesage;
+            _meta.errorMessage = errorMesage;
         }
         finally
         {
@@ -63,8 +64,8 @@ public class SaveParsingOperation
     {
         // wait for save file closing
         await Task.Delay(200, _cancelToken);
-        string extractedGamestatePath = PathHelper.GetSaveFilePath(Meta.id) + ".gamestate";
+        string extractedGamestatePath = PathHelper.GetSaveFilePath(_meta.id) + ".gamestate";
-        using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(Meta.id).Str, ZipArchiveMode.Read))
+        using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(_meta.id).Str, ZipArchiveMode.Read))
         {
             var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
             if (zipEntry is null)
@@ -75,16 +76,17 @@ public class SaveParsingOperation
 
         var gamestateStream = File.OpenRead(extractedGamestatePath);
 
-        Meta.status = SaveFileProcessingStatus.Parsing;
+        _meta.status = SaveFileProcessingStatus.Parsing;
-        var parser = new SaveParserEU4(gamestateStream, SearchQuery);
+        var parser = new SaveParserEU4(gamestateStream, _filter.SearchExpression);
         var result = parser.Parse();
+        _filter.Apply(result);
 
-        Meta.status = SaveFileProcessingStatus.SavingResults;
+        _meta.status = SaveFileProcessingStatus.SavingResults;
-        var resultFilePath = PathHelper.GetParsedSaveFilePath(Meta.id);
+        var resultFilePath = PathHelper.GetParsedSaveFilePath(_meta.id);
         await using var resultFile = File.OpenWrite(resultFilePath);
         await JsonSerializer.SerializeAsync(resultFile, result, 
             _saveSerializerOptions, _cancelToken);
-        Meta.status = SaveFileProcessingStatus.Done;
+        _meta.status = SaveFileProcessingStatus.Done;
-        Meta.SaveToFile();
+        _meta.SaveToFile();
     }
 }

ParadoxSaveParser.WebAPI/Program.cs

@@ -19,6 +19,7 @@ using DTLib.Web;
 using DTLib.Web.Routes;
 using ParadoxSaveParser.WebAPI.BackgroundTasks;
 using ParadoxSaveParser.WebAPI.Routes;
+using ParadoxSaveParser.WebAPI.SaveDataFilters;
 
 namespace ParadoxSaveParser.WebAPI;
 
@@ -99,9 +100,9 @@ public static class Program
             }
 
             var bgJobManager = new BackgroundJobManager(loggerRoot);
-            var saveParsingSearchExpressions = new Dictionary<Game, ISearchExpression>
+            var saveFilters = new Dictionary<Game, ISaveDataFilter>
             {
-                { Game.EU4, SearchExpressionCompiler.Compile("*.~") },
+                { Game.EU4, new SaveDataFilterEU4() },
             };
             
             // http server
@@ -109,7 +110,7 @@ public static class Program
             router.DefaultRoute = new ServeFilesRouteHandler("public");
             router.MapRoute("/getSaveStatus", HttpMethod.GET, new GetSaveStatusHandler(_mainCancel.Token));
             router.MapRoute("/uploadSave/eu4", HttpMethod.POST, new UploadSaveHandler(_mainCancel.Token, 
-                bgJobManager, saveParsingSearchExpressions));
+                bgJobManager, saveFilters));
             router.MapRoute("/getSaveData", HttpMethod.GET, new GetSaveDataHandler(_mainCancel.Token));
 
             var app = new WebApp(_config.BaseUrl, loggerRoot, router, _mainCancel.Token);

ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs

@@ -18,6 +18,15 @@ internal class GetSaveDataHandler : RouteHandlerBase
             return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, idOrError.Error!);
         string id = idOrError.Value!;
 
+        if(!Program._saveMetadataStorage.TryGetValue(id, out var meta))
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, 
+                new ErrorMessage(HttpStatusCode.NotFound,
+                    $"Save with id {id} not found"));
+        if(meta.status != SaveFileProcessingStatus.Done)
+            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, 
+                new ErrorMessage(HttpStatusCode.BadRequest,
+                    $"Save with id {id} has status {meta.status}"));
+        
         IOPath dataFilePath = PathHelper.GetParsedSaveFilePath(id);
         if (!File.Exists(dataFilePath))
             return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,

ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs

@@ -2,22 +2,23 @@
 using System.Net;
 using ParadoxSaveParser.WebAPI.BackgroundTasks;
 using ParadoxSaveParser.WebAPI.HttpHelpers;
+using ParadoxSaveParser.WebAPI.SaveDataFilters;
 
 namespace ParadoxSaveParser.WebAPI.Routes;
 
 public class UploadSaveHandler : RouteHandlerBase
 {
     private readonly BackgroundJobManager _bgJobManager;
-    private readonly Dictionary<Game, ISearchExpression> _searchQueries;
+    private readonly Dictionary<Game, ISaveDataFilter> _saveFilters;
 
     public UploadSaveHandler(
         CancellationToken cancelAllToken, 
         BackgroundJobManager bgJobManager,
-        Dictionary<Game, ISearchExpression> searchQueries) 
+        Dictionary<Game, ISaveDataFilter> saveFilters) 
         : base(cancelAllToken)
     {
         _bgJobManager = bgJobManager;
-        _searchQueries = searchQueries;
+        _saveFilters = saveFilters;
     }
 
     public override async Task<HttpStatusCode> HandleRequest(
@@ -53,7 +54,7 @@ public class UploadSaveHandler : RouteHandlerBase
         await remoteStream.CopyToAsync(saveFile, _cancelAllToken);
         meta.status = SaveFileProcessingStatus.Uploaded;
 
-        _bgJobManager.StartNewParsingOperation(meta, _searchQueries[meta.game], _cancelAllToken);
+        _bgJobManager.StartNewParsingOperation(meta, _saveFilters[meta.game], _cancelAllToken);
         dynamic responseData = new { saveId };
         return await ReturnHelper.ResponseJson(ctx, requestLogger, _cancelAllToken, responseData);
     }

ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs

@@ -0,0 +1,9 @@
+namespace ParadoxSaveParser.WebAPI.SaveDataFilters;
+
+public interface ISaveDataFilter
+{
+    public string SearchString { get; }
+    public ISearchExpression SearchExpression { get; }
+    
+    public void Apply(Dictionary<string, List<object>> data);
+}

ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs

@@ -0,0 +1,49 @@
+using System.Linq;
+using ParsedDict = System.Collections.Generic.Dictionary<string, System.Collections.Generic.List<object>>;
+
+namespace ParadoxSaveParser.WebAPI.SaveDataFilters;
+
+public class SaveDataFilterEU4 : ISaveDataFilter
+{
+    public string SearchString { get; }
+    public ISearchExpression SearchExpression { get; }
+    
+    public SaveDataFilterEU4()
+    {
+        SearchString =
+            """
+            (
+                active_war|
+                previous_war|
+                income_statistics|
+                nation_size_statistics|
+                inflation_statistics|
+                countries.(
+                    ---.~|REB.~|PIR.~|NAT.~|
+                    *.(
+                        flags.~|hidden_flags.~|variables.~|estate.~|active_agenda.~|
+                        power_projection.~|ai.~|history.~|navy.~|army.~|mercenary_company.~|
+                        active_relations.~|border_pct.~|border_sit.~|border_provinces.~|
+                        neighbours.~|home_neighbours.~|core_neighbours.~|inflation_history.~|
+                        opinion_cache.~|owned_provinces.~|controlled_provinces.~|core_provinces.~|
+                        claim_provinces.~|leader.~|*
+                    )
+                )
+            )
+            """
+            .Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", "");
+        SearchExpression = SearchExpressionCompiler.Compile(SearchString);
+    }
+
+    
+    public void Apply(ParsedDict data)
+    {
+        var countries = (ParsedDict)data["countries"][0];
+        var countries_filtered = countries.Where(pair 
+            => pair.Value.Count > 0 
+               && ((ParsedDict)pair.Value[0]).TryGetValue("raw_development", out var raw_development)
+               && (double)raw_development[0] > 0
+            ).ToDictionary();
+        data["countries"][0] = countries_filtered;
+    }
+}