diff --git a/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs b/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs index d41995c..1723add 100644 --- a/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs +++ b/ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs @@ -1,4 +1,6 @@ -namespace ParadoxSaveParser.WebAPI.BackgroundTasks; +using ParadoxSaveParser.WebAPI.SaveDataFilters; + +namespace ParadoxSaveParser.WebAPI.BackgroundTasks; public class BackgroundJobManager { @@ -11,11 +13,11 @@ public class BackgroundJobManager } public SaveParsingOperation StartNewParsingOperation( - SaveFileMetadata meta, ISearchExpression searchQuery, CancellationToken ct) + SaveFileMetadata meta, ISaveDataFilter filter, CancellationToken ct) { long nextId = Interlocked.Increment(ref _lastJobId); var contextLogger = new ContextLogger($"BackgroundJob-{nextId}", _parentLogger); - var op = new SaveParsingOperation(nextId, meta, searchQuery, contextLogger, ct); + var op = new SaveParsingOperation(nextId, meta, filter, contextLogger, ct); op.StartAsync(); return op; } diff --git a/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs b/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs index ff873be..93e8571 100644 --- a/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs +++ b/ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs @@ -2,15 +2,16 @@ using System.Linq; using System.Text.Encodings.Web; using System.Text.Json.Serialization; +using ParadoxSaveParser.WebAPI.SaveDataFilters; namespace ParadoxSaveParser.WebAPI.BackgroundTasks; public class SaveParsingOperation { public readonly long OperationId; - public readonly SaveFileMetadata Meta; - public readonly ISearchExpression SearchQuery; + private readonly SaveFileMetadata _meta; + private readonly ISaveDataFilter _filter; private readonly ContextLogger _logger; private readonly CancellationToken _cancelToken; @@ -22,12 +23,12 @@ public class SaveParsingOperation DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, }; - public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISearchExpression searchQuery, + public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISaveDataFilter filter, ContextLogger logger, CancellationToken cancelToken) { OperationId = operationId; - Meta = meta; - SearchQuery = searchQuery; + _meta = meta; + _filter = filter; _logger = logger; _cancelToken = cancelToken; } @@ -36,22 +37,22 @@ public class SaveParsingOperation { try { - _logger.LogInfo($"Starting background parsing operation of {Meta.game} save {Meta.id}"); - switch (Meta.game) + _logger.LogInfo($"Starting background parsing operation of {_meta.game} save {_meta.id}"); + switch (_meta.game) { case Game.EU4: await ParseSaveEU4(); break; default: - throw new ArgumentOutOfRangeException(Meta.game.ToString()); + throw new ArgumentOutOfRangeException(_meta.game.ToString()); } - _logger.LogInfo($"Finished parsing operation of {Meta.game} save {Meta.id}"); + _logger.LogInfo($"Finished parsing operation of {_meta.game} save {_meta.id}"); } catch (Exception ex) { string errorMesage = ex.ToStringDemystified(); _logger.LogError(errorMesage); - Meta.errorMessage = errorMesage; + _meta.errorMessage = errorMesage; } finally { @@ -63,8 +64,8 @@ public class SaveParsingOperation { // wait for save file closing await Task.Delay(200, _cancelToken); - string extractedGamestatePath = PathHelper.GetSaveFilePath(Meta.id) + ".gamestate"; - using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(Meta.id).Str, ZipArchiveMode.Read)) + string extractedGamestatePath = PathHelper.GetSaveFilePath(_meta.id) + ".gamestate"; + using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(_meta.id).Str, ZipArchiveMode.Read)) { var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate"); if (zipEntry is null) @@ -75,16 +76,17 @@ public class SaveParsingOperation var gamestateStream = File.OpenRead(extractedGamestatePath); - Meta.status = SaveFileProcessingStatus.Parsing; - var parser = new SaveParserEU4(gamestateStream, SearchQuery); + _meta.status = SaveFileProcessingStatus.Parsing; + var parser = new SaveParserEU4(gamestateStream, _filter.SearchExpression); var result = parser.Parse(); + _filter.Apply(result); - Meta.status = SaveFileProcessingStatus.SavingResults; - var resultFilePath = PathHelper.GetParsedSaveFilePath(Meta.id); + _meta.status = SaveFileProcessingStatus.SavingResults; + var resultFilePath = PathHelper.GetParsedSaveFilePath(_meta.id); await using var resultFile = File.OpenWrite(resultFilePath); await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions, _cancelToken); - Meta.status = SaveFileProcessingStatus.Done; - Meta.SaveToFile(); + _meta.status = SaveFileProcessingStatus.Done; + _meta.SaveToFile(); } } \ No newline at end of file diff --git a/ParadoxSaveParser.WebAPI/Program.cs b/ParadoxSaveParser.WebAPI/Program.cs index 133af0d..918d94d 100644 --- a/ParadoxSaveParser.WebAPI/Program.cs +++ b/ParadoxSaveParser.WebAPI/Program.cs @@ -19,6 +19,7 @@ using DTLib.Web; using DTLib.Web.Routes; using ParadoxSaveParser.WebAPI.BackgroundTasks; using ParadoxSaveParser.WebAPI.Routes; +using ParadoxSaveParser.WebAPI.SaveDataFilters; namespace ParadoxSaveParser.WebAPI; @@ -99,9 +100,9 @@ public static class Program } var bgJobManager = new BackgroundJobManager(loggerRoot); - var saveParsingSearchExpressions = new Dictionary + var saveFilters = new Dictionary { - { Game.EU4, SearchExpressionCompiler.Compile("*.~") }, + { Game.EU4, new SaveDataFilterEU4() }, }; // http server @@ -109,7 +110,7 @@ public static class Program router.DefaultRoute = new ServeFilesRouteHandler("public"); router.MapRoute("/getSaveStatus", HttpMethod.GET, new GetSaveStatusHandler(_mainCancel.Token)); router.MapRoute("/uploadSave/eu4", HttpMethod.POST, new UploadSaveHandler(_mainCancel.Token, - bgJobManager, saveParsingSearchExpressions)); + bgJobManager, saveFilters)); router.MapRoute("/getSaveData", HttpMethod.GET, new GetSaveDataHandler(_mainCancel.Token)); var app = new WebApp(_config.BaseUrl, loggerRoot, router, _mainCancel.Token); diff --git a/ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs b/ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs index 878d405..1aed08e 100644 --- a/ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs +++ b/ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs @@ -18,6 +18,15 @@ internal class GetSaveDataHandler : RouteHandlerBase return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, idOrError.Error!); string id = idOrError.Value!; + if(!Program._saveMetadataStorage.TryGetValue(id, out var meta)) + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, + new ErrorMessage(HttpStatusCode.NotFound, + $"Save with id {id} not found")); + if(meta.status != SaveFileProcessingStatus.Done) + return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, + new ErrorMessage(HttpStatusCode.BadRequest, + $"Save with id {id} has status {meta.status}")); + IOPath dataFilePath = PathHelper.GetParsedSaveFilePath(id); if (!File.Exists(dataFilePath)) return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, diff --git a/ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs b/ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs index dd68831..6d3b960 100644 --- a/ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs +++ b/ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs @@ -2,22 +2,23 @@ using System.Net; using ParadoxSaveParser.WebAPI.BackgroundTasks; using ParadoxSaveParser.WebAPI.HttpHelpers; +using ParadoxSaveParser.WebAPI.SaveDataFilters; namespace ParadoxSaveParser.WebAPI.Routes; public class UploadSaveHandler : RouteHandlerBase { private readonly BackgroundJobManager _bgJobManager; - private readonly Dictionary _searchQueries; + private readonly Dictionary _saveFilters; public UploadSaveHandler( CancellationToken cancelAllToken, BackgroundJobManager bgJobManager, - Dictionary searchQueries) + Dictionary saveFilters) : base(cancelAllToken) { _bgJobManager = bgJobManager; - _searchQueries = searchQueries; + _saveFilters = saveFilters; } public override async Task HandleRequest( @@ -53,7 +54,7 @@ public class UploadSaveHandler : RouteHandlerBase await remoteStream.CopyToAsync(saveFile, _cancelAllToken); meta.status = SaveFileProcessingStatus.Uploaded; - _bgJobManager.StartNewParsingOperation(meta, _searchQueries[meta.game], _cancelAllToken); + _bgJobManager.StartNewParsingOperation(meta, _saveFilters[meta.game], _cancelAllToken); dynamic responseData = new { saveId }; return await ReturnHelper.ResponseJson(ctx, requestLogger, _cancelAllToken, responseData); } diff --git a/ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs b/ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs new file mode 100644 index 0000000..a54838d --- /dev/null +++ b/ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs @@ -0,0 +1,9 @@ +namespace ParadoxSaveParser.WebAPI.SaveDataFilters; + +public interface ISaveDataFilter +{ + public string SearchString { get; } + public ISearchExpression SearchExpression { get; } + + public void Apply(Dictionary> data); +} \ No newline at end of file diff --git a/ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs b/ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs new file mode 100644 index 0000000..04ad3ca --- /dev/null +++ b/ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs @@ -0,0 +1,49 @@ +using System.Linq; +using ParsedDict = System.Collections.Generic.Dictionary>; + +namespace ParadoxSaveParser.WebAPI.SaveDataFilters; + +public class SaveDataFilterEU4 : ISaveDataFilter +{ + public string SearchString { get; } + public ISearchExpression SearchExpression { get; } + + public SaveDataFilterEU4() + { + SearchString = + """ + ( + active_war| + previous_war| + income_statistics| + nation_size_statistics| + inflation_statistics| + countries.( + ---.~|REB.~|PIR.~|NAT.~| + *.( + flags.~|hidden_flags.~|variables.~|estate.~|active_agenda.~| + power_projection.~|ai.~|history.~|navy.~|army.~|mercenary_company.~| + active_relations.~|border_pct.~|border_sit.~|border_provinces.~| + neighbours.~|home_neighbours.~|core_neighbours.~|inflation_history.~| + opinion_cache.~|owned_provinces.~|controlled_provinces.~|core_provinces.~| + claim_provinces.~|leader.~|* + ) + ) + ) + """ + .Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", ""); + SearchExpression = SearchExpressionCompiler.Compile(SearchString); + } + + + public void Apply(ParsedDict data) + { + var countries = (ParsedDict)data["countries"][0]; + var countries_filtered = countries.Where(pair + => pair.Value.Count > 0 + && ((ParsedDict)pair.Value[0]).TryGetValue("raw_development", out var raw_development) + && (double)raw_development[0] > 0 + ).ToDictionary(); + data["countries"][0] = countries_filtered; + } +} \ No newline at end of file diff --git a/ParadoxSaveParser.WebAPI/data_filtering.txt b/ParadoxSaveParser.WebAPI/SaveDataFilters/data_filtering_EU4.txt similarity index 100% rename from ParadoxSaveParser.WebAPI/data_filtering.txt rename to ParadoxSaveParser.WebAPI/SaveDataFilters/data_filtering_EU4.txt