Timerix/ParadoxSaveParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SaveDataFilter

Browse Source
This commit is contained in:
Timerix 2025-04-10 15:42:39 +05:00
parent 95c0403362
commit d5b6061cc7
8 changed files with 101 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs
View File

@ -1,4 +1,6 @@
namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
using ParadoxSaveParser.WebAPI.SaveDataFilters;
namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
public class BackgroundJobManager
{
@ -11,11 +13,11 @@ public class BackgroundJobManager
}
public SaveParsingOperation StartNewParsingOperation(
SaveFileMetadata meta, ISearchExpression searchQuery, CancellationToken ct)
SaveFileMetadata meta, ISaveDataFilter filter, CancellationToken ct)
{
long nextId = Interlocked.Increment(ref _lastJobId);
var contextLogger = new ContextLogger($"BackgroundJob-{nextId}", _parentLogger);
var op = new SaveParsingOperation(nextId, meta, searchQuery, contextLogger, ct);
var op = new SaveParsingOperation(nextId, meta, filter, contextLogger, ct);
op.StartAsync();
return op;
}

38
ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs
View File

@ -2,15 +2,16 @@
using System.Linq;
using System.Text.Encodings.Web;
using System.Text.Json.Serialization;
using ParadoxSaveParser.WebAPI.SaveDataFilters;
namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
public class SaveParsingOperation
{
public readonly long OperationId;
public readonly SaveFileMetadata Meta;
public readonly ISearchExpression SearchQuery;
private readonly SaveFileMetadata _meta;
private readonly ISaveDataFilter _filter;
private readonly ContextLogger _logger;
private readonly CancellationToken _cancelToken;
@ -22,12 +23,12 @@ public class SaveParsingOperation
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISearchExpression searchQuery,
public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISaveDataFilter filter,
ContextLogger logger, CancellationToken cancelToken)
{
OperationId = operationId;
Meta = meta;
SearchQuery = searchQuery;
_meta = meta;
_filter = filter;
_logger = logger;
_cancelToken = cancelToken;
}
@ -36,22 +37,22 @@ public class SaveParsingOperation
{
try
{
_logger.LogInfo($"Starting background parsing operation of {Meta.game} save {Meta.id}");
switch (Meta.game)
_logger.LogInfo($"Starting background parsing operation of {_meta.game} save {_meta.id}");
switch (_meta.game)
{
case Game.EU4:
await ParseSaveEU4();
break;
default:
throw new ArgumentOutOfRangeException(Meta.game.ToString());
throw new ArgumentOutOfRangeException(_meta.game.ToString());
}
_logger.LogInfo($"Finished parsing operation of {Meta.game} save {Meta.id}");
_logger.LogInfo($"Finished parsing operation of {_meta.game} save {_meta.id}");
}
catch (Exception ex)
{
string errorMesage = ex.ToStringDemystified();
_logger.LogError(errorMesage);
Meta.errorMessage = errorMesage;
_meta.errorMessage = errorMesage;
}
finally
{
@ -63,8 +64,8 @@ public class SaveParsingOperation
{
// wait for save file closing
await Task.Delay(200, _cancelToken);
string extractedGamestatePath = PathHelper.GetSaveFilePath(Meta.id) + ".gamestate";
using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(Meta.id).Str, ZipArchiveMode.Read))
string extractedGamestatePath = PathHelper.GetSaveFilePath(_meta.id) + ".gamestate";
using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(_meta.id).Str, ZipArchiveMode.Read))
{
var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
if (zipEntry is null)
@ -75,16 +76,17 @@ public class SaveParsingOperation
var gamestateStream = File.OpenRead(extractedGamestatePath);
Meta.status = SaveFileProcessingStatus.Parsing;
var parser = new SaveParserEU4(gamestateStream, SearchQuery);
_meta.status = SaveFileProcessingStatus.Parsing;
var parser = new SaveParserEU4(gamestateStream, _filter.SearchExpression);
var result = parser.Parse();
_filter.Apply(result);
Meta.status = SaveFileProcessingStatus.SavingResults;
var resultFilePath = PathHelper.GetParsedSaveFilePath(Meta.id);
_meta.status = SaveFileProcessingStatus.SavingResults;
var resultFilePath = PathHelper.GetParsedSaveFilePath(_meta.id);
await using var resultFile = File.OpenWrite(resultFilePath);
await JsonSerializer.SerializeAsync(resultFile, result,
_saveSerializerOptions, _cancelToken);
Meta.status = SaveFileProcessingStatus.Done;
Meta.SaveToFile();
_meta.status = SaveFileProcessingStatus.Done;
_meta.SaveToFile();
}
}

7
ParadoxSaveParser.WebAPI/Program.cs
View File

@ -19,6 +19,7 @@ using DTLib.Web;
using DTLib.Web.Routes;
using ParadoxSaveParser.WebAPI.BackgroundTasks;
using ParadoxSaveParser.WebAPI.Routes;
using ParadoxSaveParser.WebAPI.SaveDataFilters;
namespace ParadoxSaveParser.WebAPI;
@ -99,9 +100,9 @@ public static class Program
}
var bgJobManager = new BackgroundJobManager(loggerRoot);
var saveParsingSearchExpressions = new Dictionary<Game, ISearchExpression>
var saveFilters = new Dictionary<Game, ISaveDataFilter>
{
{ Game.EU4, SearchExpressionCompiler.Compile("*.~") },
{ Game.EU4, new SaveDataFilterEU4() },
};
// http server
@ -109,7 +110,7 @@ public static class Program
router.DefaultRoute = new ServeFilesRouteHandler("public");
router.MapRoute("/getSaveStatus", HttpMethod.GET, new GetSaveStatusHandler(_mainCancel.Token));
router.MapRoute("/uploadSave/eu4", HttpMethod.POST, new UploadSaveHandler(_mainCancel.Token,
bgJobManager, saveParsingSearchExpressions));
bgJobManager, saveFilters));
router.MapRoute("/getSaveData", HttpMethod.GET, new GetSaveDataHandler(_mainCancel.Token));
var app = new WebApp(_config.BaseUrl, loggerRoot, router, _mainCancel.Token);

9
ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs
View File

@ -18,6 +18,15 @@ internal class GetSaveDataHandler : RouteHandlerBase
return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, idOrError.Error!);
string id = idOrError.Value!;
if(!Program._saveMetadataStorage.TryGetValue(id, out var meta))
return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
new ErrorMessage(HttpStatusCode.NotFound,
$"Save with id {id} not found"));
if(meta.status != SaveFileProcessingStatus.Done)
return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
new ErrorMessage(HttpStatusCode.BadRequest,
$"Save with id {id} has status {meta.status}"));
IOPath dataFilePath = PathHelper.GetParsedSaveFilePath(id);
if (!File.Exists(dataFilePath))
return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,

9
ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs
View File

@ -2,22 +2,23 @@
using System.Net;
using ParadoxSaveParser.WebAPI.BackgroundTasks;
using ParadoxSaveParser.WebAPI.HttpHelpers;
using ParadoxSaveParser.WebAPI.SaveDataFilters;
namespace ParadoxSaveParser.WebAPI.Routes;
public class UploadSaveHandler : RouteHandlerBase
{
private readonly BackgroundJobManager _bgJobManager;
private readonly Dictionary<Game, ISearchExpression> _searchQueries;
private readonly Dictionary<Game, ISaveDataFilter> _saveFilters;
public UploadSaveHandler(
CancellationToken cancelAllToken,
BackgroundJobManager bgJobManager,
Dictionary<Game, ISearchExpression> searchQueries)
Dictionary<Game, ISaveDataFilter> saveFilters)
: base(cancelAllToken)
{
_bgJobManager = bgJobManager;
_searchQueries = searchQueries;
_saveFilters = saveFilters;
}
public override async Task<HttpStatusCode> HandleRequest(
@ -53,7 +54,7 @@ public class UploadSaveHandler : RouteHandlerBase
await remoteStream.CopyToAsync(saveFile, _cancelAllToken);
meta.status = SaveFileProcessingStatus.Uploaded;
_bgJobManager.StartNewParsingOperation(meta, _searchQueries[meta.game], _cancelAllToken);
_bgJobManager.StartNewParsingOperation(meta, _saveFilters[meta.game], _cancelAllToken);
dynamic responseData = new { saveId };
return await ReturnHelper.ResponseJson(ctx, requestLogger, _cancelAllToken, responseData);
}

9
ParadoxSaveParser.WebAPI/SaveDataFilters/ISaveDataFilter.cs Normal file
View File

@ -0,0 +1,9 @@
namespace ParadoxSaveParser.WebAPI.SaveDataFilters;
public interface ISaveDataFilter
{
public string SearchString { get; }
public ISearchExpression SearchExpression { get; }
public void Apply(Dictionary<string, List<object>> data);
}

49
ParadoxSaveParser.WebAPI/SaveDataFilters/SaveFilterEU4.cs Normal file
View File

@ -0,0 +1,49 @@
using System.Linq;
using ParsedDict = System.Collections.Generic.Dictionary<string, System.Collections.Generic.List<object>>;
namespace ParadoxSaveParser.WebAPI.SaveDataFilters;
public class SaveDataFilterEU4 : ISaveDataFilter
{
public string SearchString { get; }
public ISearchExpression SearchExpression { get; }
public SaveDataFilterEU4()
{
SearchString =
"""
(
active_war|
previous_war|
income_statistics|
nation_size_statistics|
inflation_statistics|
countries.(
---.~|REB.~|PIR.~|NAT.~|
*.(
flags.~|hidden_flags.~|variables.~|estate.~|active_agenda.~|
power_projection.~|ai.~|history.~|navy.~|army.~|mercenary_company.~|
active_relations.~|border_pct.~|border_sit.~|border_provinces.~|
neighbours.~|home_neighbours.~|core_neighbours.~|inflation_history.~|
opinion_cache.~|owned_provinces.~|controlled_provinces.~|core_provinces.~|
claim_provinces.~|leader.~|*
)
)
)
"""
.Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", "");
SearchExpression = SearchExpressionCompiler.Compile(SearchString);
}
public void Apply(ParsedDict data)
{
var countries = (ParsedDict)data["countries"][0];
var countries_filtered = countries.Where(pair
=> pair.Value.Count > 0
&& ((ParsedDict)pair.Value[0]).TryGetValue("raw_development", out var raw_development)
&& (double)raw_development[0] > 0
).ToDictionary();
data["countries"][0] = countries_filtered;
}
}

0
ParadoxSaveParser.WebAPI/data_filtering.txt → ParadoxSaveParser.WebAPI/SaveDataFilters/data_filtering_EU4.txt
View File