From 758388cda01dd00ac7c569b501e568bad80085ee Mon Sep 17 00:00:00 2001 From: Timerix Date: Sat, 5 Apr 2025 05:54:15 +0500 Subject: [PATCH] changed response logic --- .../Program.HttpHelpers.cs | 97 ++++++++++++++ .../Program.RequestHandlers.cs | 117 ++++++++++++++++ ParadoxSaveParser.WebAPI/Program.Responses.cs | 125 ------------------ ParadoxSaveParser.WebAPI/Program.cs | 35 ++--- TODO.txt | 4 +- 5 files changed, 235 insertions(+), 143 deletions(-) create mode 100644 ParadoxSaveParser.WebAPI/Program.HttpHelpers.cs create mode 100644 ParadoxSaveParser.WebAPI/Program.RequestHandlers.cs delete mode 100644 ParadoxSaveParser.WebAPI/Program.Responses.cs diff --git a/ParadoxSaveParser.WebAPI/Program.HttpHelpers.cs b/ParadoxSaveParser.WebAPI/Program.HttpHelpers.cs new file mode 100644 index 0000000..ed0132c --- /dev/null +++ b/ParadoxSaveParser.WebAPI/Program.HttpHelpers.cs @@ -0,0 +1,97 @@ +using System.Linq; +using System.Net; +using System.Text.Encodings.Web; +using System.Text.Json.Serialization; +using DTLib.Extensions; + +namespace ParadoxSaveParser.WebAPI; + +public partial class Program +{ + private static readonly JsonSerializerOptions _responseJsonSerializerOptions = new() + { + WriteIndented = false, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + MaxDepth = 1024 + }; + + public static async Task ReturnResponseString(HttpListenerContext ctx, + string value, HttpStatusCode statusCode = HttpStatusCode.OK) + { + ctx.Response.StatusCode = (int)statusCode; + ctx.Response.ContentType = "text/plain"; + await ctx.Response.OutputStream.WriteAsync( + value.ToBytes(), + _mainCancel.Token); + return statusCode; + } + + public static async Task ReturnResponseJson(HttpListenerContext ctx, + object value, HttpStatusCode statusCode = HttpStatusCode.OK) + { + ctx.Response.StatusCode = (int)statusCode; + ctx.Response.ContentType = "application/json"; + await JsonSerializer.SerializeAsync( + ctx.Response.OutputStream, + value, + value.GetType(), + _responseJsonSerializerOptions, + _mainCancel.Token); + return statusCode; + } + + public static async Task ReturnResponseError(HttpListenerContext ctx, ErrorMessage error) + { + ctx.Response.StatusCode = (int)error.StatusCode; + ctx.Response.ContentType = "application/json"; + await JsonSerializer.SerializeAsync( + ctx.Response.OutputStream, + error, + typeof(ErrorMessage), + _responseJsonSerializerOptions, + _mainCancel.Token); + return error.StatusCode; + } + + public record ErrorMessage + { + public ErrorMessage(HttpStatusCode statusCode, string message) + { + StatusCode = statusCode; + Message = message; + } + + [JsonIgnore] public HttpStatusCode StatusCode { get; } + + [JsonPropertyName("errorMessage")] public string Message { get; } + } + + public class ValueOrError + { + public readonly ErrorMessage? Error; + public readonly T? Value; + + private ValueOrError(T? value, ErrorMessage? error) + { + Value = value; + Error = error; + } + + public bool HasError => Error is not null; + + public static implicit operator ValueOrError(T v) => new(v, null); + + public static implicit operator ValueOrError(ErrorMessage e) => new(default, e); + } + + + internal static ValueOrError GetRequestQueryValue(HttpListenerContext ctx, string paramName) + { + string[]? values = ctx.Request.QueryString.GetValues(paramName); + string? value = values?.FirstOrDefault(); + if (string.IsNullOrEmpty(value)) + return new ErrorMessage(HttpStatusCode.BadRequest, + $"No request parameter '{paramName}' provided"); + return value; + } +} \ No newline at end of file diff --git a/ParadoxSaveParser.WebAPI/Program.RequestHandlers.cs b/ParadoxSaveParser.WebAPI/Program.RequestHandlers.cs new file mode 100644 index 0000000..b5e5a27 --- /dev/null +++ b/ParadoxSaveParser.WebAPI/Program.RequestHandlers.cs @@ -0,0 +1,117 @@ +using System.IO.Compression; +using System.Linq; +using System.Net; + +namespace ParadoxSaveParser.WebAPI; + +public partial class Program +{ + private static async Task UploadSaveHandler(HttpListenerContext ctx) + { + string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault(); + if (contentType != "application/octet-stream") + return await ReturnResponseError(ctx, new ErrorMessage( + HttpStatusCode.BadRequest, + $"Invalid request Content-Type: '{contentType}'")); + + string saveId = Guid.NewGuid().ToString(); + var metaFilePath = PathHelper.GetMetaFilePath(saveId); + if (File.Exists(metaFilePath)) + return await ReturnResponseError(ctx, new ErrorMessage( + HttpStatusCode.InternalServerError, + $"Guid collision! file' {metaFilePath}' already exists.")); + + var meta = new SaveFileMetadata + { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized }; + if (!_saveMetadataStorage.TryAdd(saveId, meta)) + return await ReturnResponseError(ctx, new ErrorMessage( + HttpStatusCode.InternalServerError, + $"Guid collision! Can't create metadata with id {saveId}")); + + meta.status = SaveFileProcessingStatus.Uploading; + var saveFilePath = PathHelper.GetSaveFilePath(meta.id); + await using var saveFile = File.OpenWrite(saveFilePath); + await using var remoteStream = ctx.Request.InputStream; + await remoteStream.CopyToAsync(saveFile, _mainCancel.Token); + meta.status = SaveFileProcessingStatus.Uploaded; + + return await ReturnResponseString(ctx, saveId); + } + + private static ValueOrError GetMetaFromRequestId(HttpListenerContext ctx, + string requestParamName) + { + var idOrError = GetRequestQueryValue(ctx, requestParamName); + if (idOrError.HasError) + return idOrError.Error!; + + if (!_saveMetadataStorage.TryGetValue(idOrError.Value!, out var meta)) + return new ErrorMessage(HttpStatusCode.InternalServerError, + $"Save with id {idOrError.Value} not found"); + + return meta; + } + + private static async Task GetSaveStatusHandler(HttpListenerContext ctx) + { + var metaOrError = GetMetaFromRequestId(ctx, "id"); + if (metaOrError.HasError) + return await ReturnResponseError(ctx, metaOrError.Error!); + + return await ReturnResponseJson(ctx, metaOrError.Value!); + } + + private static async Task ParseSaveEU4Handler(HttpListenerContext ctx) + { + var metaOrError = GetMetaFromRequestId(ctx, "id"); + if (metaOrError.HasError) + return await ReturnResponseError(ctx, metaOrError.Error!); + var meta = metaOrError.Value!; + + var searchQueryOrError = GetRequestQueryValue(ctx, "search"); + if (searchQueryOrError.HasError) + return await ReturnResponseError(ctx, searchQueryOrError.Error!); + string searchQuery = searchQueryOrError.Value!; + + try + { + string extractedGamestatePath = PathHelper.GetSaveFilePath(meta.id) + ".gamestate"; + using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(meta.id).Str, ZipArchiveMode.Read)) + { + var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate"); + if (zipEntry is null) + return await ReturnResponseError(ctx, new ErrorMessage( + HttpStatusCode.BadRequest, + "Invalid save format: no 'gamestate' file found")); + + zipEntry.ExtractToFile(extractedGamestatePath, true); + } + + var gamestateStream = File.OpenRead(extractedGamestatePath); + + meta.status = SaveFileProcessingStatus.Parsing; + var se = SearchExpressionCompiler.Compile(searchQuery); + var parser = new SaveParserEU4(gamestateStream, se); + var result = parser.Parse(); + + meta.status = SaveFileProcessingStatus.SavingResults; + var resultFilePath = PathHelper.GetParsedSaveFilePath(meta.id); + await using var resultFile = File.OpenWrite(resultFilePath); + await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions, _mainCancel.Token); + meta.status = SaveFileProcessingStatus.Done; + meta.SaveToFile(); + } + catch (Exception ex) + { + string errorMesage = ex.ToStringDemystified(); + _loggerRoot.LogWarn(nameof(ParseSaveEU4Handler), errorMesage); + return await ReturnResponseError(ctx, new ErrorMessage(HttpStatusCode.BadRequest, errorMesage)); + } + finally + { + GC.Collect(); + } + + return await ReturnResponseJson(ctx, meta); + } +} \ No newline at end of file diff --git a/ParadoxSaveParser.WebAPI/Program.Responses.cs b/ParadoxSaveParser.WebAPI/Program.Responses.cs deleted file mode 100644 index ea8b6a6..0000000 --- a/ParadoxSaveParser.WebAPI/Program.Responses.cs +++ /dev/null @@ -1,125 +0,0 @@ -using System.IO.Compression; -using System.Linq; -using System.Net; -using DTLib.Extensions; - -namespace ParadoxSaveParser.WebAPI; - -public partial class Program -{ - // ReSharper disable once NotAccessedPositionalProperty.Global - public record ErrorMessage(string errorMessage); - - private static async Task ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode, - object response) - { - await JsonSerializer.SerializeAsync(ctx.Response.OutputStream, response, response.GetType(), - JsonSerializerOptions.Default, _mainCancel.Token); - ctx.Response.StatusCode = (int)statusCode; - return statusCode; - } - - private static async Task ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode, - string response) - { - await ctx.Response.OutputStream.WriteAsync(response.ToBytes(), _mainCancel.Token); - ctx.Response.StatusCode = (int)statusCode; - return statusCode; - } - - private static async Task UploadSaveHandler(HttpListenerContext ctx) - { - string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault(); - if (contentType != "application/octet-stream") - return await ReturnResponse(ctx, HttpStatusCode.BadRequest, - new ErrorMessage($"Invalid request Content-Type: '{contentType}'")); - - string saveId = Guid.NewGuid().ToString(); - IOPath metaFilePath = PathHelper.GetMetaFilePath(saveId); - if (File.Exists(metaFilePath)) - return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, - new ErrorMessage($"Guid collision! file' {metaFilePath}' already exists.")); - - var meta = new SaveFileMetadata - { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized, }; - if (!_saveMetadataStorage.TryAdd(saveId, meta)) - return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, - new ErrorMessage($"Guid collision! Can't create metadata with id {saveId}")); - - meta.status = SaveFileProcessingStatus.Uploading; - IOPath saveFilePath = PathHelper.GetSaveFilePath(meta.id); - await using var saveFile = File.OpenWrite(saveFilePath); - await using var remoteStream = ctx.Request.InputStream; - await remoteStream.CopyToAsync(saveFile, _mainCancel.Token); - meta.status = SaveFileProcessingStatus.Uploaded; - - return await ReturnResponse(ctx, HttpStatusCode.OK, saveId); - } - - private static (SaveFileMetadata? meta, ErrorMessage? errorMesage) GetMetaFromRequestId(HttpListenerContext ctx, - string requestParamName) - { - var ids = ctx.Request.QueryString.GetValues(requestParamName); - string? id = ids?.FirstOrDefault(); - if (string.IsNullOrEmpty(id)) - return (null, new ErrorMessage($"No request parameter '{requestParamName}' provided")); - - if (!_saveMetadataStorage.TryGetValue(id, out var meta)) - return (null, new ErrorMessage($"Save with {id} not found")); - - return (meta, null); - } - - private static async Task GetSaveStatusHandler(HttpListenerContext ctx) - { - var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id"); - if (errorMessage is not null) - return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage); - - return await ReturnResponse(ctx, HttpStatusCode.OK, meta!); - } - - private static async Task ParseSaveEU4Handler(HttpListenerContext ctx) - { - var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id"); - if (errorMessage is not null) - return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage); - - //TODO: get actual query - string searchQuery = ""; - - try - { - using var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(meta!.id).Str, ZipArchiveMode.Read); - var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate"); - if (zipEntry is null) - return await ReturnResponse(ctx, HttpStatusCode.BadRequest, - new ErrorMessage("Invalid save format: no 'gamestate' file found")); - - string extractedGamestatePath = PathHelper.GetSaveFilePath(meta.id) + ".gamestate"; - zipEntry.ExtractToFile(extractedGamestatePath, true); - var gamestateStream = File.OpenRead(extractedGamestatePath); - - meta.status = SaveFileProcessingStatus.Parsing; - var se = SearchExpressionCompiler.Compile(searchQuery); - var parser = new SaveParserEU4(gamestateStream, se); - var result = parser.Parse(); - - meta.status = SaveFileProcessingStatus.SavingResults; - IOPath resultFilePath = PathHelper.GetParsedSaveFilePath(meta.id); - await using var resultFile = File.OpenWrite(resultFilePath); - await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions, _mainCancel.Token); - meta.status = SaveFileProcessingStatus.Done; - meta.SaveToFile(); - } - catch (Exception ex) - { - string errorMesage = ex.ToStringDemystified(); - _loggerRoot.LogWarn(nameof(ParseSaveEU4Handler), errorMesage); - return await ReturnResponse(ctx, HttpStatusCode.BadRequest, - new ErrorMessage(errorMesage)); - } - - return await ReturnResponse(ctx, HttpStatusCode.OK, meta); - } -} \ No newline at end of file diff --git a/ParadoxSaveParser.WebAPI/Program.cs b/ParadoxSaveParser.WebAPI/Program.cs index 72aaa2e..ace24e0 100644 --- a/ParadoxSaveParser.WebAPI/Program.cs +++ b/ParadoxSaveParser.WebAPI/Program.cs @@ -20,7 +20,7 @@ using DTLib.Web.Routes; namespace ParadoxSaveParser.WebAPI; -public partial class Program +public static partial class Program { private static readonly IOPath _configPath = "./config.dtsod"; private static Config _config = new(); @@ -28,23 +28,23 @@ public partial class Program private static readonly ILogger _loggerRoot = new CompositeLogger( new ConsoleLogger(), new FileLogger("logs", "ParadoxSaveParser.WebAPI")); - + private static readonly CancellationTokenSource _mainCancel = new(); - private static ConcurrentDictionary _saveMetadataStorage = new(); - - private static JsonSerializerOptions _saveSerializerOptions = new() + private static readonly ConcurrentDictionary _saveMetadataStorage = new(); + + private static readonly JsonSerializerOptions _saveSerializerOptions = new() { - WriteIndented = false, + WriteIndented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - MaxDepth = 1024, + MaxDepth = 1024 }; - + public static void Main(string[] args) { Console.InputEncoding = Encoding.UTF8; Console.OutputEncoding = Encoding.UTF8; Console.CursorVisible = false; - ContextLogger logger = new ContextLogger(nameof(Main), _loggerRoot); + var logger = new ContextLogger(nameof(Main), _loggerRoot); Console.CancelKeyPress += (_, e) => { e.Cancel = true; @@ -61,7 +61,10 @@ public partial class Program File.WriteAllText(_configPath, _config.ToString()); logger.LogWarn($"created default at {_configPath}."); } - else _config = Config.FromDtsod(new DtsodV23(File.ReadAllText(_configPath))); + else + { + _config = Config.FromDtsod(new DtsodV23(File.ReadAllText(_configPath))); + } PrepareLocalFiles(); @@ -89,21 +92,19 @@ public partial class Program { Directory.Create(PathHelper.DATA_DIR); Directory.Create(PathHelper.SAVES_DIR); - foreach (var metaFilePath in System.IO.Directory.GetFiles( + foreach (string metaFilePath in System.IO.Directory.GetFiles( PathHelper.SAVES_DIR.Str, "*.meta.json", SearchOption.TopDirectoryOnly)) { using var metaFile = File.OpenRead(metaFilePath); - var meta = JsonSerializer.Deserialize(metaFile) ?? + var meta = JsonSerializer.Deserialize(metaFile) ?? throw new NullReferenceException(metaFilePath); if (meta.status != SaveFileProcessingStatus.Done) - { - _loggerRoot.LogWarn(nameof(PrepareLocalFiles), $"metadata file '{metaFilePath}' status has invalid status {meta.status}"); - } + _loggerRoot.LogWarn(nameof(PrepareLocalFiles), + $"metadata file '{metaFilePath}' status has invalid status {meta.status}"); - if(!_saveMetadataStorage.TryAdd(meta.id, meta)) + if (!_saveMetadataStorage.TryAdd(meta.id, meta)) throw new Exception("Guid collision!"); } } - } \ No newline at end of file diff --git a/TODO.txt b/TODO.txt index 8f0e74f..3166d2e 100644 --- a/TODO.txt +++ b/TODO.txt @@ -4,6 +4,8 @@ Main: ParseSaveHandler: Make this method run as background task instead of POST query Add debug log + Save parsed in protobuf + Re-parse if saved data was parsed with another query Parser: - Add query support to parse only needed information + Optimize it (5 sec per query isn't good)