changed response logic

ParadoxSaveParser.WebAPI/Program.HttpHelpers.cs

@@ -0,0 +1,97 @@
+using System.Linq;
+using System.Net;
+using System.Text.Encodings.Web;
+using System.Text.Json.Serialization;
+using DTLib.Extensions;
+
+namespace ParadoxSaveParser.WebAPI;
+
+public partial class Program
+{
+    private static readonly JsonSerializerOptions _responseJsonSerializerOptions = new()
+    {
+        WriteIndented = false,
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        MaxDepth = 1024
+    };
+    
+    public static async Task<HttpStatusCode> ReturnResponseString(HttpListenerContext ctx,
+        string value, HttpStatusCode statusCode = HttpStatusCode.OK)
+    {
+        ctx.Response.StatusCode = (int)statusCode;
+        ctx.Response.ContentType = "text/plain";
+        await ctx.Response.OutputStream.WriteAsync(
+            value.ToBytes(),
+            _mainCancel.Token);
+        return statusCode;
+    }
+
+    public static async Task<HttpStatusCode> ReturnResponseJson(HttpListenerContext ctx,
+        object value, HttpStatusCode statusCode = HttpStatusCode.OK)
+    {
+        ctx.Response.StatusCode = (int)statusCode;
+        ctx.Response.ContentType = "application/json";
+        await JsonSerializer.SerializeAsync(
+            ctx.Response.OutputStream,
+            value,
+            value.GetType(),
+            _responseJsonSerializerOptions,
+            _mainCancel.Token);
+        return statusCode;
+    }
+
+    public static async Task<HttpStatusCode> ReturnResponseError(HttpListenerContext ctx, ErrorMessage error)
+    {
+        ctx.Response.StatusCode = (int)error.StatusCode;
+        ctx.Response.ContentType = "application/json";
+        await JsonSerializer.SerializeAsync(
+            ctx.Response.OutputStream,
+            error,
+            typeof(ErrorMessage),
+            _responseJsonSerializerOptions,
+            _mainCancel.Token);
+        return error.StatusCode;
+    }
+
+    public record ErrorMessage
+    {
+        public ErrorMessage(HttpStatusCode statusCode, string message)
+        {
+            StatusCode = statusCode;
+            Message = message;
+        }
+
+        [JsonIgnore] public HttpStatusCode StatusCode { get; }
+
+        [JsonPropertyName("errorMessage")] public string Message { get; }
+    }
+
+    public class ValueOrError<T>
+    {
+        public readonly ErrorMessage? Error;
+        public readonly T? Value;
+
+        private ValueOrError(T? value, ErrorMessage? error)
+        {
+            Value = value;
+            Error = error;
+        }
+
+        public bool HasError => Error is not null;
+
+        public static implicit operator ValueOrError<T>(T v) => new(v, null);
+
+        public static implicit operator ValueOrError<T>(ErrorMessage e) => new(default, e);
+    }
+    
+    
+    internal static ValueOrError<string> GetRequestQueryValue(HttpListenerContext ctx, string paramName)
+    {
+        string[]? values = ctx.Request.QueryString.GetValues(paramName);
+        string? value = values?.FirstOrDefault();
+        if (string.IsNullOrEmpty(value))
+            return new ErrorMessage(HttpStatusCode.BadRequest,
+                $"No request parameter '{paramName}' provided");
+        return value;
+    }
+}

ParadoxSaveParser.WebAPI/Program.RequestHandlers.cs

@@ -0,0 +1,117 @@
+using System.IO.Compression;
+using System.Linq;
+using System.Net;
+
+namespace ParadoxSaveParser.WebAPI;
+
+public partial class Program
+{
+    private static async Task<HttpStatusCode> UploadSaveHandler(HttpListenerContext ctx)
+    {
+        string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault();
+        if (contentType != "application/octet-stream")
+            return await ReturnResponseError(ctx, new ErrorMessage(
+                HttpStatusCode.BadRequest,
+                $"Invalid request Content-Type: '{contentType}'"));
+
+        string saveId = Guid.NewGuid().ToString();
+        var metaFilePath = PathHelper.GetMetaFilePath(saveId);
+        if (File.Exists(metaFilePath))
+            return await ReturnResponseError(ctx, new ErrorMessage(
+                HttpStatusCode.InternalServerError,
+                $"Guid collision! file' {metaFilePath}' already exists."));
+
+        var meta = new SaveFileMetadata
+            { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized };
+        if (!_saveMetadataStorage.TryAdd(saveId, meta))
+            return await ReturnResponseError(ctx, new ErrorMessage(
+                HttpStatusCode.InternalServerError,
+                $"Guid collision! Can't create metadata with id {saveId}"));
+
+        meta.status = SaveFileProcessingStatus.Uploading;
+        var saveFilePath = PathHelper.GetSaveFilePath(meta.id);
+        await using var saveFile = File.OpenWrite(saveFilePath);
+        await using var remoteStream = ctx.Request.InputStream;
+        await remoteStream.CopyToAsync(saveFile, _mainCancel.Token);
+        meta.status = SaveFileProcessingStatus.Uploaded;
+
+        return await ReturnResponseString(ctx, saveId);
+    }
+
+    private static ValueOrError<SaveFileMetadata> GetMetaFromRequestId(HttpListenerContext ctx,
+        string requestParamName)
+    {
+        var idOrError = GetRequestQueryValue(ctx, requestParamName);
+        if (idOrError.HasError)
+            return idOrError.Error!;
+
+        if (!_saveMetadataStorage.TryGetValue(idOrError.Value!, out var meta))
+            return new ErrorMessage(HttpStatusCode.InternalServerError,
+                $"Save with id {idOrError.Value} not found");
+
+        return meta;
+    }
+
+    private static async Task<HttpStatusCode> GetSaveStatusHandler(HttpListenerContext ctx)
+    {
+        var metaOrError = GetMetaFromRequestId(ctx, "id");
+        if (metaOrError.HasError)
+            return await ReturnResponseError(ctx, metaOrError.Error!);
+
+        return await ReturnResponseJson(ctx, metaOrError.Value!);
+    }
+
+    private static async Task<HttpStatusCode> ParseSaveEU4Handler(HttpListenerContext ctx)
+    {
+        var metaOrError = GetMetaFromRequestId(ctx, "id");
+        if (metaOrError.HasError)
+            return await ReturnResponseError(ctx, metaOrError.Error!);
+        var meta = metaOrError.Value!;
+
+        var searchQueryOrError = GetRequestQueryValue(ctx, "search");
+        if (searchQueryOrError.HasError)
+            return await ReturnResponseError(ctx, searchQueryOrError.Error!);
+        string searchQuery = searchQueryOrError.Value!;
+
+        try
+        {
+            string extractedGamestatePath = PathHelper.GetSaveFilePath(meta.id) + ".gamestate";
+            using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(meta.id).Str, ZipArchiveMode.Read))
+            {
+                var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
+                if (zipEntry is null)
+                    return await ReturnResponseError(ctx, new ErrorMessage(
+                        HttpStatusCode.BadRequest,
+                        "Invalid save format: no 'gamestate' file found"));
+
+                zipEntry.ExtractToFile(extractedGamestatePath, true);
+            }
+
+            var gamestateStream = File.OpenRead(extractedGamestatePath);
+
+            meta.status = SaveFileProcessingStatus.Parsing;
+            var se = SearchExpressionCompiler.Compile(searchQuery);
+            var parser = new SaveParserEU4(gamestateStream, se);
+            var result = parser.Parse();
+
+            meta.status = SaveFileProcessingStatus.SavingResults;
+            var resultFilePath = PathHelper.GetParsedSaveFilePath(meta.id);
+            await using var resultFile = File.OpenWrite(resultFilePath);
+            await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions, _mainCancel.Token);
+            meta.status = SaveFileProcessingStatus.Done;
+            meta.SaveToFile();
+        }
+        catch (Exception ex)
+        {
+            string errorMesage = ex.ToStringDemystified();
+            _loggerRoot.LogWarn(nameof(ParseSaveEU4Handler), errorMesage);
+            return await ReturnResponseError(ctx, new ErrorMessage(HttpStatusCode.BadRequest, errorMesage));
+        }
+        finally
+        {
+            GC.Collect();
+        }
+
+        return await ReturnResponseJson(ctx, meta);
+    }
+}

ParadoxSaveParser.WebAPI/Program.Responses.cs

@@ -1,125 +0,0 @@
-using System.IO.Compression;
-using System.Linq;
-using System.Net;
-using DTLib.Extensions;
-
-namespace ParadoxSaveParser.WebAPI;
-
-public partial class Program
-{
-    // ReSharper disable once NotAccessedPositionalProperty.Global
-    public record ErrorMessage(string errorMessage);
-
-    private static async Task<HttpStatusCode> ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode,
-        object response)
-    {
-        await JsonSerializer.SerializeAsync(ctx.Response.OutputStream, response, response.GetType(),
-            JsonSerializerOptions.Default, _mainCancel.Token);
-        ctx.Response.StatusCode = (int)statusCode;
-        return statusCode;
-    }
-
-    private static async Task<HttpStatusCode> ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode,
-        string response)
-    {
-        await ctx.Response.OutputStream.WriteAsync(response.ToBytes(), _mainCancel.Token);
-        ctx.Response.StatusCode = (int)statusCode;
-        return statusCode;
-    }
-
-    private static async Task<HttpStatusCode> UploadSaveHandler(HttpListenerContext ctx)
-    {
-        string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault();
-        if (contentType != "application/octet-stream")
-            return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
-                new ErrorMessage($"Invalid request Content-Type: '{contentType}'"));
-
-        string saveId = Guid.NewGuid().ToString();
-        IOPath metaFilePath = PathHelper.GetMetaFilePath(saveId);
-        if (File.Exists(metaFilePath))
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError,
-                new ErrorMessage($"Guid collision! file' {metaFilePath}' already exists."));
-
-        var meta = new SaveFileMetadata
-            { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized, };
-        if (!_saveMetadataStorage.TryAdd(saveId, meta))
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError,
-                new ErrorMessage($"Guid collision! Can't create metadata with id {saveId}"));
-
-        meta.status = SaveFileProcessingStatus.Uploading;
-        IOPath saveFilePath = PathHelper.GetSaveFilePath(meta.id);
-        await using var saveFile = File.OpenWrite(saveFilePath);
-        await using var remoteStream = ctx.Request.InputStream;
-        await remoteStream.CopyToAsync(saveFile, _mainCancel.Token);
-        meta.status = SaveFileProcessingStatus.Uploaded;
-
-        return await ReturnResponse(ctx, HttpStatusCode.OK, saveId);
-    }
-
-    private static (SaveFileMetadata? meta, ErrorMessage? errorMesage) GetMetaFromRequestId(HttpListenerContext ctx,
-        string requestParamName)
-    {
-        var ids = ctx.Request.QueryString.GetValues(requestParamName);
-        string? id = ids?.FirstOrDefault();
-        if (string.IsNullOrEmpty(id))
-            return (null, new ErrorMessage($"No request parameter '{requestParamName}' provided"));
-
-        if (!_saveMetadataStorage.TryGetValue(id, out var meta))
-            return (null, new ErrorMessage($"Save with {id} not found"));
-
-        return (meta, null);
-    }
-
-    private static async Task<HttpStatusCode> GetSaveStatusHandler(HttpListenerContext ctx)
-    {
-        var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id");
-        if (errorMessage is not null)
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage);
-
-        return await ReturnResponse(ctx, HttpStatusCode.OK, meta!);
-    }
-
-    private static async Task<HttpStatusCode> ParseSaveEU4Handler(HttpListenerContext ctx)
-    {
-        var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id");
-        if (errorMessage is not null)
-            return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage);
-
-        //TODO: get actual query
-        string searchQuery = "";
-        
-        try
-        {
-            using var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(meta!.id).Str, ZipArchiveMode.Read);
-            var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
-            if (zipEntry is null)
-                return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
-                    new ErrorMessage("Invalid save format: no 'gamestate' file found"));
-
-            string extractedGamestatePath = PathHelper.GetSaveFilePath(meta.id) + ".gamestate";
-            zipEntry.ExtractToFile(extractedGamestatePath, true);
-            var gamestateStream = File.OpenRead(extractedGamestatePath);
-
-            meta.status = SaveFileProcessingStatus.Parsing;
-            var se = SearchExpressionCompiler.Compile(searchQuery);
-            var parser = new SaveParserEU4(gamestateStream, se);
-            var result = parser.Parse();
-
-            meta.status = SaveFileProcessingStatus.SavingResults;
-            IOPath resultFilePath = PathHelper.GetParsedSaveFilePath(meta.id);
-            await using var resultFile = File.OpenWrite(resultFilePath);
-            await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions, _mainCancel.Token);
-            meta.status = SaveFileProcessingStatus.Done;
-            meta.SaveToFile();
-        }
-        catch (Exception ex)
-        {
-            string errorMesage = ex.ToStringDemystified();
-            _loggerRoot.LogWarn(nameof(ParseSaveEU4Handler), errorMesage);
-            return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
-                new ErrorMessage(errorMesage));
-        }
-
-        return await ReturnResponse(ctx, HttpStatusCode.OK, meta);
-    }
-}

ParadoxSaveParser.WebAPI/Program.cs

@@ -20,7 +20,7 @@ using DTLib.Web.Routes;
 
 namespace ParadoxSaveParser.WebAPI;
 
-public partial class Program
+public static partial class Program
 {
     private static readonly IOPath _configPath = "./config.dtsod";
     private static Config _config = new();
@@ -28,23 +28,23 @@ public partial class Program
     private static readonly ILogger _loggerRoot = new CompositeLogger(
         new ConsoleLogger(),
         new FileLogger("logs", "ParadoxSaveParser.WebAPI"));
-    
+
     private static readonly CancellationTokenSource _mainCancel = new();
-    private static ConcurrentDictionary<string, SaveFileMetadata> _saveMetadataStorage = new();
+    private static readonly ConcurrentDictionary<string, SaveFileMetadata> _saveMetadataStorage = new();
-    
+
-    private static JsonSerializerOptions _saveSerializerOptions = new()
+    private static readonly JsonSerializerOptions _saveSerializerOptions = new()
     {
-        WriteIndented = false,
+        WriteIndented = true,
         Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
-        MaxDepth = 1024,
+        MaxDepth = 1024
     };
-    
+
     public static void Main(string[] args)
     {
         Console.InputEncoding = Encoding.UTF8;
         Console.OutputEncoding = Encoding.UTF8;
         Console.CursorVisible = false;
-        ContextLogger logger = new ContextLogger(nameof(Main), _loggerRoot);
+        var logger = new ContextLogger(nameof(Main), _loggerRoot);
         Console.CancelKeyPress += (_, e) =>
         {
             e.Cancel = true;
@@ -61,7 +61,10 @@ public partial class Program
                 File.WriteAllText(_configPath, _config.ToString());
                 logger.LogWarn($"created default at {_configPath}.");
             }
-            else _config = Config.FromDtsod(new DtsodV23(File.ReadAllText(_configPath)));
+            else
+            {
+                _config = Config.FromDtsod(new DtsodV23(File.ReadAllText(_configPath)));
+            }
 
             PrepareLocalFiles();
 
@@ -89,21 +92,19 @@ public partial class Program
     {
         Directory.Create(PathHelper.DATA_DIR);
         Directory.Create(PathHelper.SAVES_DIR);
-        foreach (var metaFilePath in System.IO.Directory.GetFiles(
+        foreach (string metaFilePath in System.IO.Directory.GetFiles(
                      PathHelper.SAVES_DIR.Str, "*.meta.json",
                      SearchOption.TopDirectoryOnly))
         {
             using var metaFile = File.OpenRead(metaFilePath);
-            var meta = JsonSerializer.Deserialize<SaveFileMetadata>(metaFile) ?? 
+            var meta = JsonSerializer.Deserialize<SaveFileMetadata>(metaFile) ??
                        throw new NullReferenceException(metaFilePath);
             if (meta.status != SaveFileProcessingStatus.Done)
-            {
+                _loggerRoot.LogWarn(nameof(PrepareLocalFiles),
-                _loggerRoot.LogWarn(nameof(PrepareLocalFiles), $"metadata file '{metaFilePath}' status has invalid status {meta.status}");
+                    $"metadata file '{metaFilePath}' status has invalid status {meta.status}");
-            }
 
-            if(!_saveMetadataStorage.TryAdd(meta.id, meta))
+            if (!_saveMetadataStorage.TryAdd(meta.id, meta))
                 throw new Exception("Guid collision!");
         }
     }
-    
 }

TODO.txt

@@ -4,6 +4,8 @@ Main:
 ParseSaveHandler:
 	Make this method run as background task instead of POST query
 	Add debug log
+	Save parsed in protobuf
+	Re-parse if saved data was parsed with another query
 	
 Parser:
-	Add query support to parse only needed information
+	Optimize it (5 sec per query isn't good)