ParadoxSaveParser.CLI/Modes.cs

@@ -1,9 +1,5 @@
-using System.IO;
+using System.Text.Json;
-using System.IO.Compression;
-using System.Linq;
-using System.Text.Json;
 using ParadoxSaveParser.Lib;
-using Path = DTLib.Filesystem.Path;
 
 namespace ParadoxSaveParser.CLI;
 
@@ -12,11 +8,28 @@ internal enum Mode
     Unset, Search, Interactive
 }
 
-internal static partial class Modes
+internal static class Modes
 {
+    internal static void Search(string searchQuery, IOPath inputPath, IOPath? outputPath)
+    {
+        using var inputStream = File.OpenRead(inputPath);
+        
+        using var outputStream = outputPath is null
+            ? Console.OpenStandardOutput()
+            : File.OpenWrite(outputPath.Value);
+        
+        var searchExpression = SearchExpressionCompiler.Compile(searchQuery);
+        
+        var parser = new SaveParserEU4(inputStream, searchExpression);
+        var parsedValue = parser.Parse();
+        JsonSerializer.Serialize(outputStream, parsedValue, ParsedValueJsonContext.Default.DictionaryStringListObject);
+        outputStream.WriteByte((byte)'\n');
+    }
+    
     internal static void Interactive()
     {
-        ColoredConsole.Clear();
+        Console.Clear();
+        Console.ResetColor();
         ColoredConsole.WriteTitle("interactive mode", fg: ConsoleColor.Cyan);
         ColoredConsole.WriteLine($"working directory: '{Environment.CurrentDirectory}'", ConsoleColor.Gray);
         IOPath? inputPath = null;
@@ -27,7 +40,8 @@ internal static partial class Modes
             try
             {
                 ColoredConsole.Write("> ", ConsoleColor.Blue);
-                string input = ColoredConsole.ReadLine(ConsoleColor.Gray);
+                Console.ForegroundColor = ConsoleColor.Gray;
+                string? input = Console.ReadLine();
                 if (string.IsNullOrEmpty(input))
                     continue;
 
@@ -67,13 +81,13 @@ internal static partial class Modes
 
                     case "h":
                     case "help":
-                        ColoredConsole.WriteLine(helpMessage, ConsoleColor.White);
+                        ColoredConsole.WriteLine(helpMessage, fg: ConsoleColor.White);
                         break;
 
                     case "i":
                     case "input":
-                        ColoredConsole.Write("Input file path: ", ConsoleColor.Blue);
+                        input = ColoredConsole.ReadLine("Input file path",
-                        input = ColoredConsole.ReadLine(ConsoleColor.Gray);
+                            ConsoleColor.Blue);
                         if (string.IsNullOrEmpty(input))
                             throw new NullReferenceException();
                         inputPath = input;
@@ -81,8 +95,8 @@ internal static partial class Modes
 
                     case "o":
                     case "output":
-                        ColoredConsole.Write("Output file path [default=stdout]: ", ConsoleColor.Blue);
+                        input = ColoredConsole.ReadLine("Output file path [default=stdout]",
-                        input = ColoredConsole.ReadLine(ConsoleColor.Gray);
+                            ConsoleColor.Blue);
                         if (string.IsNullOrEmpty(input))
                             throw new ArgumentException("Input file path is required");
                         inputPath = input;
@@ -93,8 +107,8 @@ internal static partial class Modes
                         if (inputPath is null)
                             throw new ArgumentException("Input file path is required");
 
-                        ColoredConsole.Write("search expression: ", ConsoleColor.Blue);
+                        var searchQuery = ColoredConsole.ReadLine("search expression",
-                        var searchQuery = ColoredConsole.ReadLine(ConsoleColor.Gray);
+                            ConsoleColor.Blue);
                         if (string.IsNullOrEmpty(searchQuery))
                             throw new ArgumentException("Search expression is required");
 
@@ -109,8 +123,8 @@ internal static partial class Modes
                         break;
 
                     case "cd":
-                        ColoredConsole.Write("Change working directory to: ", ConsoleColor.Blue);
+                        input = ColoredConsole.ReadLine("Change working directory to",
-                        input = ColoredConsole.ReadLine(ConsoleColor.Gray);
+                            ConsoleColor.Blue);
                         if (!string.IsNullOrEmpty(input))
                         {
                             Environment.CurrentDirectory = new IOPath(input).Str;

ParadoxSaveParser.CLI/Modes/Search.cs

@@ -1,49 +0,0 @@
-using System.IO;
-using System.IO.Compression;
-using System.Linq;
-using System.Text.Json;
-using ParadoxSaveParser.Lib;
-
-namespace ParadoxSaveParser.CLI;
-
-internal static partial class Modes
-{
-    internal static void Search(string searchQuery, IOPath inputPath, IOPath? outputPath)
-    {
-        Stream inputStream = File.OpenRead(inputPath);
-        try
-        {
-            byte[] head4 = new byte[4];
-            inputStream.ReadExactly(head4);
-            inputStream.Seek(0, SeekOrigin.Begin);
-            if (head4.SequenceEqual<byte>([(byte)'P', (byte)'K', 3, 4]))
-            {
-                var zipArchive = new ZipArchive(inputStream, ZipArchiveMode.Read);
-                var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
-                if (zipEntry is null)
-                    throw new Exception("'gamestate' file not found in zip archive");
-                var unzipped = new MemoryStream((int)zipEntry.Length);
-                zipEntry.Open().CopyTo(unzipped);
-                zipArchive.Dispose(); // closes inputStream
-                unzipped.Seek(0, SeekOrigin.Begin);
-                inputStream = unzipped;
-            }
-
-            using var outputStream = outputPath is null
-                ? Console.OpenStandardOutput()
-                : File.OpenWrite(outputPath.Value);
-
-            var searchExpression = SearchExpressionCompiler.Compile(searchQuery);
-
-            var parser = new SaveParserEU4(inputStream, searchExpression);
-            var parsedValue = parser.Parse();
-            JsonSerializer.Serialize(outputStream, parsedValue,
-                ParsedValueJsonContext.Default.DictionaryStringListObject);
-            outputStream.WriteByte((byte)'\n');
-        }
-        finally
-        {
-            inputStream.Dispose();
-        }
-    }
-}

ParadoxSaveParser.CLI/ParadoxSaveParser.CLI.csproj

@@ -14,6 +14,6 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="DTLib" Version="1.7.1"/>
+        <PackageReference Include="DTLib" Version="1.6.5"/>
     </ItemGroup>
 </Project>

ParadoxSaveParser.CLI/ParadoxSaveParser.CLI.csproj.DotSettings

@@ -1,2 +0,0 @@
-<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
-	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=modes/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

ParadoxSaveParser.CLI/Program.cs

@@ -33,7 +33,7 @@ try
             },
             "search expression")
         )
-        .AllowNoArguments()
+        .WithNoExit()
         .ParseAndHandle(args);
 
     if (args.Length == 0)
@@ -57,10 +57,6 @@ try
             break;
     }
 }
-catch (LaunchArgumentParser.ExitAfterHelpException)
-{
-    // this exception is throwed after -h argument to close the program
-}
 catch (Exception ex)
 {
     ColoredConsole.WriteLine(ex.ToStringDemystified(), ConsoleColor.Red);

ParadoxSaveParser.Lib.Tests/ParadoxSaveParser.Lib.Tests.csproj

@@ -23,7 +23,7 @@
           <PrivateAssets>all</PrivateAssets>
           <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
-        <PackageReference Include="DTLib" Version="1.7.1"/>
+        <PackageReference Include="DTLib" Version="1.6.5"/>
         <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
         <PackageReference Include="NUnit" Version="4.3.2" />
         <PackageReference Include="NUnit.Analyzers" Version="4.7.0">

ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs

@@ -1,7 +1,6 @@
 using System.IO;
 using System.Text.Encodings.Web;
 using System.Text.Json;
-using System.Text.Json.Serialization;
 using DTLib.Extensions;
 
 namespace ParadoxSaveParser.Lib.Tests;
@@ -23,8 +22,7 @@ public class SearchExpressionTests
     {
         WriteIndented = false,
         Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
-        MaxDepth = 1024,
+        MaxDepth = 1024
-        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
     };
 
     internal static string JsonToPdx(string json)

ParadoxSaveParser.WebAPI/BackgroundTasks/BackgroundJobManager.cs

@@ -1,22 +0,0 @@
-namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
-
-public class BackgroundJobManager
-{
-    private readonly ILogger _parentLogger;
-    private long _lastJobId;
-
-    public BackgroundJobManager(ILogger logger)
-    {
-        _parentLogger = logger;
-    }
-    
-    public SaveParsingOperation StartNewParsingOperation(
-        SaveFileMetadata meta, ISearchExpression searchQuery, CancellationToken ct)
-    {
-        long nextId = Interlocked.Increment(ref _lastJobId);
-        var contextLogger = new ContextLogger($"BackgroundJob-{nextId}", _parentLogger);
-        var op = new SaveParsingOperation(nextId, meta, searchQuery, contextLogger, ct);
-        op.StartAsync();
-        return op;
-    }
-}

ParadoxSaveParser.WebAPI/BackgroundTasks/SaveParsingOperation.cs

@@ -1,90 +0,0 @@
-using System.IO.Compression;
-using System.Linq;
-using System.Text.Encodings.Web;
-using System.Text.Json.Serialization;
-
-namespace ParadoxSaveParser.WebAPI.BackgroundTasks;
-
-public class SaveParsingOperation
-{
-    public readonly long OperationId;
-    public readonly SaveFileMetadata Meta;
-    public readonly ISearchExpression SearchQuery;
-    
-    private readonly ContextLogger _logger;
-    private readonly CancellationToken _cancelToken;
-
-    private static readonly JsonSerializerOptions _saveSerializerOptions = new()
-    {
-        WriteIndented = false,
-        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
-        MaxDepth = 1024,
-        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
-    };
-
-    public SaveParsingOperation(long operationId, SaveFileMetadata meta, ISearchExpression searchQuery,
-        ContextLogger logger, CancellationToken cancelToken)
-    {
-        OperationId = operationId;
-        Meta = meta;
-        SearchQuery = searchQuery;
-        _logger = logger;
-        _cancelToken = cancelToken;
-    }
-
-    public async void StartAsync()
-    {
-        try
-        {
-            _logger.LogInfo($"Starting background parsing operation of {Meta.game} save {Meta.id}");
-            switch (Meta.game)
-            {
-                case Game.EU4:
-                    await ParseSaveEU4();
-                    break;
-                default:
-                    throw new ArgumentOutOfRangeException(Meta.game.ToString());
-            }
-            _logger.LogInfo($"Finished parsing operation of {Meta.game} save {Meta.id}");
-        }
-        catch (Exception ex)
-        {
-            string errorMesage = ex.ToStringDemystified();
-            _logger.LogError(errorMesage);
-            Meta.errorMessage = errorMesage;
-        }
-        finally
-        {
-            GC.Collect(GC.MaxGeneration, GCCollectionMode.Aggressive, true, true);
-        }
-    }
-    
-    private async Task ParseSaveEU4()
-    {
-        // wait for save file closing
-        await Task.Delay(200, _cancelToken);
-        string extractedGamestatePath = PathHelper.GetSaveFilePath(Meta.id) + ".gamestate";
-        using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(Meta.id).Str, ZipArchiveMode.Read))
-        {
-            var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
-            if (zipEntry is null)
-                throw new Exception("Invalid save format: no 'gamestate' file found");
-
-            zipEntry.ExtractToFile(extractedGamestatePath, true);
-        }
-
-        var gamestateStream = File.OpenRead(extractedGamestatePath);
-
-        Meta.status = SaveFileProcessingStatus.Parsing;
-        var parser = new SaveParserEU4(gamestateStream, SearchQuery);
-        var result = parser.Parse();
-
-        Meta.status = SaveFileProcessingStatus.SavingResults;
-        var resultFilePath = PathHelper.GetParsedSaveFilePath(Meta.id);
-        await using var resultFile = File.OpenWrite(resultFilePath);
-        await JsonSerializer.SerializeAsync(resultFile, result, 
-            _saveSerializerOptions, _cancelToken);
-        Meta.status = SaveFileProcessingStatus.Done;
-        Meta.SaveToFile();
-    }
-}

ParadoxSaveParser.WebAPI/HttpHelpers/ErrorMessage.cs

@@ -1,17 +0,0 @@
-using System.Net;
-using System.Text.Json.Serialization;
-
-namespace ParadoxSaveParser.WebAPI.HttpHelpers;
-
-public record ErrorMessage
-{
-    public ErrorMessage(HttpStatusCode statusCode, string message)
-    {
-        StatusCode = statusCode;
-        Message = message;
-    }
-
-    [JsonIgnore] public HttpStatusCode StatusCode { get; }
-
-    [JsonPropertyName("errorMessage")] public string Message { get; }
-}

ParadoxSaveParser.WebAPI/HttpHelpers/RequestHelper.cs

@@ -1,17 +0,0 @@
-using System.Linq;
-using System.Net;
-
-namespace ParadoxSaveParser.WebAPI.HttpHelpers;
-
-public class RequestHelper
-{
-    internal static ValueOrError<string> GetQueryValue(HttpListenerContext ctx, string paramName)
-    {
-        string[]? values = ctx.Request.QueryString.GetValues(paramName);
-        string? value = values?.FirstOrDefault();
-        if (string.IsNullOrEmpty(value))
-            return new ErrorMessage(HttpStatusCode.BadRequest,
-                $"No request parameter '{paramName}' provided");
-        return value;
-    }
-}

ParadoxSaveParser.WebAPI/HttpHelpers/ReturnHelper.cs

@@ -1,96 +0,0 @@
-using System.IO;
-using System.Net;
-using System.Text.Encodings.Web;
-using System.Text.Json.Serialization;
-using DTLib.Extensions;
-
-namespace ParadoxSaveParser.WebAPI.HttpHelpers;
-
-public class ReturnHelper
-{
-    private static readonly JsonSerializerOptions _responseJsonSerializerOptions = new()
-    {
-        WriteIndented = false,
-        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
-        MaxDepth = 1024,
-        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
-    };
-
-    private static async Task ResponseShort(HttpListenerContext ctx,
-        ContextLogger logger,
-        CancellationToken ct,
-        byte[] value,
-        HttpStatusCode statusCode,
-        string contentType)
-    {
-        ctx.Response.StatusCode = (int)statusCode;
-        ctx.Response.ContentType = contentType;
-        logger.LogDebug($"short response (length: {value.Length} type: {contentType})");
-        if (value.Length > 4000)
-        {
-            logger.LogWarn($"response (length: {value.Length} type: {contentType})\n"
-                + $"Content is too big for {nameof(ResponseShort)}."
-                + $"You should send stream instead of byte array.");
-        }
-        await ctx.Response.OutputStream.WriteAsync(value, ct);
-    }
-    
-    public static async Task<HttpStatusCode> ResponseString(
-        HttpListenerContext ctx,
-        ContextLogger logger,
-        CancellationToken ct,
-        string value, 
-        HttpStatusCode statusCode = HttpStatusCode.OK,
-        string contentType = "text/plain")
-    {
-        await ResponseShort(ctx, logger, ct, value.ToBytes(), statusCode, contentType);
-        logger.LogDebug(value);
-        return statusCode;
-    }
-
-    public static async Task<HttpStatusCode> ResponseJson(
-        HttpListenerContext ctx,
-        ContextLogger logger,
-        CancellationToken ct,
-        object value,
-        HttpStatusCode statusCode = HttpStatusCode.OK)
-    {
-        string json = JsonSerializer.Serialize(
-            value, 
-            value.GetType(),
-            _responseJsonSerializerOptions);
-        return await ResponseString(ctx, logger, ct, json, statusCode, "application/json");
-    }
-
-    public static async Task<HttpStatusCode> ResponseError(
-        HttpListenerContext ctx, 
-        ContextLogger logger,
-        CancellationToken ct, 
-        ErrorMessage error)
-    {
-        return await ResponseJson(ctx, logger, ct, error, error.StatusCode);
-    }
-
-    public static async Task<HttpStatusCode> ResponseStream(HttpListenerContext ctx,
-        ContextLogger logger,
-        CancellationToken ct,
-        Stream valueStream,
-        HttpStatusCode statusCode = HttpStatusCode.OK,
-        string contentType = "application/octet-stream")
-    {
-        try
-        {
-            ctx.Response.StatusCode = (int)statusCode;
-            ctx.Response.ContentType = contentType;
-            logger.LogDebug($"stream response (type: {contentType})");
-            await valueStream.CopyToAsync(ctx.Response.OutputStream, ct);
-            return statusCode;
-        }
-        catch (Exception ex)
-        {
-            return await ResponseError(ctx, logger, ct, 
-                new ErrorMessage(HttpStatusCode.InternalServerError, 
-                ex.ToStringDemystified()));
-        }
-    }
-}

ParadoxSaveParser.WebAPI/HttpHelpers/ValueOrError.cs

@@ -1,19 +0,0 @@
-namespace ParadoxSaveParser.WebAPI.HttpHelpers;
-
-public class ValueOrError<T>
-    {
-        public readonly ErrorMessage? Error;
-        public readonly T? Value;
-
-        private ValueOrError(T? value, ErrorMessage? error)
-        {
-            Value = value;
-            Error = error;
-        }
-
-        public bool HasError => Error is not null;
-
-        public static implicit operator ValueOrError<T>(T v) => new(v, null);
-
-        public static implicit operator ValueOrError<T>(ErrorMessage e) => new(default, e);
-    }

ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.csproj

@@ -13,6 +13,6 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="DTLib.Web" Version="1.3.0"/>
+        <PackageReference Include="DTLib.Web" Version="1.2.2"/>
     </ItemGroup>
 </Project>

ParadoxSaveParser.WebAPI/PathHelper.cs

@@ -1,6 +1,4 @@
-using System.IO;
+namespace ParadoxSaveParser.WebAPI;
-
-namespace ParadoxSaveParser.WebAPI;
 
 public static class PathHelper
 {
@@ -12,10 +10,4 @@ public static class PathHelper
     public static IOPath GetSaveFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".eu4");
 
     public static IOPath GetParsedSaveFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".parsed.json");
-    
-    public static void CreateProgramDirectories()
-    {
-        Directory.Create(DATA_DIR);
-        Directory.Create(SAVES_DIR);
-    }
 }

ParadoxSaveParser.WebAPI/Program.HttpHelpers.cs

@@ -0,0 +1,97 @@
+using System.Linq;
+using System.Net;
+using System.Text.Encodings.Web;
+using System.Text.Json.Serialization;
+using DTLib.Extensions;
+
+namespace ParadoxSaveParser.WebAPI;
+
+public partial class Program
+{
+    private static readonly JsonSerializerOptions _responseJsonSerializerOptions = new()
+    {
+        WriteIndented = false,
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        MaxDepth = 1024
+    };
+
+    public static async Task<HttpStatusCode> ReturnResponseString(HttpListenerContext ctx,
+        string value, HttpStatusCode statusCode = HttpStatusCode.OK)
+    {
+        ctx.Response.StatusCode = (int)statusCode;
+        ctx.Response.ContentType = "text/plain";
+        await ctx.Response.OutputStream.WriteAsync(
+            value.ToBytes(),
+            _mainCancel.Token);
+        return statusCode;
+    }
+
+    public static async Task<HttpStatusCode> ReturnResponseJson(HttpListenerContext ctx,
+        object value, HttpStatusCode statusCode = HttpStatusCode.OK)
+    {
+        ctx.Response.StatusCode = (int)statusCode;
+        ctx.Response.ContentType = "application/json";
+        await JsonSerializer.SerializeAsync(
+            ctx.Response.OutputStream,
+            value,
+            value.GetType(),
+            _responseJsonSerializerOptions,
+            _mainCancel.Token);
+        return statusCode;
+    }
+
+    public static async Task<HttpStatusCode> ReturnResponseError(HttpListenerContext ctx, ErrorMessage error)
+    {
+        ctx.Response.StatusCode = (int)error.StatusCode;
+        ctx.Response.ContentType = "application/json";
+        await JsonSerializer.SerializeAsync(
+            ctx.Response.OutputStream,
+            error,
+            typeof(ErrorMessage),
+            _responseJsonSerializerOptions,
+            _mainCancel.Token);
+        return error.StatusCode;
+    }
+
+
+    internal static ValueOrError<string> GetRequestQueryValue(HttpListenerContext ctx, string paramName)
+    {
+        string[]? values = ctx.Request.QueryString.GetValues(paramName);
+        string? value = values?.FirstOrDefault();
+        if (string.IsNullOrEmpty(value))
+            return new ErrorMessage(HttpStatusCode.BadRequest,
+                $"No request parameter '{paramName}' provided");
+        return value;
+    }
+
+    public record ErrorMessage
+    {
+        public ErrorMessage(HttpStatusCode statusCode, string message)
+        {
+            StatusCode = statusCode;
+            Message = message;
+        }
+
+        [JsonIgnore] public HttpStatusCode StatusCode { get; }
+
+        [JsonPropertyName("errorMessage")] public string Message { get; }
+    }
+
+    public class ValueOrError<T>
+    {
+        public readonly ErrorMessage? Error;
+        public readonly T? Value;
+
+        private ValueOrError(T? value, ErrorMessage? error)
+        {
+            Value = value;
+            Error = error;
+        }
+
+        public bool HasError => Error is not null;
+
+        public static implicit operator ValueOrError<T>(T v) => new(v, null);
+
+        public static implicit operator ValueOrError<T>(ErrorMessage e) => new(default, e);
+    }
+}

ParadoxSaveParser.WebAPI/Program.RequestHandlers.cs

@@ -0,0 +1,117 @@
+using System.IO.Compression;
+using System.Linq;
+using System.Net;
+
+namespace ParadoxSaveParser.WebAPI;
+
+public partial class Program
+{
+    private static async Task<HttpStatusCode> UploadSaveHandler(HttpListenerContext ctx)
+    {
+        string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault();
+        if (contentType != "application/octet-stream")
+            return await ReturnResponseError(ctx, new ErrorMessage(
+                HttpStatusCode.BadRequest,
+                $"Invalid request Content-Type: '{contentType}'"));
+
+        string saveId = Guid.NewGuid().ToString();
+        var metaFilePath = PathHelper.GetMetaFilePath(saveId);
+        if (File.Exists(metaFilePath))
+            return await ReturnResponseError(ctx, new ErrorMessage(
+                HttpStatusCode.InternalServerError,
+                $"Guid collision! file' {metaFilePath}' already exists."));
+
+        var meta = new SaveFileMetadata
+            { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized };
+        if (!_saveMetadataStorage.TryAdd(saveId, meta))
+            return await ReturnResponseError(ctx, new ErrorMessage(
+                HttpStatusCode.InternalServerError,
+                $"Guid collision! Can't create metadata with id {saveId}"));
+
+        meta.status = SaveFileProcessingStatus.Uploading;
+        var saveFilePath = PathHelper.GetSaveFilePath(meta.id);
+        await using var saveFile = File.OpenWrite(saveFilePath);
+        await using var remoteStream = ctx.Request.InputStream;
+        await remoteStream.CopyToAsync(saveFile, _mainCancel.Token);
+        meta.status = SaveFileProcessingStatus.Uploaded;
+
+        return await ReturnResponseString(ctx, saveId);
+    }
+
+    private static ValueOrError<SaveFileMetadata> GetMetaFromRequestId(HttpListenerContext ctx,
+        string requestParamName)
+    {
+        var idOrError = GetRequestQueryValue(ctx, requestParamName);
+        if (idOrError.HasError)
+            return idOrError.Error!;
+
+        if (!_saveMetadataStorage.TryGetValue(idOrError.Value!, out var meta))
+            return new ErrorMessage(HttpStatusCode.InternalServerError,
+                $"Save with id {idOrError.Value} not found");
+
+        return meta;
+    }
+
+    private static async Task<HttpStatusCode> GetSaveStatusHandler(HttpListenerContext ctx)
+    {
+        var metaOrError = GetMetaFromRequestId(ctx, "id");
+        if (metaOrError.HasError)
+            return await ReturnResponseError(ctx, metaOrError.Error!);
+
+        return await ReturnResponseJson(ctx, metaOrError.Value!);
+    }
+
+    private static async Task<HttpStatusCode> ParseSaveEU4Handler(HttpListenerContext ctx)
+    {
+        var metaOrError = GetMetaFromRequestId(ctx, "id");
+        if (metaOrError.HasError)
+            return await ReturnResponseError(ctx, metaOrError.Error!);
+        var meta = metaOrError.Value!;
+
+        var searchQueryOrError = GetRequestQueryValue(ctx, "search");
+        if (searchQueryOrError.HasError)
+            return await ReturnResponseError(ctx, searchQueryOrError.Error!);
+        string searchQuery = searchQueryOrError.Value!;
+
+        try
+        {
+            string extractedGamestatePath = PathHelper.GetSaveFilePath(meta.id) + ".gamestate";
+            using (var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(meta.id).Str, ZipArchiveMode.Read))
+            {
+                var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
+                if (zipEntry is null)
+                    return await ReturnResponseError(ctx, new ErrorMessage(
+                        HttpStatusCode.BadRequest,
+                        "Invalid save format: no 'gamestate' file found"));
+
+                zipEntry.ExtractToFile(extractedGamestatePath, true);
+            }
+
+            var gamestateStream = File.OpenRead(extractedGamestatePath);
+
+            meta.status = SaveFileProcessingStatus.Parsing;
+            var se = SearchExpressionCompiler.Compile(searchQuery);
+            var parser = new SaveParserEU4(gamestateStream, se);
+            var result = parser.Parse();
+
+            meta.status = SaveFileProcessingStatus.SavingResults;
+            var resultFilePath = PathHelper.GetParsedSaveFilePath(meta.id);
+            await using var resultFile = File.OpenWrite(resultFilePath);
+            await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions, _mainCancel.Token);
+            meta.status = SaveFileProcessingStatus.Done;
+            meta.SaveToFile();
+        }
+        catch (Exception ex)
+        {
+            string errorMesage = ex.ToStringDemystified();
+            _loggerRoot.LogWarn(nameof(ParseSaveEU4Handler), errorMesage);
+            return await ReturnResponseError(ctx, new ErrorMessage(HttpStatusCode.BadRequest, errorMesage));
+        }
+        finally
+        {
+            GC.Collect(GC.MaxGeneration, GCCollectionMode.Aggressive, true, true);
+        }
+
+        return await ReturnResponseJson(ctx, meta);
+    }
+}

ParadoxSaveParser.WebAPI/Program.cs

@@ -13,56 +13,42 @@ global using File = DTLib.Filesystem.File;
 global using Path = DTLib.Filesystem.Path;
 using System.Collections.Concurrent;
 using System.IO;
-using DTLib.Console;
+using System.Text.Encodings.Web;
 using DTLib.Dtsod;
 using DTLib.Web;
 using DTLib.Web.Routes;
-using ParadoxSaveParser.WebAPI.BackgroundTasks;
-using ParadoxSaveParser.WebAPI.Routes;
 
 namespace ParadoxSaveParser.WebAPI;
 
-public static class Program
+public static partial class Program
 {
     private static readonly IOPath _configPath = "./config.dtsod";
     private static Config _config = new();
-    private static bool IsDebug = true;
-    private static readonly CancellationTokenSource _mainCancel = new();
-    internal static readonly ConcurrentDictionary<string, SaveFileMetadata> _saveMetadataStorage = new();
 
+    private static readonly ILogger _loggerRoot = new CompositeLogger(
+        new ConsoleLogger(),
+        new FileLogger("logs", "ParadoxSaveParser.WebAPI"));
+
+    private static readonly CancellationTokenSource _mainCancel = new();
+    private static readonly ConcurrentDictionary<string, SaveFileMetadata> _saveMetadataStorage = new();
+
+    private static readonly JsonSerializerOptions _saveSerializerOptions = new()
+    {
+        WriteIndented = true,
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        MaxDepth = 1024
+    };
 
     public static void Main(string[] args)
     {
         Console.InputEncoding = Encoding.UTF8;
         Console.OutputEncoding = Encoding.UTF8;
         Console.CursorVisible = false;
-        
+        var logger = new ContextLogger(nameof(Main), _loggerRoot);
-        #if DEBUG
-        IsDebug = true;
-        #endif
-        
-        new LaunchArgumentParser(
-                new LaunchArgument(["-d", "--debug"], 
-                    "enables debug log output to console",
-                    () => IsDebug = true)
-            ).AllowNoArguments().ParseAndHandle(args);
-        
-        var loggerRoot = new CompositeLogger(
-            new ConsoleLogger
-            {
-                DebugLogEnabled = IsDebug
-            },
-            new FileLogger("logs", "ParadoxSaveParser.WebAPI")
-            {
-                DebugLogEnabled = true
-            });
-        var loggerMain = new ContextLogger(nameof(Main), loggerRoot);
-        loggerMain.LogDebug("Debug log is enabled");
-        
         Console.CancelKeyPress += (_, e) =>
         {
             e.Cancel = true;
-            loggerMain.LogInfo("Ctrl+C Pressed");
+            logger.LogInfo("Ctrl+C Pressed");
             _mainCancel.Cancel();
         };
 
@@ -71,59 +57,54 @@ public static class Program
             // config
             if (!File.Exists(_configPath))
             {
-                loggerMain.LogWarn("config file not found.");
+                logger.LogWarn("config file not found.");
                 File.WriteAllText(_configPath, _config.ToString());
-                loggerMain.LogWarn($"created default at {_configPath}.");
+                logger.LogWarn($"created default at {_configPath}.");
             }
             else
             {
                 _config = Config.FromDtsod(new DtsodV23(File.ReadAllText(_configPath)));
             }
 
-            PathHelper.CreateProgramDirectories();
+            PrepareLocalFiles();
 
-            var metaFiles = System.IO.Directory.GetFiles(
+            // http server
+            var router = new SimpleRouter(_loggerRoot);
+            router.DefaultRoute = new ServeFilesRouteHandler("public");
+            router.MapRoute("/getSaveStatus", HttpMethod.GET, GetSaveStatusHandler);
+            router.MapRoute("/uploadSave/eu4", HttpMethod.POST, UploadSaveHandler);
+            router.MapRoute("/parseSave/eu4", HttpMethod.POST, ParseSaveEU4Handler);
+
+            var app = new WebApp(_config.BaseUrl, _loggerRoot, router, _mainCancel.Token);
+            app.Run().GetAwaiter().GetResult();
+        }
+        catch (OperationCanceledException ex)
+        {
+            logger.LogWarn($"catched OperationCanceledException from {ex.Source}");
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex.ToStringDemystified());
+        }
+    }
+
+    public static void PrepareLocalFiles()
+    {
+        Directory.Create(PathHelper.DATA_DIR);
+        Directory.Create(PathHelper.SAVES_DIR);
+        foreach (string metaFilePath in System.IO.Directory.GetFiles(
                      PathHelper.SAVES_DIR.Str, "*.meta.json",
-                SearchOption.TopDirectoryOnly);
+                     SearchOption.TopDirectoryOnly))
-            foreach (string metaFilePath in metaFiles)
         {
             using var metaFile = File.OpenRead(metaFilePath);
             var meta = JsonSerializer.Deserialize<SaveFileMetadata>(metaFile) ??
                        throw new NullReferenceException(metaFilePath);
             if (meta.status != SaveFileProcessingStatus.Done)
-                    loggerMain.LogWarn(nameof(Main),
+                _loggerRoot.LogWarn(nameof(PrepareLocalFiles),
                     $"metadata file '{metaFilePath}' status has invalid status {meta.status}");
 
             if (!_saveMetadataStorage.TryAdd(meta.id, meta))
                 throw new Exception("Guid collision!");
         }
-
-            var bgJobManager = new BackgroundJobManager(loggerRoot);
-            var saveParsingSearchExpressions = new Dictionary<Game, ISearchExpression>
-            {
-                { Game.EU4, SearchExpressionCompiler.Compile("*.~") },
-            };
-            
-            // http server
-            var router = new SimpleRouter(loggerRoot);
-            router.DefaultRoute = new ServeFilesRouteHandler("public");
-            router.MapRoute("/getSaveStatus", HttpMethod.GET, new GetSaveStatusHandler(_mainCancel.Token));
-            router.MapRoute("/uploadSave/eu4", HttpMethod.POST, new UploadSaveHandler(_mainCancel.Token, 
-                bgJobManager, saveParsingSearchExpressions));
-            router.MapRoute("/getSaveData", HttpMethod.GET, new GetSaveDataHandler(_mainCancel.Token));
-
-            var app = new WebApp(_config.BaseUrl, loggerRoot, router, _mainCancel.Token);
-            app.Run().GetAwaiter().GetResult();
-        }
-        catch (OperationCanceledException ex)
-        {
-            loggerMain.LogWarn($"catched OperationCanceledException from {ex.Source}");
-        }
-        catch (Exception ex)
-        {
-            loggerMain.LogError(ex.ToStringDemystified());
     }
 }
-
-    
-}

ParadoxSaveParser.WebAPI/README.md

@@ -1,39 +0,0 @@
-# WebAPI
-Simple web application created using DTLib.Web.
-
-# Routes
-
-### POST `/uploadSave/eu4`
-- **Request:** `application/octet-stream` - .eu4 file
-- **Response:**
-    ```json
-    { "saveId": "string" }
-    ```
-    or `{ "errorMessage": "string" }`
-
-### GET `/getSaveStatus`
-- **Query Params:**
-    - `id` - id of uploaded save file
-- **Response:** [SaveFileMetadata](./SaveFileMetadata.cs)
-    ```json
-    {
-        "id": "string",
-        "game": "string",
-        "status": "string",
-        "errorMessage": "string?"
-    }
-    ```
-
-### GET `/getSaveData`
-- **Query Params:**
-    - `id` - id of uploaded save file
-- **Response:**
-    ```json5
-    {
-        "key0": [ "objects" ],
-        "key1": [ "objects" ],
-        //...
-    }
-    ```
-    or `{ "errorMessage": "string" }`
-

ParadoxSaveParser.WebAPI/Routes/GetSaveDataHandler.cs

@@ -1,33 +0,0 @@
-using System.Net;
-using ParadoxSaveParser.WebAPI.HttpHelpers;
-
-namespace ParadoxSaveParser.WebAPI.Routes;
-
-internal class GetSaveDataHandler : RouteHandlerBase
-{
-    public GetSaveDataHandler(CancellationToken cancelAllToken) 
-        : base(cancelAllToken)
-    {
-    }
-
-    public override async Task<HttpStatusCode> HandleRequest(
-        HttpListenerContext ctx, ContextLogger requestLogger)
-    {
-        var idOrError = RequestHelper.GetQueryValue(ctx, "id");
-        if (idOrError.HasError)
-            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, idOrError.Error!);
-        string id = idOrError.Value!;
-
-        IOPath dataFilePath = PathHelper.GetParsedSaveFilePath(id);
-        if (!File.Exists(dataFilePath))
-            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
-                new ErrorMessage(HttpStatusCode.InternalServerError, 
-                    $"Save with id {id} not found")
-            );
-
-        await using var dataFile = File.OpenRead(dataFilePath);
-        
-        return await ReturnHelper.ResponseStream(ctx, requestLogger, _cancelAllToken,
-            dataFile, contentType: "application/json");
-    }
-}

ParadoxSaveParser.WebAPI/Routes/GetSaveStatusHandler.cs

@@ -1,29 +0,0 @@
-using System.Net;
-using ParadoxSaveParser.WebAPI.HttpHelpers;
-
-namespace ParadoxSaveParser.WebAPI.Routes;
-
-internal class GetSaveStatusHandler : RouteHandlerBase
-{
-    public GetSaveStatusHandler(CancellationToken cancelAllToken) 
-        : base(cancelAllToken)
-    {
-    }
-
-    public override async Task<HttpStatusCode> HandleRequest(
-        HttpListenerContext ctx, ContextLogger requestLogger)
-    {
-        var idOrError = RequestHelper.GetQueryValue(ctx, "id");
-        if (idOrError.HasError)
-            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken, idOrError.Error!);
-        string id = idOrError.Value!;
-        
-        if (!Program._saveMetadataStorage.TryGetValue(id, out var meta))
-            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
-                new ErrorMessage(HttpStatusCode.InternalServerError, 
-                    $"Save with id {id} not found")
-                );
-
-        return await ReturnHelper.ResponseJson(ctx, requestLogger, _cancelAllToken, meta);
-    }
-}

ParadoxSaveParser.WebAPI/Routes/RouteHandlerBase.cs

@@ -1,16 +0,0 @@
-using System.Net;
-using DTLib.Web.Routes;
-
-namespace ParadoxSaveParser.WebAPI.Routes;
-
-public abstract class RouteHandlerBase : IRouteHandler
-{
-    protected readonly CancellationToken _cancelAllToken;
-    
-    protected RouteHandlerBase(CancellationToken cancelAllToken)
-    {
-        _cancelAllToken = cancelAllToken;
-    }
-
-    public abstract Task<HttpStatusCode> HandleRequest(HttpListenerContext ctx, ContextLogger requestLogger);
-}

ParadoxSaveParser.WebAPI/Routes/UploadSaveHandler.cs

@@ -1,60 +0,0 @@
-using System.Linq;
-using System.Net;
-using ParadoxSaveParser.WebAPI.BackgroundTasks;
-using ParadoxSaveParser.WebAPI.HttpHelpers;
-
-namespace ParadoxSaveParser.WebAPI.Routes;
-
-public class UploadSaveHandler : RouteHandlerBase
-{
-    private readonly BackgroundJobManager _bgJobManager;
-    private readonly Dictionary<Game, ISearchExpression> _searchQueries;
-
-    public UploadSaveHandler(
-        CancellationToken cancelAllToken, 
-        BackgroundJobManager bgJobManager,
-        Dictionary<Game, ISearchExpression> searchQueries) 
-        : base(cancelAllToken)
-    {
-        _bgJobManager = bgJobManager;
-        _searchQueries = searchQueries;
-    }
-
-    public override async Task<HttpStatusCode> HandleRequest(
-        HttpListenerContext ctx, ContextLogger requestLogger)
-    {
-        string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault();
-        if (contentType != "application/octet-stream")
-            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
-                    new ErrorMessage(HttpStatusCode.BadRequest,
-                $"Invalid request Content-Type: '{contentType}'")
-                    );
-
-        string saveId = Guid.NewGuid().ToString();
-        var metaFilePath = PathHelper.GetMetaFilePath(saveId);
-        if (File.Exists(metaFilePath))
-            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
-                    new ErrorMessage(HttpStatusCode.InternalServerError,
-                $"Guid collision! file' {metaFilePath}' already exists.")
-                    );
-
-        var meta = new SaveFileMetadata
-            { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized };
-        if (!Program._saveMetadataStorage.TryAdd(saveId, meta))
-            return await ReturnHelper.ResponseError(ctx, requestLogger, _cancelAllToken,
-                    new ErrorMessage(HttpStatusCode.InternalServerError,
-                $"Guid collision! Can't create metadata with id {saveId}")
-                    );
-
-        meta.status = SaveFileProcessingStatus.Uploading;
-        var saveFilePath = PathHelper.GetSaveFilePath(meta.id);
-        await using var saveFile = File.OpenWrite(saveFilePath);
-        await using var remoteStream = ctx.Request.InputStream;
-        await remoteStream.CopyToAsync(saveFile, _cancelAllToken);
-        meta.status = SaveFileProcessingStatus.Uploaded;
-
-        _bgJobManager.StartNewParsingOperation(meta, _searchQueries[meta.game], _cancelAllToken);
-        dynamic responseData = new { saveId };
-        return await ReturnHelper.ResponseJson(ctx, requestLogger, _cancelAllToken, responseData);
-    }
-}

ParadoxSaveParser.WebAPI/SaveFileMetadata.cs

@@ -20,11 +20,7 @@ public enum Game
 
 public class SaveFileMetadata
 {
-    private static readonly JsonSerializerOptions _configSerializerOptions = new()
+    private static readonly JsonSerializerOptions _jsonOptions = new() { WriteIndented = true };
-    {
-        WriteIndented = true,
-        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
-    };
     public required string id { get; init; }
 
     [JsonConverter(typeof(JsonStringEnumConverter))]
@@ -33,13 +29,9 @@ public class SaveFileMetadata
     [JsonConverter(typeof(JsonStringEnumConverter))]
     public required SaveFileProcessingStatus status { get; set; }
 
-    // if error occured during parsing, it's message is saved here
-    // status stays the same as when error occured
-    public string? errorMessage { get; set; }
-
     public void SaveToFile()
     {
         using var metaFile = File.OpenWrite(PathHelper.GetMetaFilePath(id));
-        JsonSerializer.Serialize(metaFile, this, _configSerializerOptions);
+        JsonSerializer.Serialize(metaFile, this, _jsonOptions);
     }
 }

ParadoxSaveParser.sln

@@ -7,8 +7,7 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SolutionFolder", "SolutionFolder", "{F1D312F1-0620-4E35-8D78-9A2808CDE12C}"
 	ProjectSection(SolutionItems) = preProject
 		.gitignore = .gitignore
-		TODO.md = TODO.md
+		TODO.txt = TODO.txt
-		README.md = README.md
 	EndProjectSection
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParadoxSaveParser.Lib.Tests", "ParadoxSaveParser.Lib.Tests\ParadoxSaveParser.Lib.Tests.csproj", "{23F4BE1B-3043-4821-9F65-74FF5F57FA59}"

README.md

@@ -1,16 +0,0 @@
-# Paradox Save Parser
-Yet another save files parser. 
-
-## Supported games:
-- EU4
-
-## Project structure
-- **[ParadoxSaveParser.Lib](./ParadoxSaveParser.Lib)** - 
-    Parser itself
-- **[ParadoxSaveParser.Lib.Tests](./ParadoxSaveParser.Lib.Tests)** - 
-    Tests for parser
-- **[ParadoxSaveParser.CLI](./ParadoxSaveParser.CLI)** - 
-    Command line tool to parse save files. Can be used in interactive mode.
-- **[ParadoxSaveParser.WebAPI](./ParadoxSaveParser.WebAPI)** - 
-    Backend for my save file analytics website (TODO: add repo link).
-

TODO.md

@@ -1,8 +0,0 @@
-## WebAPI:
-	Temp files management system
-	Database to store metadata (SQLite-Net Extensions)
-
-## WebAPI.SaveParsingOperation:
-	Add debug log
-	Save parsed data in protobuf
-	Re-parse if saved data was parsed with another query

TODO.txt

@@ -0,0 +1,14 @@
+Parser.CLI:
+	Temp files management system
+
+Main:
+	Temp files management system
+	Database???
+	Add query to get parsed data
+
+ParseSaveHandler:
+	Make this method run as background task instead of POST query
+	Add debug log
+	Save parsed data in protobuf
+	Re-parse if saved data was parsed with another query
+