number parsing and control characters skipping

.gitignore

@@ -9,6 +9,7 @@
 [Ll]og/
 [Ll]ogs/
 [Pp]ublish/
+data/
 
 # IDE files
 .vs/

ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj

@@ -1,13 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
     <PropertyGroup>
         <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
+        <ImplicitUsings>disable</ImplicitUsings>
         <Nullable>enable</Nullable>
     </PropertyGroup>
 
-    <ItemGroup>
-      <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    </ItemGroup>
-
 </Project>

ParadoxSaveParser.Lib/Parser.cs

@@ -1,27 +1,281 @@
-namespace ParadoxSaveParser.Lib;
+global using System;
+global using System.IO;
+global using System.Text;
+global using System.Collections.Generic;
 
-public abstract class Parser
+namespace ParadoxSaveParser.Lib;
+
+public class Parser
 {
     protected Stream _saveFile;
+    private List<Token> _tokens = new(4_194_304);
+    private int _tokenIndex;
 
-    protected Parser(Stream savefile)
+    public Parser(Stream savefile)
     {
         _saveFile = savefile;
     }
 
-    protected enum TokenType
+    protected enum TokenType : byte
     {
-        Invalid, String, Equals, BracketOpen, BracketClose,
+        Invalid,
+        StringOrNumber,
+        Equals,
+        BracketOpen,
+        BracketClose,
     }
 
     protected struct Token
     {
         public TokenType type;
+        public short column;
+        public int line;
         public string? value;
+
+        public override string ToString()
+        {
+            string s;
+            switch (type)
+            {
+                case TokenType.Invalid:
+                    s = "INVALID_TOKEN";
+                    break;
+                case TokenType.StringOrNumber:
+                    s = value ?? "NULL";
+                    break;
+                case TokenType.Equals:
+                    s = "=";
+                    break;
+                case TokenType.BracketOpen:
+                    s = "{";
+                    break;
+                case TokenType.BracketClose:
+                    s = "}";
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(type.ToString());
             }
             
-    protected void BuildAST()
+            return $"{line}:{column} '{s}'";
-    {
+        }
+    }
 
+    protected void Lex()
+    {
+        _tokens.Clear();
+        
+        string expectedHeader = "EU4txt";
+        byte[] headBytes = new byte[expectedHeader.Length];
+        _saveFile.ReadExactly(headBytes);
+        string headStr = Encoding.UTF8.GetString(headBytes);
+        if (headStr != expectedHeader)
+            throw new Exception($"Invalid gamestate header: '{headStr}'");
+        
+        StringBuilder str = new();
+        int line = 2;
+        int column = 0;
+        bool isQuoteOpen = false;
+        bool isStrInQuotes = false;
+
+        void CompleteStringToken()
+        {
+            if (isQuoteOpen)
+                return;
+            // strings in quotes can be empty
+            if (!isStrInQuotes && (str.Length <= 0 || str[0] == '#')) 
+                return;
+            _tokens.Add(new Token
+            {
+                type = TokenType.StringOrNumber,
+                column = (short)(column - str.Length),
+                line = line,
+                value = str.ToString()
+            });
+            str.Clear();
+            isStrInQuotes = false;
+        }
+
+        while (_saveFile.CanRead)
+        {
+            int c = _saveFile.ReadByte();
+            column++;
+            switch (c)
+            {
+                case -1:
+                    CompleteStringToken();
+                    return;
+                case '\"':
+                    isQuoteOpen = !isQuoteOpen;
+                    isStrInQuotes = true;
+                    break;
+                case ' ':
+                case '\t':
+                case '\r':
+                    CompleteStringToken();
+                    break;
+                case '\n':
+                    CompleteStringToken();
+                    line++;
+                    column = 0;
+                    break;
+                case '=':
+                    CompleteStringToken();
+                    _tokens.Add(new Token
+                    {
+                        type = TokenType.Equals,
+                        line = line, column = (short)column
+                    });
+                    break;
+                case '{':
+                    CompleteStringToken();
+                    _tokens.Add(new Token
+                    {
+                        type = TokenType.BracketOpen,
+                        line = line, column = (short)column
+                    });
+                    break;
+                case '}':
+                    CompleteStringToken();
+                    _tokens.Add(new Token
+                    {
+                        type = TokenType.BracketClose,
+                        line = line, column = (short)column
+                    });
+                    break;
+                default:
+                    // Skip control characters, which are invisible and causing frontend bugs.
+                    // I dont know why there are so many of them in strings.
+                    if(c >= 0x20)
+                        str.Append((char)c);
+                    break;
+            }
+        }
+    }
+
+    protected class UnexpectedTokenException : Exception
+    {
+        public UnexpectedTokenException(Token token, int tokenIndex) :
+            base($"Unexpected token at index {tokenIndex}: {token}")
+        {}
+    }
+    
+
+    private object? ParseValue()
+    {
+        Token tok = _tokens[_tokenIndex++];
+        switch (tok.type)
+        {
+            case TokenType.StringOrNumber:
+                if(string.IsNullOrEmpty(tok.value))
+                    return string.Empty;
+                if (tok.value[0] != '-' && !char.IsDigit(tok.value[0])) 
+                    return tok.value;
+                if(tok.value.Contains('.') && Double.TryParse(tok.value, out double d)) 
+                    return d;
+                if (Int64.TryParse(tok.value, out long l))
+                    return l;
+                return tok.value;
+            case TokenType.BracketOpen:
+                return ParseListOrDict();
+            case TokenType.BracketClose:
+                return null;
+            default:
+                throw new UnexpectedTokenException(tok, _tokenIndex - 1);
+        }
+    }
+
+    private object ParseListOrDict()
+    {
+        Token first = _tokens[_tokenIndex];
+        Token second = _tokens[_tokenIndex + 1];
+        if (first.type == TokenType.StringOrNumber && second.type == TokenType.Equals) 
+            return ParseDict();
+        
+        return ParseList();
+    }
+
+    private List<object> ParseList()
+    {
+        List<object> list = new();
+        while(true)
+        {
+            object? value = ParseValue();
+            if (value == null)
+                break;
+            list.Add(value);
+        }
+        return list;
+    }
+
+    
+    private Dictionary<string, List<object>> ParseDict()
+    {
+        Dictionary<string, List<object>> dict = new();
+        // root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count
+        while (_tokenIndex < _tokens.Count)
+        {
+            Token tok = _tokens[_tokenIndex++];
+            // end of dictionary 
+            if (tok.type == TokenType.BracketClose)
+                break;
+            
+            // Saves may contain some blocks without key.
+            // Such blocks are skipped because idk where to put them.
+            // Example: `technology_group=tech_cannorian{ }
+            //          { } { } { }`
+            if (tok.type == TokenType.BracketOpen)
+            {
+                int bracketBalance = 1;
+                while (bracketBalance != 0)
+                {
+                    tok = _tokens[_tokenIndex++];
+                    if (tok.type == TokenType.BracketOpen)
+                        bracketBalance++;
+                    else if (tok.type == TokenType.BracketClose)
+                        bracketBalance--;
+                }
+                
+                continue;
+            }
+            
+            if(tok.type != TokenType.StringOrNumber)
+                throw new UnexpectedTokenException(tok, _tokenIndex - 1);
+            
+            string key = tok.value!;
+            
+            tok = _tokens[_tokenIndex++];
+            if (tok.type == TokenType.BracketOpen)
+            {
+                // Saves may contain key-value definition without `=`.
+                // Example: `map_area_data{` instead of `map_area_data = {`
+                _tokenIndex--;
+            }
+            else if(tok.type != TokenType.Equals)
+                throw new UnexpectedTokenException(tok, _tokenIndex - 1);
+            
+            
+            object? value = ParseValue();
+            if (value == null)
+                throw new UnexpectedTokenException(_tokens[_tokenIndex - 1], _tokenIndex - 1);
+            if(!dict.TryGetValue(key, out List<object>? list))
+            {
+                list = new List<object>();
+                dict.Add(key, list);
+            }
+            list.Add(value);
+        }
+
+        return dict;
+    }
+    
+    public Dictionary<string, List<object>> Parse()
+    {
+        Lex();
+        if (_tokens.Count == 0)
+            throw new Exception("Save file is empty");
+
+        _tokenIndex = 0;
+        var root = ParseDict();
+        return root;
     }
 }

ParadoxSaveParser.Lib/ParserEU4.cs

@@ -1,8 +0,0 @@
-namespace ParadoxSaveParser.Lib;
-
-public class ParserEU4 : Parser
-{
-    public ParserEU4(Stream savefile) : base(savefile)
-    {
-    }
-}

ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.csproj

@@ -1,5 +1,4 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
-
     <PropertyGroup>
         <TargetFramework>net8.0</TargetFramework>
         <Nullable>enable</Nullable>
@@ -11,4 +10,23 @@
       <ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+      <PackageReference Include="DTLib.Demystifier" Version="1.1.0" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Compile Remove="data\**" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <EmbeddedResource Remove="data\**" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Content Remove="data\**" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <None Remove="data\**" />
+    </ItemGroup>
 </Project>

ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.http

@@ -1,6 +0,0 @@
-@ParadoxSaveParser.WebAPI_HostAddress = http://localhost:5226
-
-GET {{ParadoxSaveParser.WebAPI_HostAddress}}/weatherforecast/
-Accept: application/json
-
-###

ParadoxSaveParser.WebAPI/PathHelper.cs

@@ -0,0 +1,12 @@
+using System.IO;
+
+namespace ParadoxSaveParser.WebAPI;
+
+public static class PathHelper
+{
+    public const string DATA_DIR = "data";
+    public static string SAVES_DIR = Path.Join(DATA_DIR, "saves");
+    public static string GetMetaFilePath(string save_id) => Path.Join(SAVES_DIR, save_id + ".meta.json");
+    public static string GetSaveFilePath(string save_id) => Path.Join(SAVES_DIR, save_id + ".eu4");
+    public static string GetParsedSaveFilePath(string save_id) => Path.Join(SAVES_DIR, save_id + ".parsed.json");
+}

ParadoxSaveParser.WebAPI/Program.cs

@@ -1,76 +1,162 @@
 global using System;
+global using System.IO;
+global using System.Collections.Generic;
+global using System.Text;
+global using System.Text.Json;
+global using System.Threading.Tasks;
+global using DTLib.Demystifier;
+global using ParadoxSaveParser.Lib;
 using System.Collections.Concurrent;
-using System.IO;
+using System.IO.Compression;
 using System.Linq;
-using System.Net;
+using System.Text.Encodings.Web;
-using System.Text.Json;
-using System.Text.Json.Serialization;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Http;
-using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
-using ParadoxSaveParser.Lib;
 
 namespace ParadoxSaveParser.WebAPI;
 
-public enum SaveFileProcessingStatus
-{
-    NotFound, Uploading, Parsing, SavingResults, Done, Error
-}
-
-public class SaveFileMetadata
-{
-    public required string guid;
-    public required SaveFileProcessingStatus status { get; set; }
-}
-
 public class Program
 {
-    private const string DATA_DIR = "data";
-    private static string SAVES_DIR = Path.Join(DATA_DIR, "saves");
     private static ConcurrentDictionary<string, SaveFileMetadata> _saveMetadataStorage = new();
+    private static WebApplication _app = null!;
+
+    private static JsonSerializerOptions _saveSerializerOptions = new()
+    {
+        WriteIndented = false,
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        MaxDepth = 1024,
+    };
 
     public static void Main(string[] args)
     {
         var builder = WebApplication.CreateBuilder(args);
-        var app = builder.Build();
+        _app = builder.Build();
 
-        foreach (var metaFilePath in Directory.GetFiles(SAVES_DIR, "*.meta.json", SearchOption.TopDirectoryOnly))
+        Directory.CreateDirectory(PathHelper.DATA_DIR);
+        Directory.CreateDirectory(PathHelper.SAVES_DIR);
+        foreach (var metaFilePath in Directory.GetFiles(PathHelper.SAVES_DIR, "*.meta.json", SearchOption.TopDirectoryOnly))
         {
             using var metaFile = File.Open(metaFilePath, FileMode.Open, FileAccess.Read);
             var meta = JsonSerializer.Deserialize<SaveFileMetadata>(metaFile) ?? throw new NullReferenceException(metaFilePath);
             if (meta.status != SaveFileProcessingStatus.Done)
             {
-                app.Logger.Log(LogLevel.Warning, $"metadata file '{metaFilePath}' status has invalid status {meta.status}"));
+                _app.Logger.Log(LogLevel.Warning, "metadata file '{metaFilePath}' status has invalid status {status}", metaFilePath, meta.status);
             }
 
-            if(!_saveMetadataStorage.TryAdd(meta.guid, meta))
+            if(!_saveMetadataStorage.TryAdd(meta.id, meta))
                 throw new Exception("Guid collision!");
         }
         
-        app.UseHttpsRedirection();
+        _app.UseHttpsRedirection();
-        app.MapPost("/parse/eu4", async httpContext =>
+        _app.MapGet("/getSaveStatus", GetSaveStatusHandler);
+        _app.MapPost("/uploadSave/eu4", UploadSaveHandler);
+        _app.MapPost("/parseSave/eu4", ParseSaveEU4Handler);
+        _app.Run();
+    }
+
+    private static async Task UploadSaveHandler(HttpContext httpContext)
     {
         var remoteFile = httpContext.Request.Form.Files.FirstOrDefault();
-            if(remoteFile is null)
+        if (remoteFile is null || !remoteFile.FileName.EndsWith(".eu4"))
-                return;
+        {
-            string save_id = Guid.NewGuid().ToString();
+            throw new BadHttpRequestException($"Invalid file format: {remoteFile?.FileName}",
-            string meta_file_path = Path.Join(SAVES_DIR, save_id + ".meta.json");
+                StatusCodes.Status400BadRequest);
-            // string save_file_path = Path.Join(SAVES_DIR, save_id + ".eu4");
+        }
-            // await using var metaFile = File.Open(meta_file_path, FileMode.CreateNew, FileAccess.Write);
+
-            if (File.Exists(meta_file_path))
+        string saveId = Guid.NewGuid().ToString();
+        string metaFilePath = PathHelper.GetMetaFilePath(saveId);
+        if (File.Exists(metaFilePath))
         {
             httpContext.Response.StatusCode = StatusCodes.Status500InternalServerError;
-                throw new Exception($"Guid collision! file {meta_file_path} already exists.")
+            throw new BadHttpRequestException($"Guid collision! file {metaFilePath} already exists.", StatusCodes.Status500InternalServerError);
         }
 
-            await using var saveFile = 
+        var meta = new SaveFileMetadata { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized, };
-            await remoteFile.OpenReadStream().CopyToAsync();
+        if (!_saveMetadataStorage.TryAdd(saveId, meta))
-            var parser = new ParserEU4(stream);
+        {
+            throw new BadHttpRequestException($"Guid collision! Can't create metadata with id {saveId}", StatusCodes.Status500InternalServerError);
+        }
 
+        meta.status = SaveFileProcessingStatus.Uploading;
+        string saveFilePath = PathHelper.GetSaveFilePath(meta.id);
+        await using var saveFile = File.Open(saveFilePath, FileMode.CreateNew, FileAccess.ReadWrite);
+        await using var remoteStream = remoteFile.OpenReadStream();
+        await remoteStream.CopyToAsync(saveFile);
+        meta.status = SaveFileProcessingStatus.Uploaded;
         
-        });
+        await httpContext.Response.WriteAsJsonAsync(meta);
+    }
 
-        app.Run();
+    private static SaveFileMetadata GetMetaFromRequestId(HttpContext httpContext, string requestParamName)
+    {
+        httpContext.Request.Query.TryGetValue(requestParamName, out var ids);
+        string? id = ids.FirstOrDefault();
+        if (string.IsNullOrEmpty(id))
+        {
+            throw new BadHttpRequestException($"No request parameter '{requestParamName}' provided", 
+                StatusCodes.Status400BadRequest);
+        }
+
+        if (!_saveMetadataStorage.TryGetValue(id, out var meta))
+        {
+            throw new BadHttpRequestException($"Save with {id} not found", 
+                StatusCodes.Status400BadRequest);
+        }
+
+        return meta;
+    }
+    
+    private static async Task GetSaveStatusHandler(HttpContext httpContext)
+    {
+        var meta = GetMetaFromRequestId(httpContext, "id");
+        await httpContext.Response.WriteAsJsonAsync(meta);
+    }
+
+    private static async Task ParseSaveEU4Handler(HttpContext httpContext)
+    {
+        var meta = GetMetaFromRequestId(httpContext, "id");
+        if (meta.status == SaveFileProcessingStatus.Error)
+        {
+            httpContext.Response.StatusCode = StatusCodes.Status500InternalServerError;
+            await httpContext.Response.WriteAsJsonAsync(meta);
+            return;
+        }
+        
+        try
+        {
+            if (meta.status != SaveFileProcessingStatus.Uploaded)
+                throw new Exception($"Invalid save processing status: {meta.status}");
+
+            using var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(meta.id), ZipArchiveMode.Read);
+            var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
+            if (zipEntry is null)
+                throw new Exception("Invalid save format: no gamestate file found");
+            string extractedGamestatePath = PathHelper.GetSaveFilePath(meta.id) + ".gamestate";
+            zipEntry.ExtractToFile(extractedGamestatePath);
+            var gamestateStream = File.Open(extractedGamestatePath, FileMode.Open, FileAccess.Read);
+
+            meta.status = SaveFileProcessingStatus.Parsing;
+            var parser = new Parser(gamestateStream);
+            var result = parser.Parse();
+
+            meta.status = SaveFileProcessingStatus.SavingResults;
+            string resultFilePath = PathHelper.GetParsedSaveFilePath(meta.id);
+            await using var resultFile = File.Open(resultFilePath, FileMode.CreateNew, FileAccess.Write);
+            await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions);
+            meta.status = SaveFileProcessingStatus.Done;
+            meta.SaveToFile();
+        }
+        catch (Exception ex)
+        {
+            meta.status = SaveFileProcessingStatus.Error;
+            string errorMesage = ex.ToStringDemystified();
+            meta.errorMesage = errorMesage;
+            httpContext.Response.StatusCode = StatusCodes.Status500InternalServerError;
+            _app.Logger.Log(LogLevel.Error, "ParseSaveEU4 Error: {errorMesage}", errorMesage);
+        }
+
+        GC.Collect();
+        await httpContext.Response.WriteAsJsonAsync(meta);
     }
 }

ParadoxSaveParser.WebAPI/Properties/launchSettings.json

@@ -12,8 +12,6 @@
     "http": {
       "commandName": "Project",
       "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "launchUrl": "/",
       "applicationUrl": "http://localhost:5226",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
@@ -22,8 +20,6 @@
     "https": {
       "commandName": "Project",
       "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "launchUrl": "/",
       "applicationUrl": "https://localhost:7032;http://localhost:5226",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
@@ -31,8 +27,6 @@
     },
     "IIS Express": {
       "commandName": "IISExpress",
-      "launchBrowser": true,
-      "launchUrl": "/",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
       }

ParadoxSaveParser.WebAPI/SaveFileMetadata.cs

@@ -0,0 +1,35 @@
+using System.IO;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace ParadoxSaveParser.WebAPI;
+
+public enum SaveFileProcessingStatus
+{
+    Initialized, Uploading, Uploaded, Parsing, SavingResults, Done, Error
+}
+
+public enum Game
+{
+    Unknown, EU4
+}
+
+public class SaveFileMetadata
+{
+    public required string id { get; init; }
+    
+    [JsonConverter(typeof(JsonStringEnumConverter))]
+    public required Game game { get; init; }
+    
+    [JsonConverter(typeof(JsonStringEnumConverter))]
+    public required SaveFileProcessingStatus status { get; set; }
+    
+    public string? errorMesage { get; set; }
+    
+    private static readonly JsonSerializerOptions _jsonOptions = new() { WriteIndented = true };
+    public void SaveToFile()
+    {
+        using var metaFile = File.Open(PathHelper.GetMetaFilePath(id), FileMode.CreateNew, FileAccess.Write);
+        JsonSerializer.Serialize(metaFile, this, _jsonOptions);
+    }
+}

ParadoxSaveParser.sln

@@ -4,6 +4,11 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParadoxSaveParser.WebAPI",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParadoxSaveParser.Lib", "ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj", "{53ED0135-9513-4DE2-9187-CF2899F179B3}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SolutionFolder", "SolutionFolder", "{F1D312F1-0620-4E35-8D78-9A2808CDE12C}"
+	ProjectSection(SolutionItems) = preProject
+		.gitignore = .gitignore
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU