Compare commits

..

No commits in common. "05c6bdf0086ea2aee81d925092d2e2b0046bc453" and "5d88ad49c0b1c5cdad87fdbde8e6c1da4a01249e" have entirely different histories.

6 changed files with 162 additions and 395 deletions

View File

@ -5,24 +5,15 @@ global using System.Text;
namespace ParadoxSaveParser.Lib;
/// <summary>
/// Sequential parser that doesn't cache anything.
/// </summary>
public class SaveParserEU4
public class Parser
{
protected Stream _saveFile;
private BufferedEnumerator<Token> _tokens;
private SearchExpression _query;
private int _currentDepth;
/// <param name="savefile">Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive</param>
/// <param name="query">Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM,
/// so you should specify what exactly you want to get from save file</param>
public SaveParserEU4(Stream savefile, SearchExpression query)
public Parser(Stream savefile)
{
_tokens = new BufferedEnumerator<Token>(LexTextSave(), 5);
_tokens = new BufferedEnumerator<Token>(Lex(), 5);
_saveFile = savefile;
_query = query;
}
protected enum TokenType : byte
@ -69,21 +60,14 @@ public class SaveParserEU4
}
}
protected class UnexpectedTokenException : Exception
{
public UnexpectedTokenException(Token token) :
base($"Unexpected token: {token}")
{}
}
protected IEnumerator<Token> LexTextSave()
protected IEnumerator<Token> Lex()
{
string expectedHeader = "EU4txt";
byte[] headBytes = new byte[expectedHeader.Length];
_saveFile.ReadExactly(headBytes);
string headStr = Encoding.UTF8.GetString(headBytes);
if (headStr != expectedHeader)
throw new Exception($"Invalid gamestate header. Expected '{expectedHeader}', got '{headStr}'.");
throw new Exception($"Invalid gamestate header: '{headStr}'");
StringBuilder str = new();
int line = 2;
@ -101,8 +85,7 @@ public class SaveParserEU4
{
if (isQuoteOpen)
return false;
// strings in quotes may be empty
// strings in quotes can be empty
if (!isStrInQuotes && (str.Length <= 0 || str[0] == '#'))
return false;
@ -181,6 +164,13 @@ public class SaveParserEU4
}
}
protected class UnexpectedTokenException : Exception
{
public UnexpectedTokenException(Token token) :
base($"Unexpected token: {token}")
{}
}
// doesn't move next
private object? ParseValue()
@ -199,10 +189,7 @@ public class SaveParserEU4
return l;
return tok.value;
case TokenType.BracketOpen:
_currentDepth++;
var obj = ParseListOrDict();
_currentDepth--;
return obj;
return ParseListOrDict();
case TokenType.BracketClose:
return null;
default:
@ -210,34 +197,6 @@ public class SaveParserEU4
}
}
// skips next value
/// <returns>true if skipped value, false if current token is closing bracket</returns>
private bool SkipValue()
{
Token tok = _tokens.Current.Value;
if (tok.type == TokenType.BracketOpen)
{
SkipObject();
return true;
}
return tok.type != TokenType.BracketClose;
}
// skips all tokens inside curly braces block
private void SkipObject(int bracketBalance = 1)
{
while (bracketBalance != 0 && _tokens.MoveNext())
{
Token tok = _tokens.Current.Value;
if (tok.type == TokenType.BracketOpen)
bracketBalance++;
else if (tok.type == TokenType.BracketClose)
bracketBalance--;
}
}
// doesn't move next
private object ParseListOrDict()
{
@ -265,13 +224,13 @@ public class SaveParserEU4
return list;
}
// moves next
private Dictionary<string, List<object>> ParseDict()
{
Dictionary<string, List<object>> dict = new();
// root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count
for (int localIndex = 0; _tokens.MoveNext(); localIndex++)
while (_tokens.MoveNext())
{
Token tok = _tokens.Current.Value;
// end of dictionary
@ -284,7 +243,16 @@ public class SaveParserEU4
// { } { } { }`
if (tok.type == TokenType.BracketOpen)
{
SkipObject();
int bracketBalance = 1;
while (bracketBalance != 0 && _tokens.MoveNext())
{
tok = _tokens.Current.Value;
if (tok.type == TokenType.BracketOpen)
bracketBalance++;
else if (tok.type == TokenType.BracketClose)
bracketBalance--;
}
continue;
}
@ -308,12 +276,6 @@ public class SaveParserEU4
else if (tok.type != TokenType.BracketOpen)
throw new UnexpectedTokenException(tok);
if (!_query.DoesMatch(new SearchArgs(key, _currentDepth, localIndex)))
{
SkipValue();
continue;
}
object? value = ParseValue();
if (value is null)
throw new UnexpectedTokenException(_tokens.Current.Value);
@ -332,6 +294,8 @@ public class SaveParserEU4
public Dictionary<string, List<object>> Parse()
{
var root = ParseDict();
if (root.Count == 0)
throw new Exception("Save file is empty");
return root;
}
}

View File

@ -1,139 +0,0 @@
using System.Diagnostics;
using System.Linq;
namespace ParadoxSaveParser.Lib;
public record SearchArgs(string key, int currentDepth, int localIndex);
public interface ISearchExpression
{
bool DoesMatch(SearchArgs args);
}
public class SearchExpression : ISearchExpression
{
private List<ISearchExpression> _compiledExpression;
private int _expressionDepth;
private SearchExpression(List<ISearchExpression> compiledExpression, int expressionDepth)
{
_compiledExpression = compiledExpression;
_expressionDepth = expressionDepth;
}
public bool DoesMatch(SearchArgs args)
{
int index = args.currentDepth - _expressionDepth;
if (index < 0 || index >= _compiledExpression.Count)
return true;
return _compiledExpression[index].DoesMatch(args);
}
private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan<char> chars, int i) =>
chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\');
public static SearchExpression Parse(string query) => ParseInternal(query, 0);
private static SearchExpression ParseInternal(ReadOnlySpan<char> query, int expressionDepth)
{
var compiledExpression = new List<ISearchExpression>();
ISearchExpression exprPart;
int partBegin = 0;
int bracketBalance = 0;
int expressionDepthIncrement = 0;
for (int i = 0; i < query.Length; i++)
{
if (CharEqualsAndNotEscaped('(', query, i))
bracketBalance++;
else if (CharEqualsAndNotEscaped(')', query, i))
bracketBalance--;
else if (bracketBalance == 0 && CharEqualsAndNotEscaped('.', query, i))
{
var part = query.Slice(partBegin, i - partBegin);
expressionDepthIncrement++;
exprPart = ParsePart(part, query, partBegin,
expressionDepth + expressionDepthIncrement);
compiledExpression.Add(exprPart);
partBegin = i + 1;
}
}
exprPart = ParsePart(query.Slice(partBegin), query, partBegin,
expressionDepth + expressionDepthIncrement);
compiledExpression.Add(exprPart);
return new SearchExpression(compiledExpression, expressionDepth);
}
private static ISearchExpression ParsePart(ReadOnlySpan<char> part,
ReadOnlySpan<char> query, int partBegin, int expressionDepth)
{
if (part is "*")
{
return new AnyMatchExpression();
}
if (CharEqualsAndNotEscaped('[', query, partBegin))
{
part = part.Slice(1, part.Length - 2);
return new IndexMatchExpression(int.Parse(part));
}
if(part[0] is '(')
{
var subExprs = new List<ISearchExpression>();
ISearchExpression subExpr;
part = part.Slice(1, part.Length - 2);
int supExprBegin = 0;
for (int j = 0; j < part.Length; j++)
{
if (CharEqualsAndNotEscaped('|', part, j))
{
subExpr = ParseInternal(part.Slice(supExprBegin, j - supExprBegin),
expressionDepth);
subExprs.Add(subExpr);
supExprBegin = j + 1;
}
}
subExpr = ParseInternal(part.Slice(supExprBegin), expressionDepth);
subExprs.Add(subExpr);
return new MultipleMatchExpression(subExprs);
}
return new ExactMatchExpression(part.ToString());
}
private record AnyMatchExpression : ISearchExpression
{
public bool DoesMatch(SearchArgs args) => true;
}
private record MultipleMatchExpression(List<ISearchExpression> subExprs) : ISearchExpression
{
public bool DoesMatch(SearchArgs args)
{
foreach (var e in subExprs)
{
if(e.DoesMatch(args))
return true;
}
return false;
}
}
private record IndexMatchExpression(int index) : ISearchExpression
{
public bool DoesMatch(SearchArgs args) => args.localIndex == index;
}
private record ExactMatchExpression(string key) : ISearchExpression
{
public bool DoesMatch(SearchArgs args) => args.key == key;
}
}

View File

@ -13,6 +13,23 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="DTLib.Web" Version="1.2.2" />
<PackageReference Include="DTLib.Web" Version="1.2.1" />
</ItemGroup>
<ItemGroup>
<Compile Remove="data\**" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Remove="data\**" />
</ItemGroup>
<ItemGroup>
<Content Remove="data\**" />
</ItemGroup>
<ItemGroup>
<None Remove="data\**" />
<None Remove="Properties\launchSettings.json" />
</ItemGroup>
</Project>

View File

@ -1,125 +0,0 @@
using System.IO.Compression;
using System.Linq;
using System.Net;
using DTLib.Extensions;
namespace ParadoxSaveParser.WebAPI;
public partial class Program
{
// ReSharper disable once NotAccessedPositionalProperty.Global
public record ErrorMessage(string errorMessage);
private static async Task<HttpStatusCode> ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode,
object response)
{
await JsonSerializer.SerializeAsync(ctx.Response.OutputStream, response, response.GetType(),
JsonSerializerOptions.Default, _mainCancel.Token);
ctx.Response.StatusCode = (int)statusCode;
return statusCode;
}
private static async Task<HttpStatusCode> ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode,
string response)
{
await ctx.Response.OutputStream.WriteAsync(response.ToBytes(), _mainCancel.Token);
ctx.Response.StatusCode = (int)statusCode;
return statusCode;
}
private static async Task<HttpStatusCode> UploadSaveHandler(HttpListenerContext ctx)
{
string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault();
if (contentType != "application/octet-stream")
return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
new ErrorMessage($"Invalid request Content-Type: '{contentType}'"));
string saveId = Guid.NewGuid().ToString();
IOPath metaFilePath = PathHelper.GetMetaFilePath(saveId);
if (File.Exists(metaFilePath))
return await ReturnResponse(ctx, HttpStatusCode.InternalServerError,
new ErrorMessage($"Guid collision! file' {metaFilePath}' already exists."));
var meta = new SaveFileMetadata
{ id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized, };
if (!_saveMetadataStorage.TryAdd(saveId, meta))
return await ReturnResponse(ctx, HttpStatusCode.InternalServerError,
new ErrorMessage($"Guid collision! Can't create metadata with id {saveId}"));
meta.status = SaveFileProcessingStatus.Uploading;
IOPath saveFilePath = PathHelper.GetSaveFilePath(meta.id);
await using var saveFile = File.OpenWrite(saveFilePath);
await using var remoteStream = ctx.Request.InputStream;
await remoteStream.CopyToAsync(saveFile, _mainCancel.Token);
meta.status = SaveFileProcessingStatus.Uploaded;
return await ReturnResponse(ctx, HttpStatusCode.OK, saveId);
}
private static (SaveFileMetadata? meta, ErrorMessage? errorMesage) GetMetaFromRequestId(HttpListenerContext ctx,
string requestParamName)
{
var ids = ctx.Request.QueryString.GetValues(requestParamName);
string? id = ids?.FirstOrDefault();
if (string.IsNullOrEmpty(id))
return (null, new ErrorMessage($"No request parameter '{requestParamName}' provided"));
if (!_saveMetadataStorage.TryGetValue(id, out var meta))
return (null, new ErrorMessage($"Save with {id} not found"));
return (meta, null);
}
private static async Task<HttpStatusCode> GetSaveStatusHandler(HttpListenerContext ctx)
{
var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id");
if (errorMessage is not null)
return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage);
return await ReturnResponse(ctx, HttpStatusCode.OK, meta!);
}
private static async Task<HttpStatusCode> ParseSaveEU4Handler(HttpListenerContext ctx)
{
var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id");
if (errorMessage is not null)
return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage);
//TODO: get actual query
string searchQuery = "";
try
{
using var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(meta!.id).Str, ZipArchiveMode.Read);
var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
if (zipEntry is null)
return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
new ErrorMessage("Invalid save format: no 'gamestate' file found"));
string extractedGamestatePath = PathHelper.GetSaveFilePath(meta.id) + ".gamestate";
zipEntry.ExtractToFile(extractedGamestatePath, true);
var gamestateStream = File.OpenRead(extractedGamestatePath);
meta.status = SaveFileProcessingStatus.Parsing;
var se = SearchExpression.Parse(searchQuery);
var parser = new SaveParserEU4(gamestateStream, se);
var result = parser.Parse();
meta.status = SaveFileProcessingStatus.SavingResults;
IOPath resultFilePath = PathHelper.GetParsedSaveFilePath(meta.id);
await using var resultFile = File.OpenWrite(resultFilePath);
await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions, _mainCancel.Token);
meta.status = SaveFileProcessingStatus.Done;
meta.SaveToFile();
}
catch (Exception ex)
{
string errorMesage = ex.ToStringDemystified();
_loggerRoot.LogWarn(nameof(ParseSaveEU4Handler), errorMesage);
return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
new ErrorMessage(errorMesage));
}
return await ReturnResponse(ctx, HttpStatusCode.OK, meta);
}
}

View File

@ -13,6 +13,9 @@ global using File = DTLib.Filesystem.File;
global using Path = DTLib.Filesystem.Path;
using System.Collections.Concurrent;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text.Encodings.Web;
using DTLib.Dtsod;
using DTLib.Extensions;
@ -21,15 +24,11 @@ using DTLib.Web.Routes;
namespace ParadoxSaveParser.WebAPI;
public partial class Program
public class Program
{
private static readonly IOPath _configPath = "./config.dtsod";
private static Config _config = new();
private static readonly ILogger _loggerRoot = new CompositeLogger(
new ConsoleLogger(),
new FileLogger("logs", "ParadoxSaveParser.WebAPI"));
private static readonly ILogger _loggerRoot = new ConsoleLogger();
private static readonly CancellationTokenSource _mainCancel = new();
private static ConcurrentDictionary<string, SaveFileMetadata> _saveMetadataStorage = new();
@ -40,71 +39,14 @@ public partial class Program
MaxDepth = 1024,
};
static void TestSearchExpression(Stream saveStream, TestCase tc)
{
saveStream.Seek(0, SeekOrigin.Begin);
var se = SearchExpression.Parse(tc.q);
var parser = new SaveParserEU4(saveStream, se);
var rootNode = parser.Parse();
string json = JsonSerializer.Serialize(rootNode, _saveSerializerOptions);
string pdx = json.Substring(1, json.Length - 2)
.Replace(",", " ").Replace("{", "{ ").Replace("}", " }")
.Replace("\"", "").Replace("[", "").Replace("]", "").Replace(":", "=");
if(pdx == tc.a)
{
Console.WriteLine($"[OK] q:'{tc.q}' a:'{tc.a}'");
}
else
{
Console.WriteLine($"[Error] q:'{tc.q}' a:'{tc.a}' r:'{pdx}'");
}
}
record TestCase(string q, string a);
public static void Main(string[] args)
{
using var saveStream = new MemoryStream(
"EU4txt a={ b={ c=0 d=1 e=2 } f=3 }".ToBytes(),
false);
TestCase[] testCases = [
new("a",
"a={ b={ c=0 d=1 e=2 } f=3 }"),
new("a.*",
"a={ b={ c=0 d=1 e=2 } f=3 }"),
new("a.b",
"a={ b={ c=0 d=1 e=2 } }"),
new("a.[0].c",
"a={ b={ c=0 } }"),
new("a.[1]",
"a={ f=3 }"),
new("a.b.(c|d)",
"a={ b={ c=0 d=1 } }"),
new("a.(b.e|f)",
"a={ b={ e=2 } f=3 }"),
];
foreach (var test in testCases)
{
TestSearchExpression(saveStream, test);
}
/*
Console.InputEncoding = Encoding.UTF8;
Console.OutputEncoding = Encoding.UTF8;
Console.CursorVisible = false;
ContextLogger logger = new ContextLogger(nameof(Main), _loggerRoot);
Console.CancelKeyPress += (_, e) =>
Console.CancelKeyPress += (_, _) =>
{
e.Cancel = true;
logger.LogInfo("Ctrl+C Pressed");
_mainCancel.Cancel();
};
@ -140,7 +82,6 @@ public partial class Program
{
logger.LogError(ex.ToStringDemystified());
}
*/
}
public static void PrepareLocalFiles()
@ -164,4 +105,110 @@ public partial class Program
}
}
public record ErrorMessage(string errorMessage);
private static async Task<HttpStatusCode>ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode, object response)
{
await JsonSerializer.SerializeAsync(ctx.Response.OutputStream, response, response.GetType(),
JsonSerializerOptions.Default, _mainCancel.Token);
ctx.Response.StatusCode = (int)statusCode;
return statusCode;
}
private static async Task<HttpStatusCode>ReturnResponse(HttpListenerContext ctx, HttpStatusCode statusCode, string response)
{
await ctx.Response.OutputStream.WriteAsync(response.ToBytes(), _mainCancel.Token);
ctx.Response.StatusCode = (int)statusCode;
return statusCode;
}
private static async Task<HttpStatusCode>UploadSaveHandler(HttpListenerContext ctx)
{
string? contentType = ctx.Request.Headers.GetValues("Content-Type")?.FirstOrDefault();
if (contentType != "application/octet-stream")
return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
new ErrorMessage($"Invalid request Content-Type: '{contentType}'"));
string saveId = Guid.NewGuid().ToString();
IOPath metaFilePath = PathHelper.GetMetaFilePath(saveId);
if (File.Exists(metaFilePath))
return await ReturnResponse(ctx, HttpStatusCode.InternalServerError,
new ErrorMessage($"Guid collision! file' {metaFilePath}' already exists."));
var meta = new SaveFileMetadata { id = saveId, game = Game.EU4, status = SaveFileProcessingStatus.Initialized, };
if (!_saveMetadataStorage.TryAdd(saveId, meta))
return await ReturnResponse(ctx, HttpStatusCode.InternalServerError,
new ErrorMessage($"Guid collision! Can't create metadata with id {saveId}"));
meta.status = SaveFileProcessingStatus.Uploading;
IOPath saveFilePath = PathHelper.GetSaveFilePath(meta.id);
await using var saveFile = File.OpenWrite(saveFilePath);
await using var remoteStream = ctx.Request.InputStream;
await remoteStream.CopyToAsync(saveFile, _mainCancel.Token);
meta.status = SaveFileProcessingStatus.Uploaded;
return await ReturnResponse(ctx, HttpStatusCode.OK, saveId);
}
private static (SaveFileMetadata? meta, ErrorMessage? errorMesage) GetMetaFromRequestId(HttpListenerContext ctx, string requestParamName)
{
var ids = ctx.Request.QueryString.GetValues(requestParamName);
string? id = ids?.FirstOrDefault();
if (string.IsNullOrEmpty(id))
return (null, new ErrorMessage($"No request parameter '{requestParamName}' provided"));
if (!_saveMetadataStorage.TryGetValue(id, out var meta))
return (null,new ErrorMessage($"Save with {id} not found"));
return (meta, null);
}
private static async Task<HttpStatusCode>GetSaveStatusHandler(HttpListenerContext ctx)
{
var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id");
if(errorMessage is not null)
return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage);
return await ReturnResponse(ctx, HttpStatusCode.OK, meta!);
}
private static async Task<HttpStatusCode>ParseSaveEU4Handler(HttpListenerContext ctx)
{
var (meta, errorMessage) = GetMetaFromRequestId(ctx, "id");
if(errorMessage is not null)
return await ReturnResponse(ctx, HttpStatusCode.InternalServerError, errorMessage);
try
{
using var zipArchive = ZipFile.Open(PathHelper.GetSaveFilePath(meta!.id).Str, ZipArchiveMode.Read);
var zipEntry = zipArchive.Entries.FirstOrDefault(e => e.Name == "gamestate");
if (zipEntry is null)
return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
new ErrorMessage("Invalid save format: no 'gamestate' file found"));
string extractedGamestatePath = PathHelper.GetSaveFilePath(meta.id) + ".gamestate";
zipEntry.ExtractToFile(extractedGamestatePath);
var gamestateStream = File.OpenRead(extractedGamestatePath);
meta.status = SaveFileProcessingStatus.Parsing;
var parser = new Parser(gamestateStream);
var result = parser.Parse();
meta.status = SaveFileProcessingStatus.SavingResults;
IOPath resultFilePath = PathHelper.GetParsedSaveFilePath(meta.id);
await using var resultFile = File.OpenWrite(resultFilePath);
await JsonSerializer.SerializeAsync(resultFile, result, _saveSerializerOptions, _mainCancel.Token);
meta.status = SaveFileProcessingStatus.Done;
meta.SaveToFile();
}
catch (Exception ex)
{
string errorMesage = ex.ToStringDemystified();
_loggerRoot.LogWarn(nameof(ParseSaveEU4Handler), errorMesage);
return await ReturnResponse(ctx, HttpStatusCode.BadRequest,
new ErrorMessage(errorMesage));
}
return await ReturnResponse(ctx, HttpStatusCode.OK, meta);
}
}

View File

@ -1,3 +1,6 @@
DTLib.Web:
Add elapsed time to response status log message: `responded 200 (OK) in 0.03 s`
Main:
Add temporary files deletion