Timerix/ParadoxSaveParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

code reformat and cleanup

Browse Source
This commit is contained in:
Timerix 2025-04-05 05:59:22 +05:00
parent 758388cda0
commit e9c7c8f5c1
10 changed files with 357 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ParadoxSaveParser.Lib.Tests/ParadoxSaveParser.Lib.Tests.csproj
View File

@ -11,7 +11,7 @@
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.0"/>
<PackageReference Include="DTLib" Version="1.6.5" />
<PackageReference Include="DTLib" Version="1.6.5"/>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0"/>
<PackageReference Include="NUnit" Version="3.14.0"/>
<PackageReference Include="NUnit.Analyzers" Version="3.9.0"/>
@ -23,7 +23,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj" />
<ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj"/>
</ItemGroup>
</Project>

12
ParadoxSaveParser.Lib.Tests/SearchExpressionTests.cs
View File

@ -8,24 +8,24 @@ namespace ParadoxSaveParser.Lib.Tests;
[TestOf(typeof(ISearchExpression))]
public class SearchExpressionTests
{
byte[] _smallSaveData;
[SetUp]
public void Setup()
{
_smallSaveData = "EU4txt a={ b={ c=0 d=1 e=2 } f=3 }".ToBytes();
}
private byte[] _smallSaveData;
private static JsonSerializerOptions _smallSaveSerializerOptions = new()
private static readonly JsonSerializerOptions _smallSaveSerializerOptions = new()
{
WriteIndented = false,
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
MaxDepth = 1024,
MaxDepth = 1024
};
internal static string JsonToPdx(string json) =>
json.Substring(1, json.Length - 2)
internal static string JsonToPdx(string json)
=> json.Substring(1, json.Length - 2)
.Replace(",", " ").Replace("{", "{ ").Replace("}", " }")
.Replace("\"", "").Replace("[", "").Replace("]", "").Replace(":", "=");

8
ParadoxSaveParser.Lib/BufferedEnumerator.cs
View File

@ -39,11 +39,11 @@ namespace ParadoxSaveParser.Lib;
/// </code>
public class BufferedEnumerator<T> : IEnumerator<LinkedListNode<T>>
{
private IEnumerator<T> _enumerator;
private int _bufferSize;
LinkedList<T> _llist = new();
private readonly int _bufferSize;
private LinkedListNode<T>? _currentNode;
private int _currentNodeIndex = -1;
private readonly IEnumerator<T> _enumerator;
private readonly LinkedList<T> _llist = new();
public BufferedEnumerator(IEnumerator<T> enumerator, int bufferSize)
{
@ -53,7 +53,7 @@ public class BufferedEnumerator<T> : IEnumerator<LinkedListNode<T>>
public bool MoveNext()
{
if(_currentNodeIndex >= _bufferSize / 2)
if (_currentNodeIndex >= _bufferSize / 2)
_llist.RemoveFirst();
while (_llist.Count < _bufferSize && _enumerator.MoveNext())

161
ParadoxSaveParser.Lib/SaveParserEU4.cs
View File

@ -6,17 +6,21 @@ global using System.Text;
namespace ParadoxSaveParser.Lib;
/// <summary>
/// Sequential parser that doesn't cache anything.
/// Sequential parser that doesn't cache anything.
/// </summary>
public class SaveParserEU4
{
protected Stream _saveFile;
private BufferedEnumerator<Token> _tokens;
private ISearchExpression? _searchExprCurrent;
private readonly BufferedEnumerator<Token> _tokens;
/// <param name="savefile">Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive</param>
/// <param name="query">Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM,
/// so you should specify what exactly you want to get from save file</param>
/// <param name="savefile">
/// Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive
/// </param>
/// <param name="query">
/// Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM,
/// so you should specify what exactly you want to get from save file
/// </param>
public SaveParserEU4(Stream savefile, ISearchExpression? query)
{
_tokens = new BufferedEnumerator<Token>(LexTextSave(), 5);
@ -24,57 +28,6 @@ public class SaveParserEU4
_searchExprCurrent = query;
}
protected enum TokenType : byte
{
Invalid,
StringOrNumber,
Equals,
BracketOpen,
BracketClose,
}
protected struct Token
{
public required TokenType type;
public required short column;
public required int line;
public string? value;
public override string ToString()
{
string s;
switch (type)
{
case TokenType.Invalid:
s = "INVALID_TOKEN";
break;
case TokenType.StringOrNumber:
s = value ?? "NULL";
break;
case TokenType.Equals:
s = "=";
break;
case TokenType.BracketOpen:
s = "{";
break;
case TokenType.BracketClose:
s = "}";
break;
default:
throw new ArgumentOutOfRangeException(type.ToString());
}
return $"{line}:{column} '{s}'";
}
}
protected class UnexpectedTokenException : Exception
{
public UnexpectedTokenException(Token token) :
base($"Unexpected token: {token}")
{}
}
protected IEnumerator<Token> LexTextSave()
{
string expectedHeader = "EU4txt";
@ -124,7 +77,7 @@ public class SaveParserEU4
switch (c)
{
case -1:
if(TryCompleteStringToken())
if (TryCompleteStringToken())
yield return strToken;
yield break;
case '\"':
@ -134,17 +87,17 @@ public class SaveParserEU4
case ' ':
case '\t':
case '\r':
if(TryCompleteStringToken())
if (TryCompleteStringToken())
yield return strToken;
break;
case '\n':
if(TryCompleteStringToken())
if (TryCompleteStringToken())
yield return strToken;
line++;
column = 0;
break;
case '=':
if(TryCompleteStringToken())
if (TryCompleteStringToken())
yield return strToken;
yield return new Token
{
@ -153,7 +106,7 @@ public class SaveParserEU4
};
break;
case '{':
if(TryCompleteStringToken())
if (TryCompleteStringToken())
yield return strToken;
yield return new Token
{
@ -162,7 +115,7 @@ public class SaveParserEU4
};
break;
case '}':
if(TryCompleteStringToken())
if (TryCompleteStringToken())
yield return strToken;
yield return new Token
{
@ -173,7 +126,7 @@ public class SaveParserEU4
default:
// Skip control characters, which are invisible and causing frontend bugs.
// I dont know why there are so many of them in strings.
if(c >= 0x20)
if (c >= 0x20)
str.Append((char)c);
break;
}
@ -184,21 +137,21 @@ public class SaveParserEU4
// doesn't move next
private object? ParseValue()
{
Token tok = _tokens.Current.Value;
var tok = _tokens.Current.Value;
switch (tok.type)
{
case TokenType.StringOrNumber:
if(string.IsNullOrEmpty(tok.value))
if (string.IsNullOrEmpty(tok.value))
return string.Empty;
if (tok.value[0] != '-' && !char.IsDigit(tok.value[0]))
return tok.value;
if(tok.value.Contains('.') && Double.TryParse(tok.value, out double d))
if (tok.value.Contains('.') && double.TryParse(tok.value, out double d))
return d;
if (Int64.TryParse(tok.value, out long l))
if (long.TryParse(tok.value, out long l))
return l;
return tok.value;
case TokenType.BracketOpen:
var obj = ParseListOrDict();
object obj = ParseListOrDict();
return obj;
case TokenType.BracketClose:
return null;
@ -212,7 +165,7 @@ public class SaveParserEU4
/// <returns>true if skipped value, false if current token is closing bracket</returns>
private bool SkipValue()
{
Token tok = _tokens.Current.Value;
var tok = _tokens.Current.Value;
if (tok.type == TokenType.BracketOpen)
{
SkipObject();
@ -227,7 +180,7 @@ public class SaveParserEU4
{
while (bracketBalance != 0 && _tokens.MoveNext())
{
Token tok = _tokens.Current.Value;
var tok = _tokens.Current.Value;
if (tok.type == TokenType.BracketOpen)
bracketBalance++;
else if (tok.type == TokenType.BracketClose)
@ -250,15 +203,16 @@ public class SaveParserEU4
private List<object> ParseList()
{
List<object> list = new();
while(true)
while (true)
{
if(!_tokens.MoveNext())
if (!_tokens.MoveNext())
throw new Exception("Unexpected end of file");
object? value = ParseValue();
if (value is null)
break;
list.Add(value);
}
return list;
}
@ -270,7 +224,7 @@ public class SaveParserEU4
// root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count
for (int localIndex = 0; _tokens.MoveNext(); localIndex++)
{
Token tok = _tokens.Current.Value;
var tok = _tokens.Current.Value;
// end of dictionary
if (tok.type == TokenType.BracketClose)
break;
@ -285,13 +239,13 @@ public class SaveParserEU4
continue;
}
if(tok.type != TokenType.StringOrNumber)
if (tok.type != TokenType.StringOrNumber)
throw new UnexpectedTokenException(tok);
string key = tok.value!;
// next token should be `=` or `{`
if(!_tokens.MoveNext())
if (!_tokens.MoveNext())
throw new UnexpectedTokenException(tok);
tok = _tokens.Current.Value;
if (tok.type == TokenType.Equals)
@ -303,7 +257,9 @@ public class SaveParserEU4
// Saves may contain object definition without `=`.
// Example: `map_area_data {` instead of `map_area_data = {`
else if (tok.type != TokenType.BracketOpen)
{
throw new UnexpectedTokenException(tok);
}
ISearchExpression? searchExprNext = null;
if (_searchExprCurrent != null
@ -320,11 +276,12 @@ public class SaveParserEU4
throw new UnexpectedTokenException(_tokens.Current.Value);
_searchExprCurrent = searExpressionPrevious;
if(!dict.TryGetValue(key, out List<object>? list))
if (!dict.TryGetValue(key, out var list))
{
list = new List<object>();
dict.Add(key, list);
}
list.Add(value);
}
@ -336,4 +293,56 @@ public class SaveParserEU4
var root = ParseDict();
return root;
}
protected enum TokenType : byte
{
Invalid,
StringOrNumber,
Equals,
BracketOpen,
BracketClose
}
protected struct Token
{
public required TokenType type;
public required short column;
public required int line;
public string? value;
public override string ToString()
{
string s;
switch (type)
{
case TokenType.Invalid:
s = "INVALID_TOKEN";
break;
case TokenType.StringOrNumber:
s = value ?? "NULL";
break;
case TokenType.Equals:
s = "=";
break;
case TokenType.BracketOpen:
s = "{";
break;
case TokenType.BracketClose:
s = "}";
break;
default:
throw new ArgumentOutOfRangeException(type.ToString());
}
return $"{line}:{column} '{s}'";
}
}
protected class UnexpectedTokenException : Exception
{
public UnexpectedTokenException(Token token) :
base($"Unexpected token: {token}")
{
}
}
}

34
ParadoxSaveParser.Lib/SearchExpression.cs
View File

@ -9,15 +9,15 @@ public interface ISearchExpression
public static class SearchExpressionCompiler
{
private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan<char> chars, int i) =>
chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\');
private static bool CharEqualsAndNotEscaped(char c, ReadOnlySpan<char> chars, int i)
=> chars[i] == c && (i < 1 || chars[i - 1] != '\\') && (i < 2 || chars[i - 2] != '\\');
public static ISearchExpression Compile(ReadOnlySpan<char> query)
{
if(query.IsEmpty)
if (query.IsEmpty)
throw new ArgumentNullException(nameof(query));
if(query[0] is '(')
if (query[0] is '(')
{
var subExprs = new List<ISearchExpression>();
int supExprBegin = 1;
@ -25,9 +25,13 @@ public static class SearchExpressionCompiler
for (int i = supExprBegin; i < query.Length && bracketBalance != 0; i++)
{
if (CharEqualsAndNotEscaped('(', query, i))
{
bracketBalance++;
}
else if (CharEqualsAndNotEscaped(')', query, i))
{
bracketBalance--;
}
else if (bracketBalance == 1 && CharEqualsAndNotEscaped('|', query, i))
{
var subPart = query.Slice(supExprBegin, i - supExprBegin);
@ -37,13 +41,13 @@ public static class SearchExpressionCompiler
}
}
if(query[^1] != ')')
if (query[^1] != ')')
throw new NotImplementedException("Expressions after ')' are not supported");
if (bracketBalance > 0)
throw new Exception("Too many opening brackets");
if (bracketBalance < 0)
throw new Exception("Too many closing brackets");
throw new Exception("Too many closing brackets");
var subPartLast = query.Slice(supExprBegin, query.Length - supExprBegin - 1);
var subExprLast = Compile(subPartLast);
@ -54,24 +58,20 @@ public static class SearchExpressionCompiler
int partBeforePointLength = 0;
while (partBeforePointLength < query.Length)
{
if(CharEqualsAndNotEscaped('.', query, partBeforePointLength))
if (CharEqualsAndNotEscaped('.', query, partBeforePointLength))
break;
partBeforePointLength++;
}
var part = query.Slice(0, partBeforePointLength);
ReadOnlySpan<char> remaining = default;
if (partBeforePointLength < query.Length)
remaining = query.Slice(partBeforePointLength + 1);
if (part is "*")
{
return new AnyMatchExpression(remaining.IsEmpty ? null : Compile(remaining));
}
if (part is "*") return new AnyMatchExpression(remaining.IsEmpty ? null : Compile(remaining));
for (int j = 0; j < part.Length; j++)
{
if(CharEqualsAndNotEscaped('*', part, j))
if (CharEqualsAndNotEscaped('*', part, j))
throw new NotImplementedException("pattern matching other than '*' is not implemented yet");
}
if (part[0] is '[')
{
@ -97,12 +97,8 @@ public static class SearchExpressionCompiler
public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
{
foreach (var e in subExprs)
{
if(e.DoesMatch(args, out nextSearchExpression))
{
if (e.DoesMatch(args, out nextSearchExpression))
return true;
}
}
nextSearchExpression = null;
return false;

10
ParadoxSaveParser.WebAPI/Config.cs
View File

@ -5,9 +5,9 @@ namespace ParadoxSaveParser.WebAPI;
public class Config
{
public const int ActualVersion = 1;
public string BaseUrl = "http://127.0.0.1:5226/";
public int Version = ActualVersion;
public string BaseUrl = "http://127.0.0.1:5226/";
public static Config FromDtsod(DtsodV23 d)
{
@ -18,16 +18,16 @@ public class Config
};
if (cfg.Version < ActualVersion)
throw new Exception($"config is obsolete (config v{cfg.Version} < program v{ActualVersion})");
if(cfg.Version > ActualVersion)
if (cfg.Version > ActualVersion)
throw new Exception($"program is obsolete (config v{cfg.Version} > program v{ActualVersion})");
return cfg;
}
public DtsodV23 ToDtsod() =>
new()
public DtsodV23 ToDtsod()
=> new()
{
{ "version", Version },
{ "baseUrl", BaseUrl },
{ "baseUrl", BaseUrl }
};
public override string ToString() => ToDtsod().ToString();

4
ParadoxSaveParser.WebAPI/ParadoxSaveParser.WebAPI.csproj
View File

@ -9,10 +9,10 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj" />
<ProjectReference Include="..\ParadoxSaveParser.Lib\ParadoxSaveParser.Lib.csproj"/>
</ItemGroup>
<ItemGroup>
<PackageReference Include="DTLib.Web" Version="1.2.2" />
<PackageReference Include="DTLib.Web" Version="1.2.2"/>
</ItemGroup>
</Project>

3
ParadoxSaveParser.WebAPI/PathHelper.cs
View File

@ -4,7 +4,10 @@ public static class PathHelper
{
public static readonly IOPath DATA_DIR = "data";
public static readonly IOPath SAVES_DIR = Path.Concat(DATA_DIR, "saves");
public static IOPath GetMetaFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".meta.json");
public static IOPath GetSaveFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".eu4");
public static IOPath GetParsedSaveFilePath(string save_id) => Path.Concat(SAVES_DIR, save_id + ".parsed.json");
}

22
ParadoxSaveParser.WebAPI/Program.HttpHelpers.cs
View File

@ -53,6 +53,17 @@ public partial class Program
return error.StatusCode;
}
internal static ValueOrError<string> GetRequestQueryValue(HttpListenerContext ctx, string paramName)
{
string[]? values = ctx.Request.QueryString.GetValues(paramName);
string? value = values?.FirstOrDefault();
if (string.IsNullOrEmpty(value))
return new ErrorMessage(HttpStatusCode.BadRequest,
$"No request parameter '{paramName}' provided");
return value;
}
public record ErrorMessage
{
public ErrorMessage(HttpStatusCode statusCode, string message)
@ -83,15 +94,4 @@ public partial class Program
public static implicit operator ValueOrError<T>(ErrorMessage e) => new(default, e);
}
internal static ValueOrError<string> GetRequestQueryValue(HttpListenerContext ctx, string paramName)
{
string[]? values = ctx.Request.QueryString.GetValues(paramName);
string? value = values?.FirstOrDefault();
if (string.IsNullOrEmpty(value))
return new ErrorMessage(HttpStatusCode.BadRequest,
$"No request parameter '{paramName}' provided");
return value;
}
}

13
ParadoxSaveParser.WebAPI/SaveFileMetadata.cs
View File

@ -4,16 +4,23 @@ namespace ParadoxSaveParser.WebAPI;
public enum SaveFileProcessingStatus
{
Initialized, Uploading, Uploaded, Parsing, SavingResults, Done
Initialized,
Uploading,
Uploaded,
Parsing,
SavingResults,
Done
}
public enum Game
{
Unknown, EU4
Unknown,
EU4
}
public class SaveFileMetadata
{
private static readonly JsonSerializerOptions _jsonOptions = new() { WriteIndented = true };
public required string id { get; init; }
[JsonConverter(typeof(JsonStringEnumConverter))]
@ -22,8 +29,6 @@ public class SaveFileMetadata
[JsonConverter(typeof(JsonStringEnumConverter))]
public required SaveFileProcessingStatus status { get; set; }
private static readonly JsonSerializerOptions _jsonOptions = new() { WriteIndented = true };
public void SaveToFile()
{
using var metaFile = File.OpenWrite(PathHelper.GetMetaFilePath(id));