diff --git a/ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj b/ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj index f063c0b..00ebf29 100644 --- a/ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj +++ b/ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj @@ -4,5 +4,8 @@ disable enable + + + diff --git a/ParadoxSaveParser.Lib/SaveParserEU4.cs b/ParadoxSaveParser.Lib/SaveParserEU4.cs index ed82fc3..89d12ab 100644 --- a/ParadoxSaveParser.Lib/SaveParserEU4.cs +++ b/ParadoxSaveParser.Lib/SaveParserEU4.cs @@ -2,6 +2,7 @@ global using System.Collections.Generic; global using System.IO; global using System.Text; +using Microsoft.Extensions.ObjectPool; namespace ParadoxSaveParser.Lib; @@ -10,9 +11,13 @@ namespace ParadoxSaveParser.Lib; /// public class SaveParserEU4 { - protected Stream _saveFile; - private ISearchExpression? _searchExprCurrent; + protected readonly Stream _saveFile; private readonly BufferedEnumerator _tokens; + private readonly ObjectPool _stringBuilderPool; + private ISearchExpression? _searchExprCurrent; + + public int SBPoolGetCount = 0; + public int SBPoolReturnCount = 0; /// /// Uncompressed stream of gamestate file which can be extracted from save archive @@ -23,9 +28,16 @@ public class SaveParserEU4 /// public SaveParserEU4(Stream savefile, ISearchExpression? query) { - _tokens = new BufferedEnumerator(LexTextSave(), 5); _saveFile = savefile; _searchExprCurrent = query; + const int tokenBufSize = 5; + _tokens = new BufferedEnumerator(LexTextSave(), tokenBufSize); + _stringBuilderPool = new DefaultObjectPool( + new StringBuilderPooledObjectPolicy + { + InitialCapacity = tokenBufSize * 13, + MaximumRetainedCapacity = tokenBufSize * 13, + }); } protected IEnumerator LexTextSave() @@ -37,7 +49,8 @@ public class SaveParserEU4 if (headStr != expectedHeader) throw new Exception($"Invalid gamestate header. Expected '{expectedHeader}', got '{headStr}'."); - StringBuilder str = new(); + StringBuilder strb = _stringBuilderPool.Get(); + SBPoolGetCount++; int line = 2; int column = 0; bool isQuoteOpen = false; @@ -46,7 +59,8 @@ public class SaveParserEU4 { type = TokenType.Invalid, column = -1, - line = -1 + line = -1, + value = null, }; bool TryCompleteStringToken() @@ -55,17 +69,18 @@ public class SaveParserEU4 return false; // strings in quotes may be empty - if (!isStrInQuotes && (str.Length <= 0 || str[0] == '#')) + if (!isStrInQuotes && (strb.Length <= 0 || strb[0] == '#')) return false; strToken = new Token { type = TokenType.StringOrNumber, - column = (short)(column - str.Length), + column = (short)(column - strb.Length), line = line, - value = str.ToString() + value = strb, }; - str.Clear(); + strb = _stringBuilderPool.Get(); + SBPoolGetCount++; isStrInQuotes = false; return true; } @@ -79,6 +94,8 @@ public class SaveParserEU4 case -1: if (TryCompleteStringToken()) yield return strToken; + _stringBuilderPool.Return(strb); + SBPoolReturnCount++; yield break; case '\"': isQuoteOpen = !isQuoteOpen; @@ -127,10 +144,13 @@ public class SaveParserEU4 // Skip control characters, which are invisible and causing frontend bugs. // I dont know why there are so many of them in strings. if (c >= 0x20) - str.Append((char)c); + strb.Append((char)c); break; } } + + _stringBuilderPool.Return(strb); + SBPoolReturnCount++; } @@ -141,15 +161,16 @@ public class SaveParserEU4 switch (tok.type) { case TokenType.StringOrNumber: - if (string.IsNullOrEmpty(tok.value)) - return string.Empty; - if (tok.value[0] != '-' && !char.IsDigit(tok.value[0])) - return tok.value; - if (tok.value.Contains('.') && double.TryParse(tok.value, out double d)) + string tokStr = tok.value!.ToString(); + _stringBuilderPool.Return(tok.value); + SBPoolReturnCount++; + if (tokStr[0] != '-' && !char.IsDigit(tokStr[0])) + return tokStr; + if (tokStr.Contains('.') && double.TryParse(tokStr, out double d)) return d; - if (long.TryParse(tok.value, out long l)) + if (long.TryParse(tokStr, out long l)) return l; - return tok.value; + return tokStr; case TokenType.BracketOpen: object obj = ParseListOrDict(); return obj; @@ -166,13 +187,22 @@ public class SaveParserEU4 private bool SkipValue() { var tok = _tokens.Current.Value; - if (tok.type == TokenType.BracketOpen) + switch (tok.type) { - SkipObject(); - return true; + case TokenType.BracketOpen: + SkipObject(); + return true; + case TokenType.StringOrNumber: + _stringBuilderPool.Return(tok.value!); + SBPoolReturnCount++; + return true; + case TokenType.Equals: + return true; + case TokenType.BracketClose: + return false; + default: + throw new UnexpectedTokenException(tok); } - - return tok.type != TokenType.BracketClose; } // skips all tokens inside curly braces block @@ -185,6 +215,11 @@ public class SaveParserEU4 bracketBalance++; else if (tok.type == TokenType.BracketClose) bracketBalance--; + else if (tok.type == TokenType.StringOrNumber) + { + _stringBuilderPool.Return(tok.value!); + SBPoolReturnCount++; + } } } @@ -242,7 +277,7 @@ public class SaveParserEU4 if (tok.type != TokenType.StringOrNumber) throw new UnexpectedTokenException(tok); - string key = tok.value!; + var keySB = tok.value!; // next token should be `=` or `{` if (!_tokens.MoveNext()) @@ -263,9 +298,11 @@ public class SaveParserEU4 ISearchExpression? searchExprNext = null; if (_searchExprCurrent != null - && !_searchExprCurrent.DoesMatch(new SearchArgs(key, localIndex), out searchExprNext)) + && !_searchExprCurrent.DoesMatch(new SearchArgs(localIndex, keySB), out searchExprNext)) { SkipValue(); + _stringBuilderPool.Return(keySB); + SBPoolReturnCount++; continue; } @@ -276,10 +313,13 @@ public class SaveParserEU4 throw new UnexpectedTokenException(_tokens.Current.Value); _searchExprCurrent = searExpressionPrevious; - if (!dict.TryGetValue(key, out var list)) + string keyStr = keySB.ToString(); + _stringBuilderPool.Return(keySB); + SBPoolReturnCount++; + if (!dict.TryGetValue(keyStr, out var list)) { list = new List(); - dict.Add(key, list); + dict.Add(keyStr, list); } list.Add(value); @@ -308,7 +348,7 @@ public class SaveParserEU4 public required TokenType type; public required short column; public required int line; - public string? value; + public StringBuilder? value; public override string ToString() { @@ -319,7 +359,9 @@ public class SaveParserEU4 s = "INVALID_TOKEN"; break; case TokenType.StringOrNumber: - s = value ?? "NULL"; + if (value == null || value.Length == 0) + s = "NULL"; + else s = value.ToString(); break; case TokenType.Equals: s = "="; diff --git a/ParadoxSaveParser.Lib/SearchExpression.cs b/ParadoxSaveParser.Lib/SearchExpression.cs index 87ae6d1..a3c66a7 100644 --- a/ParadoxSaveParser.Lib/SearchExpression.cs +++ b/ParadoxSaveParser.Lib/SearchExpression.cs @@ -1,6 +1,25 @@ namespace ParadoxSaveParser.Lib; -public record SearchArgs(string key, int localIndex); +public record SearchArgs +{ + public readonly string KeyStr; + public readonly StringBuilder? KeySB; + public readonly int LocalIndex; + + public SearchArgs(int localIndex, string keyStr) + { + KeyStr = keyStr; + KeySB = null; + LocalIndex = localIndex; + } + + public SearchArgs(int localIndex, StringBuilder keySb) + { + KeyStr = string.Empty; + KeySB = keySb; + LocalIndex = localIndex; + } +} public interface ISearchExpression { @@ -109,7 +128,7 @@ public static class SearchExpressionCompiler { public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression) { - if (args.localIndex == index) + if (args.LocalIndex == index) { nextSearchExpression = next; return true; @@ -124,7 +143,7 @@ public static class SearchExpressionCompiler { public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression) { - if (args.key == key) + if ((args.KeySB != null && args.KeySB.Equals(key)) || args.KeyStr == key) { nextSearchExpression = next; return true; diff --git a/ParadoxSaveParser.WebAPI/Program.cs b/ParadoxSaveParser.WebAPI/Program.cs index ace24e0..155806d 100644 --- a/ParadoxSaveParser.WebAPI/Program.cs +++ b/ParadoxSaveParser.WebAPI/Program.cs @@ -12,6 +12,7 @@ global using Directory = DTLib.Filesystem.Directory; global using File = DTLib.Filesystem.File; global using Path = DTLib.Filesystem.Path; using System.Collections.Concurrent; +using System.Diagnostics; using System.IO; using System.Text.Encodings.Web; using DTLib.Dtsod; @@ -54,6 +55,21 @@ public static partial class Program try { + Stopwatch stopwatch = new(); + using var save = File.OpenRead("data/gamestate"); + stopwatch.Start(); + var parser = new SaveParserEU4(save, SearchExpressionCompiler.Compile("saved_event_target")); + var result = parser.Parse(); + stopwatch.Stop(); + using (var resultFile = File.OpenWrite("data/parsed.json")) + { + JsonSerializer.Serialize(resultFile, result, _saveSerializerOptions); + } + Console.WriteLine($"get: {parser.SBPoolGetCount} return: {parser.SBPoolReturnCount} " + + $"delta: {parser.SBPoolGetCount - parser.SBPoolReturnCount}"); + Console.WriteLine(stopwatch.Elapsed); + return; + // config if (!File.Exists(_configPath)) {