Timerix/ParadoxSaveParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

StringBuilderPool reduced memory allocation in 3000 times

Browse Source
This commit is contained in:
Timerix 2025-04-05 08:27:58 +05:00
parent b0fefbf667
commit f3106769d9
4 changed files with 111 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ParadoxSaveParser.Lib/ParadoxSaveParser.Lib.csproj
View File

@ -4,5 +4,8 @@
<ImplicitUsings>disable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.ObjectPool" Version="9.0.3" />
</ItemGroup>
</Project>

98
ParadoxSaveParser.Lib/SaveParserEU4.cs
View File

@ -2,6 +2,7 @@
global using System.Collections.Generic;
global using System.IO;
global using System.Text;
using Microsoft.Extensions.ObjectPool;
namespace ParadoxSaveParser.Lib;
@ -10,9 +11,13 @@ namespace ParadoxSaveParser.Lib;
/// </summary>
public class SaveParserEU4
{
protected Stream _saveFile;
private ISearchExpression? _searchExprCurrent;
protected readonly Stream _saveFile;
private readonly BufferedEnumerator<Token> _tokens;
private readonly ObjectPool<StringBuilder> _stringBuilderPool;
private ISearchExpression? _searchExprCurrent;
public int SBPoolGetCount = 0;
public int SBPoolReturnCount = 0;
/// <param name="savefile">
/// Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive
@ -23,9 +28,16 @@ public class SaveParserEU4
/// </param>
public SaveParserEU4(Stream savefile, ISearchExpression? query)
{
_tokens = new BufferedEnumerator<Token>(LexTextSave(), 5);
_saveFile = savefile;
_searchExprCurrent = query;
const int tokenBufSize = 5;
_tokens = new BufferedEnumerator<Token>(LexTextSave(), tokenBufSize);
_stringBuilderPool = new DefaultObjectPool<StringBuilder>(
new StringBuilderPooledObjectPolicy
{
InitialCapacity = tokenBufSize * 13,
MaximumRetainedCapacity = tokenBufSize * 13,
});
}
protected IEnumerator<Token> LexTextSave()
@ -37,7 +49,8 @@ public class SaveParserEU4
if (headStr != expectedHeader)
throw new Exception($"Invalid gamestate header. Expected '{expectedHeader}', got '{headStr}'.");
StringBuilder str = new();
StringBuilder strb = _stringBuilderPool.Get();
SBPoolGetCount++;
int line = 2;
int column = 0;
bool isQuoteOpen = false;
@ -46,7 +59,8 @@ public class SaveParserEU4
{
type = TokenType.Invalid,
column = -1,
line = -1
line = -1,
value = null,
};
bool TryCompleteStringToken()
@ -55,17 +69,18 @@ public class SaveParserEU4
return false;
// strings in quotes may be empty
if (!isStrInQuotes && (str.Length <= 0 || str[0] == '#'))
if (!isStrInQuotes && (strb.Length <= 0 || strb[0] == '#'))
return false;
strToken = new Token
{
type = TokenType.StringOrNumber,
column = (short)(column - str.Length),
column = (short)(column - strb.Length),
line = line,
value = str.ToString()
value = strb,
};
str.Clear();
strb = _stringBuilderPool.Get();
SBPoolGetCount++;
isStrInQuotes = false;
return true;
}
@ -79,6 +94,8 @@ public class SaveParserEU4
case -1:
if (TryCompleteStringToken())
yield return strToken;
_stringBuilderPool.Return(strb);
SBPoolReturnCount++;
yield break;
case '\"':
isQuoteOpen = !isQuoteOpen;
@ -127,10 +144,13 @@ public class SaveParserEU4
// Skip control characters, which are invisible and causing frontend bugs.
// I dont know why there are so many of them in strings.
if (c >= 0x20)
str.Append((char)c);
strb.Append((char)c);
break;
}
}
_stringBuilderPool.Return(strb);
SBPoolReturnCount++;
}
@ -141,15 +161,16 @@ public class SaveParserEU4
switch (tok.type)
{
case TokenType.StringOrNumber:
if (string.IsNullOrEmpty(tok.value))
return string.Empty;
if (tok.value[0] != '-' && !char.IsDigit(tok.value[0]))
return tok.value;
if (tok.value.Contains('.') && double.TryParse(tok.value, out double d))
string tokStr = tok.value!.ToString();
_stringBuilderPool.Return(tok.value);
SBPoolReturnCount++;
if (tokStr[0] != '-' && !char.IsDigit(tokStr[0]))
return tokStr;
if (tokStr.Contains('.') && double.TryParse(tokStr, out double d))
return d;
if (long.TryParse(tok.value, out long l))
if (long.TryParse(tokStr, out long l))
return l;
return tok.value;
return tokStr;
case TokenType.BracketOpen:
object obj = ParseListOrDict();
return obj;
@ -166,13 +187,22 @@ public class SaveParserEU4
private bool SkipValue()
{
var tok = _tokens.Current.Value;
if (tok.type == TokenType.BracketOpen)
switch (tok.type)
{
SkipObject();
return true;
case TokenType.BracketOpen:
SkipObject();
return true;
case TokenType.StringOrNumber:
_stringBuilderPool.Return(tok.value!);
SBPoolReturnCount++;
return true;
case TokenType.Equals:
return true;
case TokenType.BracketClose:
return false;
default:
throw new UnexpectedTokenException(tok);
}
return tok.type != TokenType.BracketClose;
}
// skips all tokens inside curly braces block
@ -185,6 +215,11 @@ public class SaveParserEU4
bracketBalance++;
else if (tok.type == TokenType.BracketClose)
bracketBalance--;
else if (tok.type == TokenType.StringOrNumber)
{
_stringBuilderPool.Return(tok.value!);
SBPoolReturnCount++;
}
}
}
@ -242,7 +277,7 @@ public class SaveParserEU4
if (tok.type != TokenType.StringOrNumber)
throw new UnexpectedTokenException(tok);
string key = tok.value!;
var keySB = tok.value!;
// next token should be `=` or `{`
if (!_tokens.MoveNext())
@ -263,9 +298,11 @@ public class SaveParserEU4
ISearchExpression? searchExprNext = null;
if (_searchExprCurrent != null
&& !_searchExprCurrent.DoesMatch(new SearchArgs(key, localIndex), out searchExprNext))
&& !_searchExprCurrent.DoesMatch(new SearchArgs(localIndex, keySB), out searchExprNext))
{
SkipValue();
_stringBuilderPool.Return(keySB);
SBPoolReturnCount++;
continue;
}
@ -276,10 +313,13 @@ public class SaveParserEU4
throw new UnexpectedTokenException(_tokens.Current.Value);
_searchExprCurrent = searExpressionPrevious;
if (!dict.TryGetValue(key, out var list))
string keyStr = keySB.ToString();
_stringBuilderPool.Return(keySB);
SBPoolReturnCount++;
if (!dict.TryGetValue(keyStr, out var list))
{
list = new List<object>();
dict.Add(key, list);
dict.Add(keyStr, list);
}
list.Add(value);
@ -308,7 +348,7 @@ public class SaveParserEU4
public required TokenType type;
public required short column;
public required int line;
public string? value;
public StringBuilder? value;
public override string ToString()
{
@ -319,7 +359,9 @@ public class SaveParserEU4
s = "INVALID_TOKEN";
break;
case TokenType.StringOrNumber:
s = value ?? "NULL";
if (value == null || value.Length == 0)
s = "NULL";
else s = value.ToString();
break;
case TokenType.Equals:
s = "=";

25
ParadoxSaveParser.Lib/SearchExpression.cs
View File

@ -1,6 +1,25 @@
namespace ParadoxSaveParser.Lib;
public record SearchArgs(string key, int localIndex);
public record SearchArgs
{
public readonly string KeyStr;
public readonly StringBuilder? KeySB;
public readonly int LocalIndex;
public SearchArgs(int localIndex, string keyStr)
{
KeyStr = keyStr;
KeySB = null;
LocalIndex = localIndex;
}
public SearchArgs(int localIndex, StringBuilder keySb)
{
KeyStr = string.Empty;
KeySB = keySb;
LocalIndex = localIndex;
}
}
public interface ISearchExpression
{
@ -109,7 +128,7 @@ public static class SearchExpressionCompiler
{
public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
{
if (args.localIndex == index)
if (args.LocalIndex == index)
{
nextSearchExpression = next;
return true;
@ -124,7 +143,7 @@ public static class SearchExpressionCompiler
{
public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
{
if (args.key == key)
if ((args.KeySB != null && args.KeySB.Equals(key)) || args.KeyStr == key)
{
nextSearchExpression = next;
return true;

16
ParadoxSaveParser.WebAPI/Program.cs
View File

@ -12,6 +12,7 @@ global using Directory = DTLib.Filesystem.Directory;
global using File = DTLib.Filesystem.File;
global using Path = DTLib.Filesystem.Path;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.IO;
using System.Text.Encodings.Web;
using DTLib.Dtsod;
@ -54,6 +55,21 @@ public static partial class Program
try
{
Stopwatch stopwatch = new();
using var save = File.OpenRead("data/gamestate");
stopwatch.Start();
var parser = new SaveParserEU4(save, SearchExpressionCompiler.Compile("saved_event_target"));
var result = parser.Parse();
stopwatch.Stop();
using (var resultFile = File.OpenWrite("data/parsed.json"))
{
JsonSerializer.Serialize(resultFile, result, _saveSerializerOptions);
}
Console.WriteLine($"get: {parser.SBPoolGetCount} return: {parser.SBPoolReturnCount} " +
$"delta: {parser.SBPoolGetCount - parser.SBPoolReturnCount}");
Console.WriteLine(stopwatch.Elapsed);
return;
// config
if (!File.Exists(_configPath))
{