Timerix/ParadoxSaveParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BufferedEnumerator that keeps 5 tokens in memory instead of 12,000,000

Browse Source
This commit is contained in:
Timerix 2025-03-22 20:39:06 +05:00
parent 17981347f4
commit bf00ffe291
3 changed files with 156 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
ParadoxSaveParser.Lib/BufferedEnumerator.cs Normal file
View File

@ -0,0 +1,83 @@
using System.Collections;
namespace ParadoxSaveParser.Lib;
/// <summary>
/// Enumerator wrapper that stores <c>N/2</c> items before and <c>N/2-1</c> after <c>Current</c> item.
/// </summary>
/// <code language="cs">
/// IEnumerator&lt;int&gt; Enumerator()
/// {
/// for(int i = 0; i &lt; 6; i++)
/// yield return i;
/// }
///
/// var en = Enumerator();
/// var bufen = new BufferedEnumerator&lt;int&gt;(en, 5);
///
/// while(bufen.MoveNext())
/// {
/// var cur = bufen.Current;
/// for (var prev = cur.List?.First; prev != cur; prev = prev?.Next)
/// Console.Write($"{prev?.Value} ");
///
/// Console.Write($"| {cur.Value} |");
///
/// for (var next = cur.Next; next != null; next = next.Next)
/// Console.Write($" {next.Value}");
/// Console.WriteLine();
/// }
/// </code>
/// Output:
/// <code>
/// | 0 | 1 2 3 4
/// 0 | 1 | 2 3 4
/// 0 1 | 2 | 3 4
/// 1 2 | 3 | 4 5
/// 2 3 | 4 | 5
/// 3 4 | 5 |
/// </code>
public class BufferedEnumerator<T> : IEnumerator<LinkedListNode<T>>
{
private IEnumerator<T> _enumerator;
private int _bufferSize;
LinkedList<T> _llist = new();
private LinkedListNode<T>? _currentNode;
private int _currentNodeIndex = -1;
public BufferedEnumerator(IEnumerator<T> enumerator, int bufferSize)
{
_enumerator = enumerator;
_bufferSize = bufferSize;
}
public bool MoveNext()
{
if(_currentNodeIndex >= _bufferSize / 2)
_llist.RemoveFirst();
while (_llist.Count < _bufferSize && _enumerator.MoveNext())
{
_llist.AddLast(_enumerator.Current);
}
if (_llist.Count == 0)
return false;
_currentNodeIndex++;
_currentNode = _currentNode == null ? _llist.First : _currentNode.Next;
return _currentNode != null;
}
public void Reset()
{
throw new NotImplementedException();
}
public LinkedListNode<T> Current => _currentNode!;
object IEnumerator.Current => Current;
public void Dispose()
{
}
}

126
ParadoxSaveParser.Lib/Parser.cs
View File

@ -8,11 +8,11 @@ namespace ParadoxSaveParser.Lib;
public class Parser public class Parser
{ {
protected Stream _saveFile; protected Stream _saveFile;
private List<Token> _tokens = new(4_194_304); private BufferedEnumerator<Token> _tokens;
private int _tokenIndex;
public Parser(Stream savefile) public Parser(Stream savefile)
{ {
_tokens = new BufferedEnumerator<Token>(Lex(), 5);
_saveFile = savefile; _saveFile = savefile;
} }
@ -27,9 +27,9 @@ public class Parser
protected struct Token protected struct Token
{ {
public TokenType type; public required TokenType type;
public short column; public required short column;
public int line; public required int line;
public string? value; public string? value;
public override string ToString() public override string ToString()
@ -60,10 +60,8 @@ public class Parser
} }
} }
protected void Lex() protected IEnumerator<Token> Lex()
{ {
_tokens.Clear();
string expectedHeader = "EU4txt"; string expectedHeader = "EU4txt";
byte[] headBytes = new byte[expectedHeader.Length]; byte[] headBytes = new byte[expectedHeader.Length];
_saveFile.ReadExactly(headBytes); _saveFile.ReadExactly(headBytes);
@ -76,23 +74,31 @@ public class Parser
int column = 0; int column = 0;
bool isQuoteOpen = false; bool isQuoteOpen = false;
bool isStrInQuotes = false; bool isStrInQuotes = false;
Token strToken = new()
void CompleteStringToken() {
type = TokenType.Invalid,
column = -1,
line = -1
};
bool TryCompleteStringToken()
{ {
if (isQuoteOpen) if (isQuoteOpen)
return; return false;
// strings in quotes can be empty // strings in quotes can be empty
if (!isStrInQuotes && (str.Length <= 0 || str[0] == '#')) if (!isStrInQuotes && (str.Length <= 0 || str[0] == '#'))
return; return false;
_tokens.Add(new Token
strToken = new Token
{ {
type = TokenType.StringOrNumber, type = TokenType.StringOrNumber,
column = (short)(column - str.Length), column = (short)(column - str.Length),
line = line, line = line,
value = str.ToString() value = str.ToString()
}); };
str.Clear(); str.Clear();
isStrInQuotes = false; isStrInQuotes = false;
return true;
} }
while (_saveFile.CanRead) while (_saveFile.CanRead)
@ -102,8 +108,9 @@ public class Parser
switch (c) switch (c)
{ {
case -1: case -1:
CompleteStringToken(); if(TryCompleteStringToken())
return; yield return strToken;
yield break;
case '\"': case '\"':
isQuoteOpen = !isQuoteOpen; isQuoteOpen = !isQuoteOpen;
isStrInQuotes = true; isStrInQuotes = true;
@ -111,36 +118,41 @@ public class Parser
case ' ': case ' ':
case '\t': case '\t':
case '\r': case '\r':
CompleteStringToken(); if(TryCompleteStringToken())
yield return strToken;
break; break;
case '\n': case '\n':
CompleteStringToken(); if(TryCompleteStringToken())
yield return strToken;
line++; line++;
column = 0; column = 0;
break; break;
case '=': case '=':
CompleteStringToken(); if(TryCompleteStringToken())
_tokens.Add(new Token yield return strToken;
yield return new Token
{ {
type = TokenType.Equals, type = TokenType.Equals,
line = line, column = (short)column line = line, column = (short)column
}); };
break; break;
case '{': case '{':
CompleteStringToken(); if(TryCompleteStringToken())
_tokens.Add(new Token yield return strToken;
yield return new Token
{ {
type = TokenType.BracketOpen, type = TokenType.BracketOpen,
line = line, column = (short)column line = line, column = (short)column
}); };
break; break;
case '}': case '}':
CompleteStringToken(); if(TryCompleteStringToken())
_tokens.Add(new Token yield return strToken;
yield return new Token
{ {
type = TokenType.BracketClose, type = TokenType.BracketClose,
line = line, column = (short)column line = line, column = (short)column
}); };
break; break;
default: default:
// Skip control characters, which are invisible and causing frontend bugs. // Skip control characters, which are invisible and causing frontend bugs.
@ -154,15 +166,16 @@ public class Parser
protected class UnexpectedTokenException : Exception protected class UnexpectedTokenException : Exception
{ {
public UnexpectedTokenException(Token token, int tokenIndex) : public UnexpectedTokenException(Token token) :
base($"Unexpected token at index {tokenIndex}: {token}") base($"Unexpected token: {token}")
{} {}
} }
// doesn't move next
private object? ParseValue() private object? ParseValue()
{ {
Token tok = _tokens[_tokenIndex++]; Token tok = _tokens.Current.Value;
switch (tok.type) switch (tok.type)
{ {
case TokenType.StringOrNumber: case TokenType.StringOrNumber:
@ -180,25 +193,29 @@ public class Parser
case TokenType.BracketClose: case TokenType.BracketClose:
return null; return null;
default: default:
throw new UnexpectedTokenException(tok, _tokenIndex - 1); throw new UnexpectedTokenException(tok);
} }
} }
// doesn't move next
private object ParseListOrDict() private object ParseListOrDict()
{ {
Token first = _tokens[_tokenIndex]; var first = _tokens.Current.Next;
Token second = _tokens[_tokenIndex + 1]; var second = _tokens.Current.Next?.Next;
if (first.type == TokenType.StringOrNumber && second.type == TokenType.Equals) if (first?.Value.type == TokenType.StringOrNumber && second?.Value.type == TokenType.Equals)
return ParseDict(); return ParseDict();
return ParseList(); return ParseList();
} }
// moves next
private List<object> ParseList() private List<object> ParseList()
{ {
List<object> list = new(); List<object> list = new();
while(true) while(true)
{ {
if(!_tokens.MoveNext())
throw new Exception("Unexpected end of file");
object? value = ParseValue(); object? value = ParseValue();
if (value == null) if (value == null)
break; break;
@ -208,13 +225,14 @@ public class Parser
} }
// moves next
private Dictionary<string, List<object>> ParseDict() private Dictionary<string, List<object>> ParseDict()
{ {
Dictionary<string, List<object>> dict = new(); Dictionary<string, List<object>> dict = new();
// root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count // root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count
while (_tokenIndex < _tokens.Count) while (_tokens.MoveNext())
{ {
Token tok = _tokens[_tokenIndex++]; Token tok = _tokens.Current.Value;
// end of dictionary // end of dictionary
if (tok.type == TokenType.BracketClose) if (tok.type == TokenType.BracketClose)
break; break;
@ -226,9 +244,9 @@ public class Parser
if (tok.type == TokenType.BracketOpen) if (tok.type == TokenType.BracketOpen)
{ {
int bracketBalance = 1; int bracketBalance = 1;
while (bracketBalance != 0) while (bracketBalance != 0 && _tokens.MoveNext())
{ {
tok = _tokens[_tokenIndex++]; tok = _tokens.Current.Value;
if (tok.type == TokenType.BracketOpen) if (tok.type == TokenType.BracketOpen)
bracketBalance++; bracketBalance++;
else if (tok.type == TokenType.BracketClose) else if (tok.type == TokenType.BracketClose)
@ -239,24 +257,29 @@ public class Parser
} }
if(tok.type != TokenType.StringOrNumber) if(tok.type != TokenType.StringOrNumber)
throw new UnexpectedTokenException(tok, _tokenIndex - 1); throw new UnexpectedTokenException(tok);
string key = tok.value!; string key = tok.value!;
tok = _tokens[_tokenIndex++]; // next token should be `=` or `{`
if (tok.type == TokenType.BracketOpen) if(!_tokens.MoveNext())
throw new UnexpectedTokenException(tok);
tok = _tokens.Current.Value;
if (tok.type == TokenType.Equals)
{ {
// Saves may contain key-value definition without `=`. // skip `=`
// Example: `map_area_data{` instead of `map_area_data = {` if (!_tokens.MoveNext())
_tokenIndex--; throw new UnexpectedTokenException(tok);
} }
else if(tok.type != TokenType.Equals) // Saves may contain object definition without `=`.
throw new UnexpectedTokenException(tok, _tokenIndex - 1); // Example: `map_area_data {` instead of `map_area_data = {`
else if (tok.type != TokenType.BracketOpen)
throw new UnexpectedTokenException(tok);
object? value = ParseValue(); object? value = ParseValue();
if (value == null) if (value == null)
throw new UnexpectedTokenException(_tokens[_tokenIndex - 1], _tokenIndex - 1); throw new UnexpectedTokenException(_tokens.Current.Value);
if(!dict.TryGetValue(key, out List<object>? list)) if(!dict.TryGetValue(key, out List<object>? list))
{ {
list = new List<object>(); list = new List<object>();
@ -270,12 +293,9 @@ public class Parser
public Dictionary<string, List<object>> Parse() public Dictionary<string, List<object>> Parse()
{ {
Lex();
if (_tokens.Count == 0)
throw new Exception("Save file is empty");
_tokenIndex = 0;
var root = ParseDict(); var root = ParseDict();
if (root.Count == 0)
throw new Exception("Save file is empty");
return root; return root;
} }
} }

1
ParadoxSaveParser.WebAPI/Program.cs
View File

@ -156,7 +156,6 @@ public class Program
_app.Logger.Log(LogLevel.Error, "ParseSaveEU4 Error: {errorMesage}", errorMesage); _app.Logger.Log(LogLevel.Error, "ParseSaveEU4 Error: {errorMesage}", errorMesage);
} }
GC.Collect();
await httpContext.Response.WriteAsJsonAsync(meta); await httpContext.Response.WriteAsJsonAsync(meta);
} }
} }