Timerix/ParadoxSaveParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: Timerix:52d5320899d303627bc63a3cbe3bd3b28b15f723
Branches Tags
Timerix:main
..
compare: Timerix:17981347f4e60aaf90b95ee49facdab2ce1cbbc9
Branches Tags
Timerix:main

No commits in common. "52d5320899d303627bc63a3cbe3bd3b28b15f723" and "17981347f4e60aaf90b95ee49facdab2ce1cbbc9" have entirely different histories.
52d5320899 ... 17981347f4

5 changed files with 54 additions and 167 deletions
Show Stats Expand all files Collapse all files

83
ParadoxSaveParser.Lib/BufferedEnumerator.cs
View File

@ -1,83 +0,0 @@
using System.Collections;
namespace ParadoxSaveParser.Lib;
/// <summary>
/// Enumerator wrapper that stores <c>N/2</c> items before and <c>N/2-1</c> after <c>Current</c> item.
/// </summary>
/// <code language="cs">
/// IEnumerator&lt;int&gt; Enumerator()
/// {
/// for(int i = 0; i &lt; 6; i++)
/// yield return i;
/// }
///
/// var en = Enumerator();
/// var bufen = new BufferedEnumerator&lt;int&gt;(en, 5);
///
/// while(bufen.MoveNext())
/// {
/// var cur = bufen.Current;
/// for (var prev = cur.List?.First; prev != cur; prev = prev?.Next)
/// Console.Write($"{prev?.Value} ");
///
/// Console.Write($"| {cur.Value} |");
///
/// for (var next = cur.Next; next != null; next = next.Next)
/// Console.Write($" {next.Value}");
/// Console.WriteLine();
/// }
/// </code>
/// Output:
/// <code>
/// | 0 | 1 2 3 4
/// 0 | 1 | 2 3 4
/// 0 1 | 2 | 3 4
/// 1 2 | 3 | 4 5
/// 2 3 | 4 | 5
/// 3 4 | 5 |
/// </code>
public class BufferedEnumerator<T> : IEnumerator<LinkedListNode<T>>
{
private IEnumerator<T> _enumerator;
private int _bufferSize;
LinkedList<T> _llist = new();
private LinkedListNode<T>? _currentNode;
private int _currentNodeIndex = -1;
public BufferedEnumerator(IEnumerator<T> enumerator, int bufferSize)
{
_enumerator = enumerator;
_bufferSize = bufferSize;
}
public bool MoveNext()
{
if(_currentNodeIndex >= _bufferSize / 2)
_llist.RemoveFirst();
while (_llist.Count < _bufferSize && _enumerator.MoveNext())
{
_llist.AddLast(_enumerator.Current);
}
if (_llist.Count == 0)
return false;
_currentNodeIndex++;
_currentNode = _currentNode == null ? _llist.First : _currentNode.Next;
return _currentNode != null;
}
public void Reset()
{
throw new NotImplementedException();
}
public LinkedListNode<T> Current => _currentNode!;
object IEnumerator.Current => Current;
public void Dispose()
{
}
}

126
ParadoxSaveParser.Lib/Parser.cs
View File

@ -8,11 +8,11 @@ namespace ParadoxSaveParser.Lib;
public class Parser public class Parser
{ {
protected Stream _saveFile; protected Stream _saveFile;
private BufferedEnumerator<Token> _tokens; private List<Token> _tokens = new(4_194_304);
private int _tokenIndex;
public Parser(Stream savefile) public Parser(Stream savefile)
{ {
_tokens = new BufferedEnumerator<Token>(Lex(), 5);
_saveFile = savefile; _saveFile = savefile;
} }
@ -27,9 +27,9 @@ public class Parser
protected struct Token protected struct Token
{ {
public required TokenType type; public TokenType type;
public required short column; public short column;
public required int line; public int line;
public string? value; public string? value;
public override string ToString() public override string ToString()
@ -60,8 +60,10 @@ public class Parser
} }
} }
protected IEnumerator<Token> Lex() protected void Lex()
{ {
_tokens.Clear();
string expectedHeader = "EU4txt"; string expectedHeader = "EU4txt";
byte[] headBytes = new byte[expectedHeader.Length]; byte[] headBytes = new byte[expectedHeader.Length];
_saveFile.ReadExactly(headBytes); _saveFile.ReadExactly(headBytes);
@ -74,31 +76,23 @@ public class Parser
int column = 0; int column = 0;
bool isQuoteOpen = false; bool isQuoteOpen = false;
bool isStrInQuotes = false; bool isStrInQuotes = false;
Token strToken = new()
{ void CompleteStringToken()
type = TokenType.Invalid,
column = -1,
line = -1
};
bool TryCompleteStringToken()
{ {
if (isQuoteOpen) if (isQuoteOpen)
return false; return;
// strings in quotes can be empty // strings in quotes can be empty
if (!isStrInQuotes && (str.Length <= 0 || str[0] == '#')) if (!isStrInQuotes && (str.Length <= 0 || str[0] == '#'))
return false; return;
_tokens.Add(new Token
strToken = new Token
{ {
type = TokenType.StringOrNumber, type = TokenType.StringOrNumber,
column = (short)(column - str.Length), column = (short)(column - str.Length),
line = line, line = line,
value = str.ToString() value = str.ToString()
}; });
str.Clear(); str.Clear();
isStrInQuotes = false; isStrInQuotes = false;
return true;
} }
while (_saveFile.CanRead) while (_saveFile.CanRead)
@ -108,9 +102,8 @@ public class Parser
switch (c) switch (c)
{ {
case -1: case -1:
if(TryCompleteStringToken()) CompleteStringToken();
yield return strToken; return;
yield break;
case '\"': case '\"':
isQuoteOpen = !isQuoteOpen; isQuoteOpen = !isQuoteOpen;
isStrInQuotes = true; isStrInQuotes = true;
@ -118,41 +111,36 @@ public class Parser
case ' ': case ' ':
case '\t': case '\t':
case '\r': case '\r':
if(TryCompleteStringToken()) CompleteStringToken();
yield return strToken;
break; break;
case '\n': case '\n':
if(TryCompleteStringToken()) CompleteStringToken();
yield return strToken;
line++; line++;
column = 0; column = 0;
break; break;
case '=': case '=':
if(TryCompleteStringToken()) CompleteStringToken();
yield return strToken; _tokens.Add(new Token
yield return new Token
{ {
type = TokenType.Equals, type = TokenType.Equals,
line = line, column = (short)column line = line, column = (short)column
}; });
break; break;
case '{': case '{':
if(TryCompleteStringToken()) CompleteStringToken();
yield return strToken; _tokens.Add(new Token
yield return new Token
{ {
type = TokenType.BracketOpen, type = TokenType.BracketOpen,
line = line, column = (short)column line = line, column = (short)column
}; });
break; break;
case '}': case '}':
if(TryCompleteStringToken()) CompleteStringToken();
yield return strToken; _tokens.Add(new Token
yield return new Token
{ {
type = TokenType.BracketClose, type = TokenType.BracketClose,
line = line, column = (short)column line = line, column = (short)column
}; });
break; break;
default: default:
// Skip control characters, which are invisible and causing frontend bugs. // Skip control characters, which are invisible and causing frontend bugs.
@ -166,16 +154,15 @@ public class Parser
protected class UnexpectedTokenException : Exception protected class UnexpectedTokenException : Exception
{ {
public UnexpectedTokenException(Token token) : public UnexpectedTokenException(Token token, int tokenIndex) :
base($"Unexpected token: {token}") base($"Unexpected token at index {tokenIndex}: {token}")
{} {}
} }
// doesn't move next
private object? ParseValue() private object? ParseValue()
{ {
Token tok = _tokens.Current.Value; Token tok = _tokens[_tokenIndex++];
switch (tok.type) switch (tok.type)
{ {
case TokenType.StringOrNumber: case TokenType.StringOrNumber:
@ -193,29 +180,25 @@ public class Parser
case TokenType.BracketClose: case TokenType.BracketClose:
return null; return null;
default: default:
throw new UnexpectedTokenException(tok); throw new UnexpectedTokenException(tok, _tokenIndex - 1);
} }
} }
// doesn't move next
private object ParseListOrDict() private object ParseListOrDict()
{ {
var first = _tokens.Current.Next; Token first = _tokens[_tokenIndex];
var second = _tokens.Current.Next?.Next; Token second = _tokens[_tokenIndex + 1];
if (first?.Value.type == TokenType.StringOrNumber && second?.Value.type == TokenType.Equals) if (first.type == TokenType.StringOrNumber && second.type == TokenType.Equals)
return ParseDict(); return ParseDict();
return ParseList(); return ParseList();
} }
// moves next
private List<object> ParseList() private List<object> ParseList()
{ {
List<object> list = new(); List<object> list = new();
while(true) while(true)
{ {
if(!_tokens.MoveNext())
throw new Exception("Unexpected end of file");
object? value = ParseValue(); object? value = ParseValue();
if (value == null) if (value == null)
break; break;
@ -225,14 +208,13 @@ public class Parser
} }
// moves next
private Dictionary<string, List<object>> ParseDict() private Dictionary<string, List<object>> ParseDict()
{ {
Dictionary<string, List<object>> dict = new(); Dictionary<string, List<object>> dict = new();
// root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count // root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count
while (_tokens.MoveNext()) while (_tokenIndex < _tokens.Count)
{ {
Token tok = _tokens.Current.Value; Token tok = _tokens[_tokenIndex++];
// end of dictionary // end of dictionary
if (tok.type == TokenType.BracketClose) if (tok.type == TokenType.BracketClose)
break; break;
@ -244,9 +226,9 @@ public class Parser
if (tok.type == TokenType.BracketOpen) if (tok.type == TokenType.BracketOpen)
{ {
int bracketBalance = 1; int bracketBalance = 1;
while (bracketBalance != 0 && _tokens.MoveNext()) while (bracketBalance != 0)
{ {
tok = _tokens.Current.Value; tok = _tokens[_tokenIndex++];
if (tok.type == TokenType.BracketOpen) if (tok.type == TokenType.BracketOpen)
bracketBalance++; bracketBalance++;
else if (tok.type == TokenType.BracketClose) else if (tok.type == TokenType.BracketClose)
@ -257,29 +239,24 @@ public class Parser
} }
if(tok.type != TokenType.StringOrNumber) if(tok.type != TokenType.StringOrNumber)
throw new UnexpectedTokenException(tok); throw new UnexpectedTokenException(tok, _tokenIndex - 1);
string key = tok.value!; string key = tok.value!;
// next token should be `=` or `{` tok = _tokens[_tokenIndex++];
if(!_tokens.MoveNext()) if (tok.type == TokenType.BracketOpen)
throw new UnexpectedTokenException(tok);
tok = _tokens.Current.Value;
if (tok.type == TokenType.Equals)
{ {
// skip `=` // Saves may contain key-value definition without `=`.
if (!_tokens.MoveNext()) // Example: `map_area_data{` instead of `map_area_data = {`
throw new UnexpectedTokenException(tok); _tokenIndex--;
} }
// Saves may contain object definition without `=`. else if(tok.type != TokenType.Equals)
// Example: `map_area_data {` instead of `map_area_data = {` throw new UnexpectedTokenException(tok, _tokenIndex - 1);
else if (tok.type != TokenType.BracketOpen)
throw new UnexpectedTokenException(tok);
object? value = ParseValue(); object? value = ParseValue();
if (value == null) if (value == null)
throw new UnexpectedTokenException(_tokens.Current.Value); throw new UnexpectedTokenException(_tokens[_tokenIndex - 1], _tokenIndex - 1);
if(!dict.TryGetValue(key, out List<object>? list)) if(!dict.TryGetValue(key, out List<object>? list))
{ {
list = new List<object>(); list = new List<object>();
@ -293,9 +270,12 @@ public class Parser
public Dictionary<string, List<object>> Parse() public Dictionary<string, List<object>> Parse()
{ {
var root = ParseDict(); Lex();
if (root.Count == 0) if (_tokens.Count == 0)
throw new Exception("Save file is empty"); throw new Exception("Save file is empty");
_tokenIndex = 0;
var root = ParseDict();
return root; return root;
} }
} }

1
ParadoxSaveParser.WebAPI/Program.cs
View File

@ -156,6 +156,7 @@ public class Program
_app.Logger.Log(LogLevel.Error, "ParseSaveEU4 Error: {errorMesage}", errorMesage); _app.Logger.Log(LogLevel.Error, "ParseSaveEU4 Error: {errorMesage}", errorMesage);
} }
GC.Collect();
await httpContext.Response.WriteAsJsonAsync(meta); await httpContext.Response.WriteAsJsonAsync(meta);
} }
} }

1
ParadoxSaveParser.sln
View File

@ -7,7 +7,6 @@ EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SolutionFolder", "SolutionFolder", "{F1D312F1-0620-4E35-8D78-9A2808CDE12C}" Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SolutionFolder", "SolutionFolder", "{F1D312F1-0620-4E35-8D78-9A2808CDE12C}"
ProjectSection(SolutionItems) = preProject ProjectSection(SolutionItems) = preProject
.gitignore = .gitignore .gitignore = .gitignore
TODO.txt = TODO.txt
EndProjectSection EndProjectSection
EndProject EndProject
Global Global

10
TODO.txt
View File

@ -1,10 +0,0 @@
Main:
Move from asp.net to my own http server
Add temporary files deletion
ParseSaveHandler:
Separate status and error message from metadata
Make this method run as background task instead of POST query
Parser:
Add query support to parse only needed information