global using System; global using System.Collections.Generic; global using System.IO; global using System.Text; using System.Globalization; namespace ParadoxSaveParser.Lib; /// /// Sequential parser that doesn't cache anything. /// public class SaveParserEU4 { private const int DefaultBufferSize = 64 * 1024; private static ReadOnlySpan Header => "EU4txt"u8; private readonly Tokenizer _tokens; private ISearchExpression? _searchExprCurrent; /// /// Encoding of the strings inside the save. Saves of different localizations use /// different ones, so it can be changed; maps /// every byte to one character and never fails, which makes it a safe default. /// public Encoding Encoding { get; } /// /// Uncompressed stream of gamestate file which can be extracted from save archive /// /// /// Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM, /// so you should specify what exactly you want to get from save file /// /// Encoding of the strings inside the save. Latin1 by default. /// Size of the read buffer. Mostly useful for tests. public SaveParserEU4(Stream savefile, ISearchExpression? query, Encoding? encoding = null, int bufferSize = DefaultBufferSize) { Encoding = encoding ?? Encoding.Latin1; _searchExprCurrent = query; _tokens = new Tokenizer(savefile, bufferSize); } public Dictionary Parse() { _tokens.ReadHeader(Header, Encoding); return ParseDict(); } // doesn't move next private object? ParseValue() { switch (_tokens.Type) { case TokenType.StringOrNumber: return ParseScalar(_tokens.Text); case TokenType.BracketOpen: return ParseListOrDict(); case TokenType.BracketClose: return null; default: throw new UnexpectedTokenException(_tokens, Encoding); } } private object ParseScalar(ReadOnlySpan text) { // string values can be empty if (text.Length == 0) return string.Empty; if (text.SequenceEqual("yes"u8)) return true; if (text.SequenceEqual("no"u8)) return false; byte first = text[0]; if (first != (byte)'-' && !char.IsAsciiDigit((char)first)) return DecodeString(text); if (text.Contains((byte)'.') && double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out double d)) return d; if (long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out long l)) return l; return DecodeString(text); } private string DecodeString(ReadOnlySpan text) { // Skip control characters, which are invisible and causing frontend bugs. // I dont know why there are so many of them in strings. if (!text.ContainsAnyInRange((byte)0, (byte)0x1F)) return Encoding.GetString(text); Span cleaned = text.Length <= 256 ? stackalloc byte[text.Length] : new byte[text.Length]; int length = 0; foreach (byte b in text) if (b >= 0x20) cleaned[length++] = b; return Encoding.GetString(cleaned[..length]); } // skips next value /// true if skipped value, false if current token is closing bracket private bool SkipValue() { switch (_tokens.Type) { case TokenType.BracketOpen: _tokens.SkipBlock(); return true; case TokenType.StringOrNumber: return true; case TokenType.BracketClose: return false; default: throw new UnexpectedTokenException(_tokens, Encoding); } } private static bool IsEmptyCollection(object value) => value is Dictionary { Count: 0 } or List { Count: 0 }; // doesn't move next private object ParseListOrDict() { if (_tokens.PeekType(1) == TokenType.StringOrNumber && _tokens.PeekType(2) == TokenType.Equals) return ParseDict(); return ParseList(); } // moves next private List ParseList() { List list = new(); for (int i = 0; ; i++) { if (!_tokens.Read()) throw new Exception("Unexpected end of file"); ISearchExpression? searchExprNext = null; if (_searchExprCurrent != null && !_searchExprCurrent.DoesMatch(new MatchCandidate(i), out searchExprNext)) { if (!SkipValue()) break; continue; } var searchExprPrev = _searchExprCurrent; _searchExprCurrent = searchExprNext; object? value = ParseValue(); _searchExprCurrent = searchExprPrev; if (value is null) break; // do dot add empty collections into list if (IsEmptyCollection(value)) continue; list.Add(value); } return list; } // moves next private Dictionary ParseDict() { Dictionary dict = new(); // root is a dict without closing bracket, so this method must check for end of file for (int localIndex = 0; _tokens.Read(); localIndex++) { // end of dictionary if (_tokens.Type == TokenType.BracketClose) break; // Saves may contain some blocks without key. // Such blocks are skipped because idk where to put them. // Example: `technology_group=tech_cannorian{ } // { } { } { }` if (_tokens.Type == TokenType.BracketOpen) { _tokens.SkipBlock(); continue; } if (_tokens.Type != TokenType.StringOrNumber) throw new UnexpectedTokenException(_tokens, Encoding); // The key is matched before the value is read, so that a key rejected by the query // never has to become a string. ISearchExpression? searchExprNext = null; bool matches = _searchExprCurrent == null || _searchExprCurrent.DoesMatch( new MatchCandidate(localIndex, _tokens.Text, Encoding), out searchExprNext); string? keyStr = matches ? DecodeString(_tokens.Text) : null; // next token should be `=` or `{` if (!_tokens.Read()) throw new UnexpectedTokenException(_tokens, Encoding); if (_tokens.Type == TokenType.Equals) { // skip `=` if (!_tokens.Read()) throw new UnexpectedTokenException(_tokens, Encoding); } // Saves may contain object definition without `=`. // Example: `map_area_data {` instead of `map_area_data = {` else if (_tokens.Type != TokenType.BracketOpen) { throw new UnexpectedTokenException(_tokens, Encoding); } if (!matches) { if (!SkipValue()) throw new UnexpectedTokenException(_tokens, Encoding); continue; } var searExpressionPrevious = _searchExprCurrent; _searchExprCurrent = searchExprNext; object? value = ParseValue(); if (value is null) throw new UnexpectedTokenException(_tokens, Encoding); _searchExprCurrent = searExpressionPrevious; // Paradox save format has another way of defining list: // a = 1 // a = 2 // It means `a = { 1 2 }` if (dict.TryGetValue(keyStr!, out var firstValue)) { // Do dot add empty collections into list. // `key:{}` is okay, but i don't want to see `key:[{},{},{},{},{},{}]` if (IsEmptyCollection(value)) continue; if (firstValue is List existingList) existingList.Add(value); else dict[keyStr!] = new List { firstValue, value }; } else { dict.Add(keyStr!, value); } } return dict; } internal class UnexpectedTokenException : Exception { public UnexpectedTokenException(Tokenizer tokens, Encoding encoding) : base($"Unexpected token: {tokens.Line}:{tokens.Column} '{tokens.Describe(encoding)}'") { } } }