Parser rewrite
This commit is contained in:
@@ -1,121 +0,0 @@
|
||||
using System.Collections;
|
||||
|
||||
namespace ParadoxSaveParser.Lib;
|
||||
|
||||
/// <summary>
|
||||
/// Enumerator wrapper that stores <c>N/2</c> items before and <c>N/2-1</c> after <c>Current</c> item.
|
||||
/// </summary>
|
||||
/// <code language="cs">
|
||||
/// IEnumerator<int> Enumerator()
|
||||
/// {
|
||||
/// for(int i = 0; i < 6; i++)
|
||||
/// yield return i;
|
||||
/// }
|
||||
///
|
||||
/// var en = Enumerator();
|
||||
/// var bufen = new BufferedEnumerator<int>(en, 5);
|
||||
///
|
||||
/// while(bufen.MoveNext())
|
||||
/// {
|
||||
/// var cur = bufen.Current;
|
||||
/// for (var prev = cur.List?.First; prev != cur; prev = prev?.Next)
|
||||
/// Console.Write($"{prev?.Value} ");
|
||||
///
|
||||
/// Console.Write($"| {cur.Value} |");
|
||||
///
|
||||
/// for (var next = cur.Next; next is not null; next = next.Next)
|
||||
/// Console.Write($" {next.Value}");
|
||||
/// Console.WriteLine();
|
||||
/// }
|
||||
/// </code>
|
||||
/// Output:
|
||||
/// <code>
|
||||
/// | 0 | 1 2 3 4
|
||||
/// 0 | 1 | 2 3 4
|
||||
/// 0 1 | 2 | 3 4
|
||||
/// 1 2 | 3 | 4 5
|
||||
/// 2 3 | 4 | 5
|
||||
/// 3 4 | 5 |
|
||||
/// </code>
|
||||
public class BufferedEnumerator<T> : IEnumerator<BufferedEnumerator<T>.Node>
|
||||
{
|
||||
public class Node
|
||||
{
|
||||
#nullable disable
|
||||
public Node Previous;
|
||||
public Node Next;
|
||||
public T Value;
|
||||
#nullable enable
|
||||
}
|
||||
|
||||
private readonly IEnumerator<T> _enumerator;
|
||||
private readonly Node[] _ringBuffer;
|
||||
private Node? _currentNode;
|
||||
private int _currentBufferIndex = -1;
|
||||
private int _lastValueIndex = -1;
|
||||
|
||||
public BufferedEnumerator(IEnumerator<T> enumerator, int bufferSize)
|
||||
{
|
||||
_enumerator = enumerator;
|
||||
_ringBuffer = new Node[bufferSize];
|
||||
}
|
||||
|
||||
private void InitBuffer()
|
||||
{
|
||||
_ringBuffer[0] = new Node
|
||||
{
|
||||
Value = default!
|
||||
};
|
||||
for (int i = 1; i < _ringBuffer.Length; i++)
|
||||
{
|
||||
_ringBuffer[i] = new Node
|
||||
{
|
||||
Previous = _ringBuffer[i - 1],
|
||||
Value = default!,
|
||||
};
|
||||
_ringBuffer[i - 1].Next = _ringBuffer[i];
|
||||
}
|
||||
_ringBuffer[^1].Next = _ringBuffer[0];
|
||||
_ringBuffer[0].Previous = _ringBuffer[^1];
|
||||
}
|
||||
|
||||
public bool MoveNext()
|
||||
{
|
||||
if (_currentBufferIndex == -1)
|
||||
{
|
||||
InitBuffer();
|
||||
|
||||
int beforeMidpoint = _ringBuffer.Length / 2 - 1;
|
||||
for (int i = 0; i <= beforeMidpoint && _enumerator.MoveNext(); i++)
|
||||
{
|
||||
_ringBuffer[i].Value = _enumerator.Current;
|
||||
}
|
||||
}
|
||||
|
||||
_currentBufferIndex = (_currentBufferIndex + 1) % _ringBuffer.Length;
|
||||
if (_enumerator.MoveNext())
|
||||
{
|
||||
int midpoint = (_currentBufferIndex + _ringBuffer.Length / 2) % _ringBuffer.Length;
|
||||
_ringBuffer[midpoint].Value = _enumerator.Current;
|
||||
_lastValueIndex = midpoint;
|
||||
}
|
||||
if(_currentBufferIndex == (_lastValueIndex + 1) % _ringBuffer.Length)
|
||||
return false;
|
||||
|
||||
_currentNode = _ringBuffer[_currentBufferIndex];
|
||||
return true;
|
||||
}
|
||||
|
||||
public void Reset()
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public Node Current => _currentNode!;
|
||||
|
||||
object IEnumerator.Current => Current;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,8 @@
|
||||
<ImplicitUsings>disable</ImplicitUsings>
|
||||
<InvariantGlobalization>true</InvariantGlobalization>
|
||||
</PropertyGroup>
|
||||
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.ObjectPool" Version="10.0.12" />
|
||||
<InternalsVisibleTo Include="ParadoxSaveParser.Lib.Tests" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -1,430 +1,261 @@
|
||||
global using System;
|
||||
global using System.Collections.Generic;
|
||||
global using System.IO;
|
||||
global using System.Text;
|
||||
using Microsoft.Extensions.ObjectPool;
|
||||
|
||||
namespace ParadoxSaveParser.Lib;
|
||||
|
||||
/// <summary>
|
||||
/// Sequential parser that doesn't cache anything.
|
||||
/// </summary>
|
||||
public class SaveParserEU4
|
||||
{
|
||||
protected readonly Stream _saveFile;
|
||||
private readonly BufferedEnumerator<Token> _tokens;
|
||||
private readonly ObjectPool<StringBuilder> _stringBuilderPool;
|
||||
private ISearchExpression? _searchExprCurrent;
|
||||
|
||||
/// <param name="savefile">
|
||||
/// Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive
|
||||
/// </param>
|
||||
/// <param name="query">
|
||||
/// Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM,
|
||||
/// so you should specify what exactly you want to get from save file
|
||||
/// </param>
|
||||
public SaveParserEU4(Stream savefile, ISearchExpression? query)
|
||||
{
|
||||
_saveFile = savefile;
|
||||
_searchExprCurrent = query;
|
||||
const int tokenBufSize = 5;
|
||||
_tokens = new BufferedEnumerator<Token>(LexTextSave(), tokenBufSize);
|
||||
_stringBuilderPool = new DefaultObjectPool<StringBuilder>(
|
||||
new StringBuilderPooledObjectPolicy
|
||||
{
|
||||
InitialCapacity = tokenBufSize * 13,
|
||||
MaximumRetainedCapacity = tokenBufSize * 13,
|
||||
});
|
||||
}
|
||||
|
||||
protected IEnumerator<Token> LexTextSave()
|
||||
{
|
||||
string expectedHeader = "EU4txt";
|
||||
byte[] headBytes = new byte[expectedHeader.Length];
|
||||
_saveFile.ReadExactly(headBytes);
|
||||
string headStr = Encoding.UTF8.GetString(headBytes);
|
||||
if (headStr != expectedHeader)
|
||||
throw new Exception($"Invalid gamestate header. Expected '{expectedHeader}', got '{headStr}'.");
|
||||
|
||||
StringBuilder strb = _stringBuilderPool.Get();
|
||||
int line = 2;
|
||||
int column = 0;
|
||||
bool isQuoteOpen = false;
|
||||
bool isStrInQuotes = false;
|
||||
Token strToken = new()
|
||||
{
|
||||
type = TokenType.Invalid,
|
||||
column = -1,
|
||||
line = -1,
|
||||
value = null,
|
||||
};
|
||||
|
||||
bool TryCompleteStringToken()
|
||||
{
|
||||
if (isQuoteOpen)
|
||||
return false;
|
||||
|
||||
// strings in quotes may be empty
|
||||
if (!isStrInQuotes && (strb.Length <= 0 || strb[0] == '#'))
|
||||
return false;
|
||||
|
||||
strToken = new Token
|
||||
{
|
||||
type = TokenType.StringOrNumber,
|
||||
column = (short)(column - strb.Length),
|
||||
line = line,
|
||||
value = strb,
|
||||
};
|
||||
strb = _stringBuilderPool.Get();
|
||||
isStrInQuotes = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Reading the save one byte at a time through Stream.ReadByte() costs a virtual call
|
||||
// per byte, which dominated the parsing time. Bytes are pulled into this buffer
|
||||
// instead, so the stream is touched once per 64 KB and the inner loop reads an array.
|
||||
byte[] buffer = new byte[64 * 1024];
|
||||
int bufferLength;
|
||||
while ((bufferLength = _saveFile.Read(buffer, 0, buffer.Length)) > 0)
|
||||
{
|
||||
for (int i = 0; i < bufferLength; i++)
|
||||
{
|
||||
int c = buffer[i];
|
||||
column++;
|
||||
switch (c)
|
||||
{
|
||||
case '\"':
|
||||
isQuoteOpen = !isQuoteOpen;
|
||||
isStrInQuotes = true;
|
||||
break;
|
||||
case ' ':
|
||||
case '\t':
|
||||
case '\r':
|
||||
if (TryCompleteStringToken())
|
||||
yield return strToken;
|
||||
break;
|
||||
case '\n':
|
||||
if (TryCompleteStringToken())
|
||||
yield return strToken;
|
||||
line++;
|
||||
column = 0;
|
||||
break;
|
||||
case '=':
|
||||
if (TryCompleteStringToken())
|
||||
yield return strToken;
|
||||
yield return new Token
|
||||
{
|
||||
type = TokenType.Equals,
|
||||
line = line, column = (short)column
|
||||
};
|
||||
break;
|
||||
case '{':
|
||||
if (TryCompleteStringToken())
|
||||
yield return strToken;
|
||||
yield return new Token
|
||||
{
|
||||
type = TokenType.BracketOpen,
|
||||
line = line, column = (short)column
|
||||
};
|
||||
break;
|
||||
case '}':
|
||||
if (TryCompleteStringToken())
|
||||
yield return strToken;
|
||||
yield return new Token
|
||||
{
|
||||
type = TokenType.BracketClose,
|
||||
line = line, column = (short)column
|
||||
};
|
||||
break;
|
||||
default:
|
||||
// Skip control characters, which are invisible and causing frontend bugs.
|
||||
// I dont know why there are so many of them in strings.
|
||||
if (c >= 0x20)
|
||||
strb.Append((char)c);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// end of file: the last token may still be unterminated
|
||||
if (TryCompleteStringToken())
|
||||
yield return strToken;
|
||||
_stringBuilderPool.Return(strb);
|
||||
}
|
||||
|
||||
|
||||
// doesn't move next
|
||||
private object? ParseValue()
|
||||
{
|
||||
var tok = _tokens.Current.Value;
|
||||
switch (tok.type)
|
||||
{
|
||||
case TokenType.StringOrNumber:
|
||||
try
|
||||
{
|
||||
// string values can be empty
|
||||
if (tok.value!.Length == 0)
|
||||
return string.Empty;
|
||||
if (tok.value.Equals("yes"))
|
||||
return true;
|
||||
if (tok.value.Equals("no"))
|
||||
return false;
|
||||
|
||||
string tokStr = tok.value.ToString();
|
||||
if (tokStr[0] != '-' && !char.IsDigit(tokStr[0]))
|
||||
return tokStr;
|
||||
if (tokStr.Contains('.') && double.TryParse(tokStr, out double d))
|
||||
return d;
|
||||
if (long.TryParse(tokStr, out long l))
|
||||
return l;
|
||||
return tokStr;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_stringBuilderPool.Return(tok.value!);
|
||||
}
|
||||
case TokenType.BracketOpen:
|
||||
object obj = ParseListOrDict();
|
||||
return obj;
|
||||
case TokenType.BracketClose:
|
||||
return null;
|
||||
default:
|
||||
throw new UnexpectedTokenException(tok);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// skips next value
|
||||
/// <returns>true if skipped value, false if current token is closing bracket</returns>
|
||||
private bool SkipValue()
|
||||
{
|
||||
var tok = _tokens.Current.Value;
|
||||
switch (tok.type)
|
||||
{
|
||||
case TokenType.BracketOpen:
|
||||
SkipObject();
|
||||
return true;
|
||||
case TokenType.StringOrNumber:
|
||||
_stringBuilderPool.Return(tok.value!);
|
||||
return true;
|
||||
case TokenType.BracketClose:
|
||||
return false;
|
||||
default:
|
||||
throw new UnexpectedTokenException(tok);
|
||||
}
|
||||
}
|
||||
|
||||
// skips all tokens inside curly braces block
|
||||
private void SkipObject(int bracketBalance = 1)
|
||||
{
|
||||
while (bracketBalance != 0 && _tokens.MoveNext())
|
||||
{
|
||||
var tok = _tokens.Current.Value;
|
||||
if (tok.type == TokenType.BracketOpen)
|
||||
bracketBalance++;
|
||||
else if (tok.type == TokenType.BracketClose)
|
||||
bracketBalance--;
|
||||
else if (tok.type == TokenType.StringOrNumber)
|
||||
{
|
||||
_stringBuilderPool.Return(tok.value!);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsEmptyCollection(object value)
|
||||
=> value is Dictionary<string, object> { Count: 0 } or List<object> { Count: 0 };
|
||||
|
||||
// doesn't move next
|
||||
private object ParseListOrDict()
|
||||
{
|
||||
var first = _tokens.Current.Next;
|
||||
var second = _tokens.Current.Next?.Next;
|
||||
if (first?.Value.type == TokenType.StringOrNumber && second?.Value.type == TokenType.Equals)
|
||||
return ParseDict();
|
||||
|
||||
return ParseList();
|
||||
}
|
||||
|
||||
// moves next
|
||||
private List<object> ParseList()
|
||||
{
|
||||
List<object> list = new();
|
||||
for (int i = 0; ; i++)
|
||||
{
|
||||
if (!_tokens.MoveNext())
|
||||
throw new Exception("Unexpected end of file");
|
||||
|
||||
ISearchExpression? searchExprNext = null;
|
||||
if (_searchExprCurrent != null
|
||||
&& !_searchExprCurrent.DoesMatch(new SearchArgs(i, string.Empty), out searchExprNext))
|
||||
{
|
||||
if(!SkipValue())
|
||||
break;
|
||||
continue;
|
||||
}
|
||||
var searchExprPrev = _searchExprCurrent;
|
||||
_searchExprCurrent = searchExprNext;
|
||||
object? value = ParseValue();
|
||||
_searchExprCurrent = searchExprPrev;
|
||||
if (value is null)
|
||||
break;
|
||||
|
||||
// do dot add empty collections into list
|
||||
if (IsEmptyCollection(value))
|
||||
continue;
|
||||
|
||||
list.Add(value);
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
// moves next
|
||||
private Dictionary<string, object> ParseDict()
|
||||
{
|
||||
Dictionary<string, object> dict = new();
|
||||
|
||||
// root is a dict without closing bracket, so this method must check _tokenIndex < _tokens.Count
|
||||
for (int localIndex = 0; _tokens.MoveNext(); localIndex++)
|
||||
{
|
||||
var tok = _tokens.Current.Value;
|
||||
// end of dictionary
|
||||
if (tok.type == TokenType.BracketClose)
|
||||
break;
|
||||
|
||||
// Saves may contain some blocks without key.
|
||||
// Such blocks are skipped because idk where to put them.
|
||||
// Example: `technology_group=tech_cannorian{ }
|
||||
// { } { } { }`
|
||||
if (tok.type == TokenType.BracketOpen)
|
||||
{
|
||||
SkipObject();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (tok.type != TokenType.StringOrNumber)
|
||||
throw new UnexpectedTokenException(tok);
|
||||
|
||||
var keySB = tok.value!;
|
||||
|
||||
// next token should be `=` or `{`
|
||||
if (!_tokens.MoveNext())
|
||||
throw new UnexpectedTokenException(tok);
|
||||
tok = _tokens.Current.Value;
|
||||
if (tok.type == TokenType.Equals)
|
||||
{
|
||||
// skip `=`
|
||||
if (!_tokens.MoveNext())
|
||||
throw new UnexpectedTokenException(tok);
|
||||
}
|
||||
// Saves may contain object definition without `=`.
|
||||
// Example: `map_area_data {` instead of `map_area_data = {`
|
||||
else if (tok.type != TokenType.BracketOpen)
|
||||
{
|
||||
throw new UnexpectedTokenException(tok);
|
||||
}
|
||||
|
||||
ISearchExpression? searchExprNext = null;
|
||||
if (_searchExprCurrent != null
|
||||
&& !_searchExprCurrent.DoesMatch(new SearchArgs(localIndex, keySB), out searchExprNext))
|
||||
{
|
||||
if(!SkipValue())
|
||||
throw new UnexpectedTokenException(_tokens.Current.Value);
|
||||
_stringBuilderPool.Return(keySB);
|
||||
continue;
|
||||
}
|
||||
|
||||
var searExpressionPrevious = _searchExprCurrent;
|
||||
_searchExprCurrent = searchExprNext;
|
||||
object? value = ParseValue();
|
||||
if (value is null)
|
||||
throw new UnexpectedTokenException(_tokens.Current.Value);
|
||||
_searchExprCurrent = searExpressionPrevious;
|
||||
|
||||
string keyStr = keySB.ToString();
|
||||
_stringBuilderPool.Return(keySB);
|
||||
|
||||
// Paradox save format has another way of defining list:
|
||||
// a = 1
|
||||
// a = 2
|
||||
// It means `a = { 1 2 }`
|
||||
if (dict.TryGetValue(keyStr, out var firstValue))
|
||||
{
|
||||
// Do dot add empty collections into list.
|
||||
// `key:{}` is okay, but i don't want to see `key:[{},{},{},{},{},{}]`
|
||||
if (IsEmptyCollection(value))
|
||||
continue;
|
||||
|
||||
if (firstValue is List<object> existingList)
|
||||
existingList.Add(value);
|
||||
else dict[keyStr] = new List<object> { firstValue, value };
|
||||
}
|
||||
else
|
||||
{
|
||||
dict.Add(keyStr, value);
|
||||
}
|
||||
}
|
||||
|
||||
return dict;
|
||||
}
|
||||
|
||||
public Dictionary<string, object> Parse()
|
||||
{
|
||||
var root = ParseDict();
|
||||
return root;
|
||||
}
|
||||
|
||||
protected enum TokenType : byte
|
||||
{
|
||||
Invalid,
|
||||
StringOrNumber,
|
||||
Equals,
|
||||
BracketOpen,
|
||||
BracketClose
|
||||
}
|
||||
|
||||
protected struct Token
|
||||
{
|
||||
public required TokenType type;
|
||||
public required short column;
|
||||
public required int line;
|
||||
public StringBuilder? value;
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
string s;
|
||||
switch (type)
|
||||
{
|
||||
case TokenType.Invalid:
|
||||
s = "INVALID_TOKEN";
|
||||
break;
|
||||
case TokenType.StringOrNumber:
|
||||
if (value == null || value.Length == 0)
|
||||
s = "NULL";
|
||||
else s = value.ToString();
|
||||
break;
|
||||
case TokenType.Equals:
|
||||
s = "=";
|
||||
break;
|
||||
case TokenType.BracketOpen:
|
||||
s = "{";
|
||||
break;
|
||||
case TokenType.BracketClose:
|
||||
s = "}";
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(type.ToString());
|
||||
}
|
||||
|
||||
return $"{line}:{column} '{s}'";
|
||||
}
|
||||
}
|
||||
|
||||
protected class UnexpectedTokenException : Exception
|
||||
{
|
||||
public UnexpectedTokenException(Token token) :
|
||||
base($"Unexpected token: {token}")
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
global using System;
|
||||
global using System.Collections.Generic;
|
||||
global using System.IO;
|
||||
global using System.Text;
|
||||
using System.Globalization;
|
||||
|
||||
namespace ParadoxSaveParser.Lib;
|
||||
|
||||
/// <summary>
|
||||
/// Sequential parser that doesn't cache anything.
|
||||
/// </summary>
|
||||
public class SaveParserEU4
|
||||
{
|
||||
private const int DefaultBufferSize = 64 * 1024;
|
||||
private static ReadOnlySpan<byte> Header => "EU4txt"u8;
|
||||
|
||||
private readonly Tokenizer _tokens;
|
||||
private ISearchExpression? _searchExprCurrent;
|
||||
|
||||
/// <summary>
|
||||
/// Encoding of the strings inside the save. Saves of different localizations use
|
||||
/// different ones, so it can be changed; <see cref="System.Text.Encoding.Latin1" /> maps
|
||||
/// every byte to one character and never fails, which makes it a safe default.
|
||||
/// </summary>
|
||||
public Encoding Encoding { get; }
|
||||
|
||||
/// <param name="savefile">
|
||||
/// Uncompressed stream of <c>gamestate</c> file which can be extracted from save archive
|
||||
/// </param>
|
||||
/// <param name="query">
|
||||
/// Parsing whole save takes 10 seconds on mid pc and takes 1GB of RAM,
|
||||
/// so you should specify what exactly you want to get from save file
|
||||
/// </param>
|
||||
/// <param name="encoding">Encoding of the strings inside the save. Latin1 by default.</param>
|
||||
/// <param name="bufferSize">Size of the read buffer. Mostly useful for tests.</param>
|
||||
public SaveParserEU4(Stream savefile, ISearchExpression? query,
|
||||
Encoding? encoding = null, int bufferSize = DefaultBufferSize)
|
||||
{
|
||||
Encoding = encoding ?? Encoding.Latin1;
|
||||
_searchExprCurrent = query;
|
||||
_tokens = new Tokenizer(savefile, bufferSize);
|
||||
}
|
||||
|
||||
public Dictionary<string, object> Parse()
|
||||
{
|
||||
_tokens.ReadHeader(Header, Encoding);
|
||||
return ParseDict();
|
||||
}
|
||||
|
||||
// doesn't move next
|
||||
private object? ParseValue()
|
||||
{
|
||||
switch (_tokens.Type)
|
||||
{
|
||||
case TokenType.StringOrNumber:
|
||||
return ParseScalar(_tokens.Text);
|
||||
case TokenType.BracketOpen:
|
||||
return ParseListOrDict();
|
||||
case TokenType.BracketClose:
|
||||
return null;
|
||||
default:
|
||||
throw new UnexpectedTokenException(_tokens, Encoding);
|
||||
}
|
||||
}
|
||||
|
||||
private object ParseScalar(ReadOnlySpan<byte> text)
|
||||
{
|
||||
// string values can be empty
|
||||
if (text.Length == 0)
|
||||
return string.Empty;
|
||||
if (text.SequenceEqual("yes"u8))
|
||||
return true;
|
||||
if (text.SequenceEqual("no"u8))
|
||||
return false;
|
||||
|
||||
byte first = text[0];
|
||||
if (first != (byte)'-' && !char.IsAsciiDigit((char)first))
|
||||
return DecodeString(text);
|
||||
if (text.Contains((byte)'.')
|
||||
&& double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out double d))
|
||||
return d;
|
||||
if (long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out long l))
|
||||
return l;
|
||||
return DecodeString(text);
|
||||
}
|
||||
|
||||
private string DecodeString(ReadOnlySpan<byte> text)
|
||||
{
|
||||
// Skip control characters, which are invisible and causing frontend bugs.
|
||||
// I dont know why there are so many of them in strings.
|
||||
if (!text.ContainsAnyInRange((byte)0, (byte)0x1F))
|
||||
return Encoding.GetString(text);
|
||||
|
||||
Span<byte> cleaned = text.Length <= 256 ? stackalloc byte[text.Length] : new byte[text.Length];
|
||||
int length = 0;
|
||||
foreach (byte b in text)
|
||||
if (b >= 0x20)
|
||||
cleaned[length++] = b;
|
||||
return Encoding.GetString(cleaned[..length]);
|
||||
}
|
||||
|
||||
// skips next value
|
||||
/// <returns>true if skipped value, false if current token is closing bracket</returns>
|
||||
private bool SkipValue()
|
||||
{
|
||||
switch (_tokens.Type)
|
||||
{
|
||||
case TokenType.BracketOpen:
|
||||
_tokens.SkipBlock();
|
||||
return true;
|
||||
case TokenType.StringOrNumber:
|
||||
return true;
|
||||
case TokenType.BracketClose:
|
||||
return false;
|
||||
default:
|
||||
throw new UnexpectedTokenException(_tokens, Encoding);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsEmptyCollection(object value)
|
||||
=> value is Dictionary<string, object> { Count: 0 } or List<object> { Count: 0 };
|
||||
|
||||
// doesn't move next
|
||||
private object ParseListOrDict()
|
||||
{
|
||||
if (_tokens.PeekType(1) == TokenType.StringOrNumber && _tokens.PeekType(2) == TokenType.Equals)
|
||||
return ParseDict();
|
||||
|
||||
return ParseList();
|
||||
}
|
||||
|
||||
// moves next
|
||||
private List<object> ParseList()
|
||||
{
|
||||
List<object> list = new();
|
||||
for (int i = 0; ; i++)
|
||||
{
|
||||
if (!_tokens.Read())
|
||||
throw new Exception("Unexpected end of file");
|
||||
|
||||
ISearchExpression? searchExprNext = null;
|
||||
if (_searchExprCurrent != null
|
||||
&& !_searchExprCurrent.DoesMatch(new MatchCandidate(i), out searchExprNext))
|
||||
{
|
||||
if (!SkipValue())
|
||||
break;
|
||||
continue;
|
||||
}
|
||||
|
||||
var searchExprPrev = _searchExprCurrent;
|
||||
_searchExprCurrent = searchExprNext;
|
||||
object? value = ParseValue();
|
||||
_searchExprCurrent = searchExprPrev;
|
||||
if (value is null)
|
||||
break;
|
||||
|
||||
// do dot add empty collections into list
|
||||
if (IsEmptyCollection(value))
|
||||
continue;
|
||||
|
||||
list.Add(value);
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
// moves next
|
||||
private Dictionary<string, object> ParseDict()
|
||||
{
|
||||
Dictionary<string, object> dict = new();
|
||||
|
||||
// root is a dict without closing bracket, so this method must check for end of file
|
||||
for (int localIndex = 0; _tokens.Read(); localIndex++)
|
||||
{
|
||||
// end of dictionary
|
||||
if (_tokens.Type == TokenType.BracketClose)
|
||||
break;
|
||||
|
||||
// Saves may contain some blocks without key.
|
||||
// Such blocks are skipped because idk where to put them.
|
||||
// Example: `technology_group=tech_cannorian{ }
|
||||
// { } { } { }`
|
||||
if (_tokens.Type == TokenType.BracketOpen)
|
||||
{
|
||||
_tokens.SkipBlock();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (_tokens.Type != TokenType.StringOrNumber)
|
||||
throw new UnexpectedTokenException(_tokens, Encoding);
|
||||
|
||||
// The key is matched before the value is read, so that a key rejected by the query
|
||||
// never has to become a string.
|
||||
ISearchExpression? searchExprNext = null;
|
||||
bool matches = _searchExprCurrent == null
|
||||
|| _searchExprCurrent.DoesMatch(
|
||||
new MatchCandidate(localIndex, _tokens.Text, Encoding), out searchExprNext);
|
||||
string? keyStr = matches ? DecodeString(_tokens.Text) : null;
|
||||
|
||||
// next token should be `=` or `{`
|
||||
if (!_tokens.Read())
|
||||
throw new UnexpectedTokenException(_tokens, Encoding);
|
||||
if (_tokens.Type == TokenType.Equals)
|
||||
{
|
||||
// skip `=`
|
||||
if (!_tokens.Read())
|
||||
throw new UnexpectedTokenException(_tokens, Encoding);
|
||||
}
|
||||
// Saves may contain object definition without `=`.
|
||||
// Example: `map_area_data {` instead of `map_area_data = {`
|
||||
else if (_tokens.Type != TokenType.BracketOpen)
|
||||
{
|
||||
throw new UnexpectedTokenException(_tokens, Encoding);
|
||||
}
|
||||
|
||||
if (!matches)
|
||||
{
|
||||
if (!SkipValue())
|
||||
throw new UnexpectedTokenException(_tokens, Encoding);
|
||||
continue;
|
||||
}
|
||||
|
||||
var searExpressionPrevious = _searchExprCurrent;
|
||||
_searchExprCurrent = searchExprNext;
|
||||
object? value = ParseValue();
|
||||
if (value is null)
|
||||
throw new UnexpectedTokenException(_tokens, Encoding);
|
||||
_searchExprCurrent = searExpressionPrevious;
|
||||
|
||||
// Paradox save format has another way of defining list:
|
||||
// a = 1
|
||||
// a = 2
|
||||
// It means `a = { 1 2 }`
|
||||
if (dict.TryGetValue(keyStr!, out var firstValue))
|
||||
{
|
||||
// Do dot add empty collections into list.
|
||||
// `key:{}` is okay, but i don't want to see `key:[{},{},{},{},{},{}]`
|
||||
if (IsEmptyCollection(value))
|
||||
continue;
|
||||
|
||||
if (firstValue is List<object> existingList)
|
||||
existingList.Add(value);
|
||||
else dict[keyStr!] = new List<object> { firstValue, value };
|
||||
}
|
||||
else
|
||||
{
|
||||
dict.Add(keyStr!, value);
|
||||
}
|
||||
}
|
||||
|
||||
return dict;
|
||||
}
|
||||
|
||||
internal class UnexpectedTokenException : Exception
|
||||
{
|
||||
public UnexpectedTokenException(Tokenizer tokens, Encoding encoding) :
|
||||
base($"Unexpected token: {tokens.Line}:{tokens.Column} '{tokens.Describe(encoding)}'")
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,29 +1,37 @@
|
||||
namespace ParadoxSaveParser.Lib;
|
||||
|
||||
public readonly record struct SearchArgs
|
||||
/// <summary>
|
||||
/// The node a search expression is tested against: its key inside the parent dictionary,
|
||||
/// still in the raw bytes of the save, and its position inside the parent list.
|
||||
/// Keys stay undecoded so that a key rejected by the query never becomes a string.
|
||||
/// </summary>
|
||||
public readonly ref struct MatchCandidate
|
||||
{
|
||||
public readonly string KeyStr;
|
||||
public readonly StringBuilder? KeySB;
|
||||
public readonly int LocalIndex;
|
||||
public readonly ReadOnlySpan<byte> Key;
|
||||
public readonly int Index;
|
||||
|
||||
public SearchArgs(int localIndex, string keyStr)
|
||||
/// <summary>Encoding <see cref="Key" /> is written in.</summary>
|
||||
public readonly Encoding Encoding;
|
||||
|
||||
/// <summary>A list item, which has a position but no key.</summary>
|
||||
public MatchCandidate(int index)
|
||||
{
|
||||
KeyStr = keyStr;
|
||||
KeySB = null;
|
||||
LocalIndex = localIndex;
|
||||
Key = default;
|
||||
Index = index;
|
||||
Encoding = Encoding.Latin1;
|
||||
}
|
||||
|
||||
public SearchArgs(int localIndex, StringBuilder keySb)
|
||||
|
||||
public MatchCandidate(int index, ReadOnlySpan<byte> key, Encoding encoding)
|
||||
{
|
||||
KeyStr = string.Empty;
|
||||
KeySB = keySb;
|
||||
LocalIndex = localIndex;
|
||||
Key = key;
|
||||
Index = index;
|
||||
Encoding = encoding;
|
||||
}
|
||||
}
|
||||
|
||||
public interface ISearchExpression
|
||||
{
|
||||
bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression);
|
||||
bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression);
|
||||
}
|
||||
|
||||
public static class SearchExpressionCompiler
|
||||
@@ -108,7 +116,7 @@ public static class SearchExpressionCompiler
|
||||
|
||||
private record AnyMatchExpression(ISearchExpression? next) : ISearchExpression
|
||||
{
|
||||
public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
|
||||
public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
|
||||
{
|
||||
nextSearchExpression = next;
|
||||
return true;
|
||||
@@ -117,7 +125,7 @@ public static class SearchExpressionCompiler
|
||||
|
||||
private record NoMatchExpression : ISearchExpression
|
||||
{
|
||||
public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
|
||||
public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
|
||||
{
|
||||
nextSearchExpression = null;
|
||||
return false;
|
||||
@@ -126,10 +134,10 @@ public static class SearchExpressionCompiler
|
||||
|
||||
private record MultipleMatchExpression(List<ISearchExpression> subExprs) : ISearchExpression
|
||||
{
|
||||
public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
|
||||
public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
|
||||
{
|
||||
foreach (var e in subExprs)
|
||||
if (e.DoesMatch(args, out nextSearchExpression))
|
||||
if (e.DoesMatch(candidate, out nextSearchExpression))
|
||||
return true;
|
||||
|
||||
nextSearchExpression = null;
|
||||
@@ -139,9 +147,9 @@ public static class SearchExpressionCompiler
|
||||
|
||||
private record IndexMatchExpression(int index, ISearchExpression? next) : ISearchExpression
|
||||
{
|
||||
public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
|
||||
public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
|
||||
{
|
||||
if (args.LocalIndex == index)
|
||||
if (candidate.Index == index)
|
||||
{
|
||||
nextSearchExpression = next;
|
||||
return true;
|
||||
@@ -154,9 +162,20 @@ public static class SearchExpressionCompiler
|
||||
|
||||
private record ExactMatchExpression(string key, ISearchExpression? next) : ISearchExpression
|
||||
{
|
||||
public bool DoesMatch(SearchArgs args, out ISearchExpression? nextSearchExpression)
|
||||
// the key is compared as bytes, so it is encoded once for whatever encoding the
|
||||
// parser reads the save in
|
||||
private byte[]? _keyBytes;
|
||||
private Encoding? _keyEncoding;
|
||||
|
||||
public bool DoesMatch(MatchCandidate candidate, out ISearchExpression? nextSearchExpression)
|
||||
{
|
||||
if ((args.KeySB != null && args.KeySB.Equals(key)) || args.KeyStr == key)
|
||||
if (!ReferenceEquals(_keyEncoding, candidate.Encoding))
|
||||
{
|
||||
_keyEncoding = candidate.Encoding;
|
||||
_keyBytes = candidate.Encoding.GetBytes(key);
|
||||
}
|
||||
|
||||
if (candidate.Key.SequenceEqual(_keyBytes))
|
||||
{
|
||||
nextSearchExpression = next;
|
||||
return true;
|
||||
|
||||
@@ -0,0 +1,456 @@
|
||||
using System.Buffers;
|
||||
|
||||
namespace ParadoxSaveParser.Lib;
|
||||
|
||||
internal enum TokenType : byte
|
||||
{
|
||||
// default value, so a slot that was never scanned is recognizably empty
|
||||
Invalid,
|
||||
// any bare or quoted value: the tokenizer does not tell numbers from strings
|
||||
StringOrNumber,
|
||||
Equals,
|
||||
BracketOpen,
|
||||
BracketClose,
|
||||
EndOfFile
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Splits a Paradox text save into tokens, reading the stream through a reusable buffer.
|
||||
/// Token text is exposed as a span into that buffer, so a token costs no allocation at all
|
||||
/// unless it happens to straddle a buffer refill.
|
||||
/// <para>
|
||||
/// The parser can tell the tokenizer to throw away a whole <c>{...}</c> block with
|
||||
/// <see cref="SkipBlock" />. Blocks rejected by the search expression are then never
|
||||
/// turned into tokens or strings, which is what makes a query cheaper than a full parse.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal sealed class Tokenizer
|
||||
{
|
||||
/// <summary>Bytes that end an unquoted token.</summary>
|
||||
private static readonly SearchValues<byte> TokenEnd = SearchValues.Create(" \t\r\n={}\""u8);
|
||||
|
||||
private static readonly SearchValues<byte> Whitespace = SearchValues.Create(" \t\r\n"u8);
|
||||
|
||||
/// <summary>Everything <see cref="SkipBlock" /> has to look at while counting depth.</summary>
|
||||
private static readonly SearchValues<byte> BlockChars = SearchValues.Create("{}\""u8);
|
||||
|
||||
/// <summary>One scanned token. Reused forever, so scanning a token allocates nothing.</summary>
|
||||
private sealed class Slot
|
||||
{
|
||||
public TokenType Type;
|
||||
// position of the token's first byte in the file, for error messages
|
||||
public int Line;
|
||||
public short Column;
|
||||
|
||||
// text is either a range of the shared buffer, or, once a refill would overwrite it,
|
||||
// a copy in Scratch
|
||||
public int Start;
|
||||
public int Length;
|
||||
// grown on demand and kept between tokens, so long tokens stop reallocating after a while
|
||||
public byte[] Scratch = [];
|
||||
public bool InScratch;
|
||||
}
|
||||
|
||||
private readonly Stream _stream;
|
||||
private readonly byte[] _buffer;
|
||||
|
||||
// current token plus up to two lookahead tokens
|
||||
private readonly Slot[] _slots = [new Slot(), new Slot(), new Slot()];
|
||||
// index of the current token; the slots are used as a ring, so the array is never shifted
|
||||
private int _current;
|
||||
// how many slots after _current already hold a scanned, not yet consumed token
|
||||
private int _lookahead;
|
||||
|
||||
// read position and amount of valid data in _buffer
|
||||
private int _pos;
|
||||
private int _length;
|
||||
// set once the stream has no more bytes; stops Fill() from calling Read() again
|
||||
private bool _eof;
|
||||
// position of _pos in the file; _column is 0-based here and reported 1-based
|
||||
private int _line = 1;
|
||||
private int _column;
|
||||
|
||||
public Tokenizer(Stream stream, int bufferSize)
|
||||
{
|
||||
if (bufferSize < 16)
|
||||
throw new ArgumentOutOfRangeException(nameof(bufferSize), bufferSize, "buffer is too small");
|
||||
_stream = stream;
|
||||
_buffer = new byte[bufferSize];
|
||||
}
|
||||
|
||||
public TokenType Type => _slots[_current].Type;
|
||||
public int Line => _slots[_current].Line;
|
||||
public short Column => _slots[_current].Column;
|
||||
|
||||
/// <summary>
|
||||
/// Bytes of the current token, without quotes. Valid only until the next
|
||||
/// <see cref="Read" />, because the buffer underneath it gets reused.
|
||||
/// </summary>
|
||||
public ReadOnlySpan<byte> Text
|
||||
{
|
||||
get
|
||||
{
|
||||
var slot = _slots[_current];
|
||||
// a token that survived a refill was copied out; everything else still points into the buffer
|
||||
return slot.InScratch
|
||||
? slot.Scratch.AsSpan(0, slot.Length)
|
||||
: _buffer.AsSpan(slot.Start, slot.Length);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Consumes the file's magic header and checks it, before any token is scanned.</summary>
|
||||
public void ReadHeader(ReadOnlySpan<byte> expected, Encoding encoding)
|
||||
{
|
||||
// read straight from the stream: this runs before the buffer holds anything
|
||||
Span<byte> head = stackalloc byte[expected.Length];
|
||||
_stream.ReadExactly(head);
|
||||
if (!head.SequenceEqual(expected))
|
||||
throw new Exception($"Invalid gamestate header. " +
|
||||
$"Expected '{encoding.GetString(expected)}', got '{encoding.GetString(head)}'.");
|
||||
}
|
||||
|
||||
/// <returns>false at end of file</returns>
|
||||
public bool Read()
|
||||
{
|
||||
_current = NextSlot(_current);
|
||||
// the next slot may already be filled by an earlier PeekType, then there is nothing to scan
|
||||
if (_lookahead > 0)
|
||||
_lookahead--;
|
||||
else Scan(_slots[_current]);
|
||||
return _slots[_current].Type != TokenType.EndOfFile;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Type of a token that has not been consumed yet, 1 or 2 tokens ahead of the current one.
|
||||
/// Only types are available: the parser never needs the text of a token it has not reached.
|
||||
/// </summary>
|
||||
public TokenType PeekType(int offset)
|
||||
{
|
||||
// scan only as far as asked, so lookahead never runs into a block SkipBlock is about to drop
|
||||
while (_lookahead < offset)
|
||||
{
|
||||
// first slot after the ones that are already filled
|
||||
int slot = _current;
|
||||
for (int i = 0; i <= _lookahead; i++)
|
||||
slot = NextSlot(slot);
|
||||
Scan(_slots[slot]);
|
||||
_lookahead++;
|
||||
}
|
||||
|
||||
int index = _current;
|
||||
for (int i = 0; i < offset; i++)
|
||||
index = NextSlot(index);
|
||||
return _slots[index].Type;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Throws away the block opened by the current <c>{</c> token without tokenizing it:
|
||||
/// raw bytes are scanned for braces until the depth returns to zero.
|
||||
/// Braces inside quoted strings are text and do not change the depth.
|
||||
/// Leaves the closing <c>}</c> as the current token.
|
||||
/// </summary>
|
||||
public void SkipBlock()
|
||||
{
|
||||
int depth = 1;
|
||||
|
||||
// tokens that lookahead already pulled out of the buffer still count towards the depth
|
||||
while (depth > 0 && _lookahead > 0)
|
||||
{
|
||||
_current = NextSlot(_current);
|
||||
_lookahead--;
|
||||
switch (_slots[_current].Type)
|
||||
{
|
||||
case TokenType.BracketOpen:
|
||||
depth++;
|
||||
break;
|
||||
case TokenType.BracketClose:
|
||||
depth--;
|
||||
break;
|
||||
case TokenType.EndOfFile:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
bool inQuotes = false;
|
||||
while (depth > 0)
|
||||
{
|
||||
if (_pos >= _length && !Fill())
|
||||
break; // unbalanced braces: the file ended inside the block
|
||||
|
||||
var span = _buffer.AsSpan(_pos, _length - _pos);
|
||||
// inside a string only the closing quote matters, braces there are ordinary characters
|
||||
int i = inQuotes ? span.IndexOf((byte)'\"') : span.IndexOfAny(BlockChars);
|
||||
if (i < 0)
|
||||
{
|
||||
// nothing interesting in this bufferful, drop all of it and refill
|
||||
Consume(span.Length);
|
||||
continue;
|
||||
}
|
||||
|
||||
byte b = span[i];
|
||||
Consume(i + 1);
|
||||
if (b == (byte)'\"')
|
||||
inQuotes = !inQuotes;
|
||||
else if (b == (byte)'{')
|
||||
depth++;
|
||||
else depth--;
|
||||
}
|
||||
|
||||
// hand the parser the closing brace it expects, without having tokenized anything inside
|
||||
var current = _slots[_current];
|
||||
current.Type = depth == 0 ? TokenType.BracketClose : TokenType.EndOfFile;
|
||||
current.Length = 0;
|
||||
current.InScratch = false;
|
||||
current.Line = _line;
|
||||
current.Column = (short)_column;
|
||||
}
|
||||
|
||||
public string Describe(Encoding encoding) => Type switch
|
||||
{
|
||||
TokenType.StringOrNumber => Text.Length == 0 ? "NULL" : encoding.GetString(Text),
|
||||
TokenType.Equals => "=",
|
||||
TokenType.BracketOpen => "{",
|
||||
TokenType.BracketClose => "}",
|
||||
TokenType.EndOfFile => "END_OF_FILE",
|
||||
_ => "INVALID_TOKEN",
|
||||
};
|
||||
|
||||
/// <summary>Slot indices wrap around: the three slots form a ring buffer.</summary>
|
||||
private int NextSlot(int i) => i + 1 == _slots.Length ? 0 : i + 1;
|
||||
|
||||
/// <summary>Reads the next token from the stream into <paramref name="slot" />.</summary>
|
||||
private void Scan(Slot slot)
|
||||
{
|
||||
// loops only to skip comments, which produce no token
|
||||
while (true)
|
||||
{
|
||||
if (!SkipWhitespace())
|
||||
{
|
||||
slot.Type = TokenType.EndOfFile;
|
||||
slot.Length = 0;
|
||||
slot.InScratch = false;
|
||||
slot.Line = _line;
|
||||
slot.Column = (short)_column;
|
||||
return;
|
||||
}
|
||||
|
||||
slot.Line = _line;
|
||||
slot.Column = (short)(_column + 1); // columns are reported 1-based
|
||||
// SkipWhitespace guarantees at least one buffered byte here
|
||||
byte b = _buffer[_pos];
|
||||
switch (b)
|
||||
{
|
||||
case (byte)'=':
|
||||
ConsumeFlat(1);
|
||||
SetDelimiter(slot, TokenType.Equals);
|
||||
return;
|
||||
case (byte)'{':
|
||||
ConsumeFlat(1);
|
||||
SetDelimiter(slot, TokenType.BracketOpen);
|
||||
return;
|
||||
case (byte)'}':
|
||||
ConsumeFlat(1);
|
||||
SetDelimiter(slot, TokenType.BracketClose);
|
||||
return;
|
||||
case (byte)'\"':
|
||||
ReadQuoted(slot);
|
||||
return;
|
||||
default:
|
||||
ReadBare(slot);
|
||||
// comments are dropped, same as before: a token starting with '#' is not emitted
|
||||
if (slot.Length > 0 && FirstByte(slot) == (byte)'#')
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Single-character tokens carry no text of their own.</summary>
|
||||
private static void SetDelimiter(Slot slot, TokenType type)
|
||||
{
|
||||
slot.Type = type;
|
||||
slot.Length = 0;
|
||||
slot.InScratch = false;
|
||||
}
|
||||
|
||||
/// <summary>Reads an unquoted token, which ends at the first delimiter byte.</summary>
|
||||
private void ReadBare(Slot slot)
|
||||
{
|
||||
StartText(slot);
|
||||
while (true)
|
||||
{
|
||||
var span = _buffer.AsSpan(_pos, _length - _pos);
|
||||
// one vectorized search replaces a loop over the token's bytes
|
||||
int end = span.IndexOfAny(TokenEnd);
|
||||
if (end >= 0)
|
||||
{
|
||||
// the delimiter itself is left for the next Scan to classify
|
||||
AppendText(slot, span[..end]);
|
||||
ConsumeFlat(end);
|
||||
return;
|
||||
}
|
||||
|
||||
// token runs to the end of the buffer and continues in the next one
|
||||
AppendText(slot, span);
|
||||
ConsumeFlat(span.Length);
|
||||
if (!Fill())
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a quoted token. Everything up to the next <c>"</c> is text, braces included;
|
||||
/// the format has no escape sequences, so a quote always closes the string.
|
||||
/// </summary>
|
||||
private void ReadQuoted(Slot slot)
|
||||
{
|
||||
ConsumeFlat(1); // opening quote
|
||||
StartText(slot);
|
||||
while (true)
|
||||
{
|
||||
if (_pos >= _length && !Fill())
|
||||
return; // unterminated string at end of file
|
||||
|
||||
var span = _buffer.AsSpan(_pos, _length - _pos);
|
||||
int end = span.IndexOf((byte)'\"');
|
||||
if (end >= 0)
|
||||
{
|
||||
AppendText(slot, span[..end]);
|
||||
Consume(end + 1); // content plus the closing quote
|
||||
return;
|
||||
}
|
||||
|
||||
// Consume, not ConsumeFlat: a quoted value is allowed to span several lines
|
||||
AppendText(slot, span);
|
||||
Consume(span.Length);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Begins a text token at the current read position.</summary>
|
||||
private void StartText(Slot slot)
|
||||
{
|
||||
slot.Type = TokenType.StringOrNumber;
|
||||
slot.Start = _pos;
|
||||
slot.Length = 0;
|
||||
slot.InScratch = false;
|
||||
}
|
||||
|
||||
/// <summary>Extends a text token by one chunk of bytes taken from the buffer.</summary>
|
||||
private void AppendText(Slot slot, ReadOnlySpan<byte> chunk)
|
||||
{
|
||||
if (!slot.InScratch)
|
||||
{
|
||||
// chunks always continue where the previous one ended, so the range just grows
|
||||
// still contiguous in the buffer, nothing to copy
|
||||
if (slot.Length == 0)
|
||||
slot.Start = _pos;
|
||||
slot.Length += chunk.Length;
|
||||
return;
|
||||
}
|
||||
|
||||
// the start of this token is already out of the buffer, so the rest has to follow it
|
||||
EnsureScratch(slot, slot.Length + chunk.Length);
|
||||
chunk.CopyTo(slot.Scratch.AsSpan(slot.Length));
|
||||
slot.Length += chunk.Length;
|
||||
}
|
||||
|
||||
/// <summary>First byte of a token, wherever its text currently lives. Used to spot comments.</summary>
|
||||
private byte FirstByte(Slot slot) => slot.InScratch ? slot.Scratch[0] : _buffer[slot.Start];
|
||||
|
||||
private static void EnsureScratch(Slot slot, int size)
|
||||
{
|
||||
if (slot.Scratch.Length >= size)
|
||||
return;
|
||||
// doubling keeps a token that is appended chunk by chunk from resizing on every chunk
|
||||
int capacity = Math.Max(size, slot.Scratch.Length * 2);
|
||||
Array.Resize(ref slot.Scratch, capacity);
|
||||
}
|
||||
|
||||
/// <summary>Moves the read position onto the next non-whitespace byte.</summary>
|
||||
/// <returns>false at end of file</returns>
|
||||
private bool SkipWhitespace()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if (_pos >= _length && !Fill())
|
||||
return false;
|
||||
|
||||
var span = _buffer.AsSpan(_pos, _length - _pos);
|
||||
int i = span.IndexOfAnyExcept(Whitespace);
|
||||
if (i < 0)
|
||||
{
|
||||
// whitespace to the end of the buffer, keep going in the next one
|
||||
Consume(span.Length);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (i > 0)
|
||||
Consume(i);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Refills the buffer from the stream.</summary>
|
||||
/// <returns>false if the stream is exhausted</returns>
|
||||
private bool Fill()
|
||||
{
|
||||
if (_eof)
|
||||
return false;
|
||||
|
||||
// text of live tokens lives in the buffer, so it has to be copied out before overwriting
|
||||
MaterializeSlots();
|
||||
// a short read is fine, the next Fill picks up the rest
|
||||
_length = _stream.Read(_buffer, 0, _buffer.Length);
|
||||
_pos = 0;
|
||||
if (_length > 0)
|
||||
return true;
|
||||
|
||||
_eof = true;
|
||||
_length = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies every token that still points into the buffer out to its own scratch array.
|
||||
/// Called just before a refill, which is the only moment that text can be lost.
|
||||
/// </summary>
|
||||
private void MaterializeSlots()
|
||||
{
|
||||
foreach (var slot in _slots)
|
||||
{
|
||||
// delimiters and empty tokens own no text, and a copied one needs no second copy
|
||||
if (slot.InScratch || slot.Length == 0 || slot.Type != TokenType.StringOrNumber)
|
||||
continue;
|
||||
EnsureScratch(slot, slot.Length);
|
||||
_buffer.AsSpan(slot.Start, slot.Length).CopyTo(slot.Scratch);
|
||||
slot.InScratch = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Consumes bytes that cannot contain a line break.</summary>
|
||||
private void ConsumeFlat(int count)
|
||||
{
|
||||
_pos += count;
|
||||
_column += count;
|
||||
}
|
||||
|
||||
/// <summary>Consumes bytes that may contain line breaks, keeping line and column exact.</summary>
|
||||
private void Consume(int count)
|
||||
{
|
||||
var slice = _buffer.AsSpan(_pos, count);
|
||||
// the last break decides the column, and only then is counting all of them worth it
|
||||
int lastBreak = slice.LastIndexOf((byte)'\n');
|
||||
if (lastBreak < 0)
|
||||
{
|
||||
_column += count;
|
||||
}
|
||||
else
|
||||
{
|
||||
_line += slice.Count((byte)'\n');
|
||||
// bytes left after the final break
|
||||
_column = count - lastBreak - 1;
|
||||
}
|
||||
|
||||
_pos += count;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user