Files
Dec2017-Script-Dump/protobuf3/JsonTokenizer.cs
T
2026-06-04 11:42:34 +02:00

695 lines
20 KiB
C#

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Text;
namespace Google.Protobuf
{
// Token: 0x02000012 RID: 18
internal abstract class JsonTokenizer
{
// Token: 0x06000121 RID: 289 RVA: 0x0000751F File Offset: 0x0000571F
internal static JsonTokenizer FromTextReader(TextReader reader)
{
return new JsonTokenizer.JsonTextTokenizer(reader);
}
// Token: 0x06000122 RID: 290 RVA: 0x00007527 File Offset: 0x00005727
internal static JsonTokenizer FromReplayedTokens(IList<JsonToken> tokens, JsonTokenizer continuation)
{
return new JsonTokenizer.JsonReplayTokenizer(tokens, continuation);
}
// Token: 0x17000023 RID: 35
// (get) Token: 0x06000123 RID: 291 RVA: 0x00007530 File Offset: 0x00005730
// (set) Token: 0x06000124 RID: 292 RVA: 0x00007538 File Offset: 0x00005738
internal int ObjectDepth { get; private set; }
// Token: 0x06000125 RID: 293 RVA: 0x00007544 File Offset: 0x00005744
internal void PushBack(JsonToken token)
{
if (this.bufferedToken != null)
{
throw new InvalidOperationException("Can't push back twice");
}
this.bufferedToken = token;
if (token.Type == JsonToken.TokenType.StartObject)
{
int num = this.ObjectDepth;
this.ObjectDepth = num - 1;
return;
}
if (token.Type == JsonToken.TokenType.EndObject)
{
int num = this.ObjectDepth;
this.ObjectDepth = num + 1;
}
}
// Token: 0x06000126 RID: 294 RVA: 0x000075A0 File Offset: 0x000057A0
internal JsonToken Next()
{
JsonToken jsonToken;
if (this.bufferedToken != null)
{
jsonToken = this.bufferedToken;
this.bufferedToken = null;
}
else
{
jsonToken = this.NextImpl();
}
if (jsonToken.Type == JsonToken.TokenType.StartObject)
{
int num = this.ObjectDepth;
this.ObjectDepth = num + 1;
}
else if (jsonToken.Type == JsonToken.TokenType.EndObject)
{
int num = this.ObjectDepth;
this.ObjectDepth = num - 1;
}
return jsonToken;
}
// Token: 0x06000127 RID: 295
protected abstract JsonToken NextImpl();
// Token: 0x04000046 RID: 70
private JsonToken bufferedToken;
// Token: 0x02000082 RID: 130
private class JsonReplayTokenizer : JsonTokenizer
{
// Token: 0x060006C3 RID: 1731 RVA: 0x00016852 File Offset: 0x00014A52
internal JsonReplayTokenizer(IList<JsonToken> tokens, JsonTokenizer nextTokenizer)
{
this.tokens = tokens;
this.nextTokenizer = nextTokenizer;
}
// Token: 0x060006C4 RID: 1732 RVA: 0x00016868 File Offset: 0x00014A68
protected override JsonToken NextImpl()
{
if (this.nextTokenIndex >= this.tokens.Count)
{
return this.nextTokenizer.Next();
}
IList<JsonToken> list = this.tokens;
int num = this.nextTokenIndex;
this.nextTokenIndex = num + 1;
return list[num];
}
// Token: 0x040002AB RID: 683
private readonly IList<JsonToken> tokens;
// Token: 0x040002AC RID: 684
private readonly JsonTokenizer nextTokenizer;
// Token: 0x040002AD RID: 685
private int nextTokenIndex;
}
// Token: 0x02000083 RID: 131
private sealed class JsonTextTokenizer : JsonTokenizer
{
// Token: 0x060006C5 RID: 1733 RVA: 0x000168B0 File Offset: 0x00014AB0
internal JsonTextTokenizer(TextReader reader)
{
this.reader = new JsonTokenizer.JsonTextTokenizer.PushBackReader(reader);
this.state = JsonTokenizer.JsonTextTokenizer.State.StartOfDocument;
this.containerStack.Push(JsonTokenizer.JsonTextTokenizer.ContainerType.Document);
}
// Token: 0x060006C6 RID: 1734 RVA: 0x000168E4 File Offset: 0x00014AE4
protected override JsonToken NextImpl()
{
if (this.state == JsonTokenizer.JsonTextTokenizer.State.ReaderExhausted)
{
throw new InvalidOperationException("Next() called after end of document");
}
char? c;
char value;
for (;;)
{
c = this.reader.Read();
if (c == null)
{
break;
}
value = c.Value;
if (value > ']')
{
goto IL_106;
}
if (value > ':')
{
goto IL_F1;
}
switch (value)
{
case '\t':
case '\n':
case '\r':
break;
case '\v':
case '\f':
goto IL_2BB;
default:
switch (value)
{
case ' ':
continue;
case '"':
goto IL_188;
case ',':
this.ValidateState(JsonTokenizer.JsonTextTokenizer.State.ObjectAfterProperty | JsonTokenizer.JsonTextTokenizer.State.ArrayAfterValue, "Invalid state to read a colon: ");
this.state = ((this.state == JsonTokenizer.JsonTextTokenizer.State.ObjectAfterProperty) ? JsonTokenizer.JsonTextTokenizer.State.ObjectAfterComma : JsonTokenizer.JsonTextTokenizer.State.ArrayAfterComma);
continue;
case '-':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
goto IL_29D;
case ':':
this.ValidateState(JsonTokenizer.JsonTextTokenizer.State.ObjectBeforeColon, "Invalid state to read a colon: ");
this.state = JsonTokenizer.JsonTextTokenizer.State.ObjectAfterColon;
continue;
}
goto Block_6;
}
}
this.ValidateState(JsonTokenizer.JsonTextTokenizer.State.ExpectedEndOfDocument, "Unexpected end of document in state: ");
this.state = JsonTokenizer.JsonTextTokenizer.State.ReaderExhausted;
return JsonToken.EndDocument;
Block_6:
goto IL_2BB;
IL_F1:
if (value == '[')
{
this.ValidateState(JsonTokenizer.JsonTextTokenizer.ValueStates, "Invalid state to read an open square bracket: ");
this.state = JsonTokenizer.JsonTextTokenizer.State.ArrayStart;
this.containerStack.Push(JsonTokenizer.JsonTextTokenizer.ContainerType.Array);
return JsonToken.StartArray;
}
if (value != ']')
{
goto IL_2BB;
}
this.ValidateState(JsonTokenizer.JsonTextTokenizer.State.ArrayStart | JsonTokenizer.JsonTextTokenizer.State.ArrayAfterValue, "Invalid state to read a close square bracket: ");
this.PopContainer();
return JsonToken.EndArray;
IL_106:
if (value <= 'n')
{
if (value == 'f')
{
this.ConsumeLiteral("false");
this.ValidateAndModifyStateForValue("Invalid state to read a false literal: ");
return JsonToken.False;
}
if (value != 'n')
{
goto IL_2BB;
}
this.ConsumeLiteral("null");
this.ValidateAndModifyStateForValue("Invalid state to read a null literal: ");
return JsonToken.Null;
}
else
{
if (value == 't')
{
this.ConsumeLiteral("true");
this.ValidateAndModifyStateForValue("Invalid state to read a true literal: ");
return JsonToken.True;
}
if (value == '{')
{
this.ValidateState(JsonTokenizer.JsonTextTokenizer.ValueStates, "Invalid state to read an open brace: ");
this.state = JsonTokenizer.JsonTextTokenizer.State.ObjectStart;
this.containerStack.Push(JsonTokenizer.JsonTextTokenizer.ContainerType.Object);
return JsonToken.StartObject;
}
if (value != '}')
{
goto IL_2BB;
}
this.ValidateState(JsonTokenizer.JsonTextTokenizer.State.ObjectStart | JsonTokenizer.JsonTextTokenizer.State.ObjectAfterProperty, "Invalid state to read a close brace: ");
this.PopContainer();
return JsonToken.EndObject;
}
IL_188:
string text = this.ReadString();
if ((this.state & (JsonTokenizer.JsonTextTokenizer.State.ObjectStart | JsonTokenizer.JsonTextTokenizer.State.ObjectAfterComma)) != (JsonTokenizer.JsonTextTokenizer.State)0)
{
this.state = JsonTokenizer.JsonTextTokenizer.State.ObjectBeforeColon;
return JsonToken.Name(text);
}
this.ValidateAndModifyStateForValue("Invalid state to read a double quote: ");
return JsonToken.Value(text);
IL_29D:
double num = this.ReadNumber(c.Value);
this.ValidateAndModifyStateForValue("Invalid state to read a number token: ");
return JsonToken.Value(num);
IL_2BB:
throw new InvalidJsonException("Invalid first character of token: " + c.Value.ToString());
}
// Token: 0x060006C7 RID: 1735 RVA: 0x00016BCA File Offset: 0x00014DCA
private void ValidateState(JsonTokenizer.JsonTextTokenizer.State validStates, string errorPrefix)
{
if ((validStates & this.state) == (JsonTokenizer.JsonTextTokenizer.State)0)
{
throw this.reader.CreateException(errorPrefix + this.state);
}
}
// Token: 0x060006C8 RID: 1736 RVA: 0x00016BF4 File Offset: 0x00014DF4
private string ReadString()
{
StringBuilder stringBuilder = new StringBuilder();
bool flag = false;
char c;
for (;;)
{
c = this.reader.ReadOrFail("Unexpected end of text while reading string");
if (c < ' ')
{
break;
}
if (c == '"')
{
goto Block_2;
}
if (c == '\\')
{
c = this.ReadEscapedCharacter();
}
if (flag != char.IsLowSurrogate(c))
{
goto Block_5;
}
flag = char.IsHighSurrogate(c);
stringBuilder.Append(c);
}
throw this.reader.CreateException(string.Format(CultureInfo.InvariantCulture, "Invalid character in string literal: U+{0:x4}", new object[] { (int)c }));
Block_2:
if (flag)
{
throw this.reader.CreateException("Invalid use of surrogate pair code units");
}
return stringBuilder.ToString();
Block_5:
throw this.reader.CreateException("Invalid use of surrogate pair code units");
}
// Token: 0x060006C9 RID: 1737 RVA: 0x00016CA4 File Offset: 0x00014EA4
private char ReadEscapedCharacter()
{
char c = this.reader.ReadOrFail("Unexpected end of text while reading character escape sequence");
if (c <= '\\')
{
if (c == '"')
{
return '"';
}
if (c == '/')
{
return '/';
}
if (c == '\\')
{
return '\\';
}
}
else if (c <= 'f')
{
if (c == 'b')
{
return '\b';
}
if (c == 'f')
{
return '\f';
}
}
else
{
if (c == 'n')
{
return '\n';
}
switch (c)
{
case 'r':
return '\r';
case 't':
return '\t';
case 'u':
return this.ReadUnicodeEscape();
}
}
throw this.reader.CreateException(string.Format(CultureInfo.InvariantCulture, "Invalid character in character escape sequence: U+{0:x4}", new object[] { (int)c }));
}
// Token: 0x060006CA RID: 1738 RVA: 0x00016D50 File Offset: 0x00014F50
private char ReadUnicodeEscape()
{
int num = 0;
for (int i = 0; i < 4; i++)
{
char c = this.reader.ReadOrFail("Unexpected end of text while reading Unicode escape sequence");
int num2;
if (c >= '0' && c <= '9')
{
num2 = (int)(c - '0');
}
else if (c >= 'a' && c <= 'f')
{
num2 = (int)(c - 'a' + '\n');
}
else
{
if (c < 'A' || c > 'F')
{
throw this.reader.CreateException(string.Format(CultureInfo.InvariantCulture, "Invalid character in character escape sequence: U+{0:x4}", new object[] { (int)c }));
}
num2 = (int)(c - 'A' + '\n');
}
num = (num << 4) + num2;
}
return (char)num;
}
// Token: 0x060006CB RID: 1739 RVA: 0x00016DEC File Offset: 0x00014FEC
private void ConsumeLiteral(string text)
{
for (int i = 1; i < text.Length; i++)
{
char? c = this.reader.Read();
if (c == null)
{
throw this.reader.CreateException("Unexpected end of text while reading literal token " + text);
}
if (c.Value != text[i])
{
throw this.reader.CreateException("Unexpected character while reading literal token " + text);
}
}
}
// Token: 0x060006CC RID: 1740 RVA: 0x00016E60 File Offset: 0x00015060
private double ReadNumber(char initialCharacter)
{
StringBuilder stringBuilder = new StringBuilder();
if (initialCharacter == '-')
{
stringBuilder.Append("-");
}
else
{
this.reader.PushBack(initialCharacter);
}
char? c = this.ReadInt(stringBuilder);
char? c2 = c;
if (((c2 != null) ? new int?((int)c2.GetValueOrDefault()) : null) == 46)
{
c = this.ReadFrac(stringBuilder);
}
c2 = c;
if (!(((c2 != null) ? new int?((int)c2.GetValueOrDefault()) : null) == 101))
{
c2 = c;
if (!(((c2 != null) ? new int?((int)c2.GetValueOrDefault()) : null) == 69))
{
goto IL_F7;
}
}
c = this.ReadExp(stringBuilder);
IL_F7:
if (c != null)
{
this.reader.PushBack(c.Value);
}
double num;
try
{
num = double.Parse(stringBuilder.ToString(), NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, CultureInfo.InvariantCulture);
}
catch (OverflowException)
{
throw this.reader.CreateException("Numeric value out of range: " + stringBuilder);
}
return num;
}
// Token: 0x060006CD RID: 1741 RVA: 0x00016FC4 File Offset: 0x000151C4
private char? ReadInt(StringBuilder builder)
{
char c = this.reader.ReadOrFail("Invalid numeric literal");
if (c < '0' || c > '9')
{
throw this.reader.CreateException("Invalid numeric literal");
}
builder.Append(c);
int num;
char? c2 = this.ConsumeDigits(builder, out num);
if (c == '0' && num != 0)
{
throw this.reader.CreateException("Invalid numeric literal: leading 0 for non-zero value.");
}
return c2;
}
// Token: 0x060006CE RID: 1742 RVA: 0x00017028 File Offset: 0x00015228
private char? ReadFrac(StringBuilder builder)
{
builder.Append('.');
int num;
char? c = this.ConsumeDigits(builder, out num);
if (num == 0)
{
throw this.reader.CreateException("Invalid numeric literal: fraction with no trailing digits");
}
return c;
}
// Token: 0x060006CF RID: 1743 RVA: 0x0001705C File Offset: 0x0001525C
private char? ReadExp(StringBuilder builder)
{
builder.Append('E');
char? c = this.reader.Read();
if (c == null)
{
throw this.reader.CreateException("Invalid numeric literal: exponent with no trailing digits");
}
char? c2 = c;
if (!(((c2 != null) ? new int?((int)c2.GetValueOrDefault()) : null) == 45))
{
c2 = c;
if (!(((c2 != null) ? new int?((int)c2.GetValueOrDefault()) : null) == 43))
{
this.reader.PushBack(c.Value);
goto IL_CD;
}
}
builder.Append(c.Value);
IL_CD:
int num;
c = this.ConsumeDigits(builder, out num);
if (num == 0)
{
throw this.reader.CreateException("Invalid numeric literal: exponent without value");
}
return c;
}
// Token: 0x060006D0 RID: 1744 RVA: 0x00017158 File Offset: 0x00015358
private char? ConsumeDigits(StringBuilder builder, out int count)
{
count = 0;
char? c;
for (;;)
{
c = this.reader.Read();
if (c == null || c.Value < '0' || c.Value > '9')
{
break;
}
count++;
builder.Append(c.Value);
}
return c;
}
// Token: 0x060006D1 RID: 1745 RVA: 0x000171AC File Offset: 0x000153AC
private void ValidateAndModifyStateForValue(string errorPrefix)
{
this.ValidateState(JsonTokenizer.JsonTextTokenizer.ValueStates, errorPrefix);
JsonTokenizer.JsonTextTokenizer.State state = this.state;
if (state <= JsonTokenizer.JsonTextTokenizer.State.ObjectAfterColon)
{
if (state == JsonTokenizer.JsonTextTokenizer.State.StartOfDocument)
{
this.state = JsonTokenizer.JsonTextTokenizer.State.ExpectedEndOfDocument;
return;
}
if (state == JsonTokenizer.JsonTextTokenizer.State.ObjectAfterColon)
{
this.state = JsonTokenizer.JsonTextTokenizer.State.ObjectAfterProperty;
return;
}
}
else if (state == JsonTokenizer.JsonTextTokenizer.State.ArrayStart || state == JsonTokenizer.JsonTextTokenizer.State.ArrayAfterComma)
{
this.state = JsonTokenizer.JsonTextTokenizer.State.ArrayAfterValue;
return;
}
throw new InvalidOperationException("ValidateAndModifyStateForValue does not handle all value states (and should)");
}
// Token: 0x060006D2 RID: 1746 RVA: 0x00017218 File Offset: 0x00015418
private void PopContainer()
{
this.containerStack.Pop();
JsonTokenizer.JsonTextTokenizer.ContainerType containerType = this.containerStack.Peek();
switch (containerType)
{
case JsonTokenizer.JsonTextTokenizer.ContainerType.Document:
this.state = JsonTokenizer.JsonTextTokenizer.State.ExpectedEndOfDocument;
return;
case JsonTokenizer.JsonTextTokenizer.ContainerType.Object:
this.state = JsonTokenizer.JsonTextTokenizer.State.ObjectAfterProperty;
return;
case JsonTokenizer.JsonTextTokenizer.ContainerType.Array:
this.state = JsonTokenizer.JsonTextTokenizer.State.ArrayAfterValue;
return;
default:
throw new InvalidOperationException("Unexpected container type: " + containerType);
}
}
// Token: 0x040002AE RID: 686
private static readonly JsonTokenizer.JsonTextTokenizer.State ValueStates = JsonTokenizer.JsonTextTokenizer.State.StartOfDocument | JsonTokenizer.JsonTextTokenizer.State.ObjectAfterColon | JsonTokenizer.JsonTextTokenizer.State.ArrayStart | JsonTokenizer.JsonTextTokenizer.State.ArrayAfterComma;
// Token: 0x040002AF RID: 687
private readonly Stack<JsonTokenizer.JsonTextTokenizer.ContainerType> containerStack = new Stack<JsonTokenizer.JsonTextTokenizer.ContainerType>();
// Token: 0x040002B0 RID: 688
private readonly JsonTokenizer.JsonTextTokenizer.PushBackReader reader;
// Token: 0x040002B1 RID: 689
private JsonTokenizer.JsonTextTokenizer.State state;
// Token: 0x020000D0 RID: 208
private enum ContainerType
{
// Token: 0x0400031A RID: 794
Document,
// Token: 0x0400031B RID: 795
Object,
// Token: 0x0400031C RID: 796
Array
}
// Token: 0x020000D1 RID: 209
[Flags]
private enum State
{
// Token: 0x0400031E RID: 798
StartOfDocument = 1,
// Token: 0x0400031F RID: 799
ExpectedEndOfDocument = 2,
// Token: 0x04000320 RID: 800
ReaderExhausted = 4,
// Token: 0x04000321 RID: 801
ObjectStart = 8,
// Token: 0x04000322 RID: 802
ObjectBeforeColon = 16,
// Token: 0x04000323 RID: 803
ObjectAfterColon = 32,
// Token: 0x04000324 RID: 804
ObjectAfterProperty = 64,
// Token: 0x04000325 RID: 805
ObjectAfterComma = 128,
// Token: 0x04000326 RID: 806
ArrayStart = 256,
// Token: 0x04000327 RID: 807
ArrayAfterValue = 512,
// Token: 0x04000328 RID: 808
ArrayAfterComma = 1024
}
// Token: 0x020000D2 RID: 210
private class PushBackReader
{
// Token: 0x060007AE RID: 1966 RVA: 0x00017CA7 File Offset: 0x00015EA7
internal PushBackReader(TextReader reader)
{
this.reader = reader;
}
// Token: 0x060007AF RID: 1967 RVA: 0x00017CB8 File Offset: 0x00015EB8
internal char? Read()
{
if (this.nextChar != null)
{
char? c = this.nextChar;
this.nextChar = null;
return c;
}
int num = this.reader.Read();
if (num != -1)
{
return new char?((char)num);
}
return null;
}
// Token: 0x060007B0 RID: 1968 RVA: 0x00017D08 File Offset: 0x00015F08
internal char ReadOrFail(string messageOnFailure)
{
char? c = this.Read();
if (c == null)
{
throw this.CreateException(messageOnFailure);
}
return c.Value;
}
// Token: 0x060007B1 RID: 1969 RVA: 0x00017D34 File Offset: 0x00015F34
internal void PushBack(char c)
{
if (this.nextChar != null)
{
throw new InvalidOperationException("Cannot push back when already buffering a character");
}
this.nextChar = new char?(c);
}
// Token: 0x060007B2 RID: 1970 RVA: 0x00017D5A File Offset: 0x00015F5A
internal InvalidJsonException CreateException(string message)
{
return new InvalidJsonException(message);
}
// Token: 0x04000329 RID: 809
private readonly TextReader reader;
// Token: 0x0400032A RID: 810
private char? nextChar;
}
}
}
}