Discrepancies in Utf8JsonReader between single- and multi-segment modes for invalid JSON #30706

static class ReproProgramCompare
{
    public static void Run()
    {
        RunTestCase("fals");
        RunTestCase("tb:");
        RunTestCase("{\"\":tr");
        RunTestCase("[[n{\"a\":");
        RunTestCase("f-2.2e-2,-");

        RunTestCase("/+");
        RunTestCase("{/");
        RunTestCase("{/s");
        RunTestCase("{ /");
        RunTestCase("{} /");
    }

    static void RunTestCase(string jsonString)
    {
        var result = GetResultCompare(new JsonInput(Encoding.UTF8.GetBytes(jsonString), true, false, JsonCommentHandling.Skip));

        Console.WriteLine($"Test case: " + jsonString);

        var messageOutput = $"BytesConsumed: {result.Result1.BytesConsumed}, CurrentDepth: {result.Result1.CurrentDepth}, TokenStartIndex: {result.Result1.TokenStartIndex}, TokenType: {result.Result1.TokenType}, Exception: {result.Result1.Exception?.Message}";
        messageOutput += Environment.NewLine + $"BytesConsumed: {result.Result2.BytesConsumed}, CurrentDepth: {result.Result2.CurrentDepth}, TokenStartIndex: {result.Result2.TokenStartIndex}, TokenType: {result.Result2.TokenType}, Exception: {result.Result2.Exception?.Message}";

        Console.WriteLine(messageOutput);
        Console.WriteLine();
    }

    static CompareResult GetResultCompare(JsonInput jsonInput)
    {
        var result1 = GetResultJsonReaderSingleSegment(jsonInput);
        var result2 = GetResultJsonReaderMultiSegment(jsonInput);

        string differenceString = null;

        var hasExceptionDifference =
            (result1.Exception != null) != (result2.Exception != null) ||
            (result1.Exception != null && result2.Exception != null && (result1.Exception.GetType() != result2.Exception.GetType() || result1.Exception.Message != result2.Exception.Message));

        if (
            hasExceptionDifference ||
            result1.BytesConsumed != result2.BytesConsumed ||
            result1.CurrentDepth != result2.CurrentDepth ||
            result1.TokenStartIndex != result2.TokenStartIndex ||
            result1.TokenType != result2.TokenType ||
            false)
        {
            differenceString = $"Exception: {hasExceptionDifference}, BytesConsumed: {result1.BytesConsumed != result2.BytesConsumed}, CurrentDepth: {result1.CurrentDepth != result2.CurrentDepth}, TokenStartIndex: {result1.TokenStartIndex != result2.TokenStartIndex}, TokenType: {result1.TokenType != result2.TokenType}";
        }

        return new CompareResult(result1, result2, differenceString);
    }

    class CompareResult
    {
        public JsonResult Result1 { get; }
        public JsonResult Result2 { get; }
        public string DifferenceString { get; }

        public CompareResult(JsonResult result1, JsonResult result2, string differenceString)
        {
            Result1 = result1;
            Result2 = result2;
            DifferenceString = differenceString;
        }

        public override string ToString()
        {
            return $"{nameof(Result1)}: {Result1}, {nameof(Result2)}: {Result2}, {nameof(DifferenceString)}: {DifferenceString}";
        }
    }

    static JsonResult GetResultJsonReaderMultiSegment(JsonInput jsonInput)
    {
        static IEnumerable<Memory<byte>> SplitMemory(Memory<byte> memory, int chunkSize)
        {
            for (int startIndex = 0; startIndex < memory.Length; startIndex += chunkSize)
                yield return memory.Slice(startIndex, Math.Min(chunkSize, memory.Length - startIndex));
        }

        var memories = SplitMemory(jsonInput.JsonBytes, 1);

        var jsonReader = new Utf8JsonReader(CreateReadOnlySequence(memories), jsonInput.IsFinalBlock, new JsonReaderState(jsonInput.GetJsonReaderOptions()));

        var jsonResult = ConsumeJsonReader(jsonReader, jsonInput.JsonBytes.Length);

        return jsonResult;
    }

    static JsonResult GetResultJsonReaderSingleSegment(JsonInput jsonInput)
    {
        var utf8JsonGuardPage = jsonInput.JsonBytes;

        var jsonReader = new Utf8JsonReader(utf8JsonGuardPage, jsonInput.IsFinalBlock, new JsonReaderState(jsonInput.GetJsonReaderOptions()));

        return ConsumeJsonReader(jsonReader, utf8JsonGuardPage.Length);
    }

    static JsonResult ConsumeJsonReader(Utf8JsonReader jsonReader, int inputLength)
    {
        Exception exception = null;
        try
        {
            long lastBytesConsumed = 0;
            long lastTokenStartIndex = -1;
            JsonTokenType lastTokenType = JsonTokenType.None;

            while (jsonReader.Read())
            {
                if (jsonReader.BytesConsumed <= lastBytesConsumed)
                    throw new Exception("State: BytesConsumed.");

                if (jsonReader.TokenStartIndex <= lastTokenStartIndex)
                    throw new Exception("State: TokenStartIndex.");

                if (jsonReader.TokenType == lastTokenType &&
                    (lastTokenType == JsonTokenType.False ||
                     lastTokenType == JsonTokenType.True ||
                     lastTokenType == JsonTokenType.Null ||
                     lastTokenType == JsonTokenType.False ||
                     lastTokenType == JsonTokenType.String ||
                     lastTokenType == JsonTokenType.Number ||
                     lastTokenType == JsonTokenType.PropertyName ||
                     false))
                    throw new Exception("State: TokenType.");

                lastBytesConsumed = jsonReader.BytesConsumed;
                lastTokenStartIndex = jsonReader.TokenStartIndex;
                lastTokenType = jsonReader.TokenType;
            }

            if (jsonReader.IsFinalBlock && jsonReader.BytesConsumed != inputLength)
                throw new Exception("State: Incomplete.");

            if (jsonReader.IsFinalBlock &&
                (jsonReader.TokenType == JsonTokenType.StartArray ||
                 jsonReader.TokenType == JsonTokenType.StartObject ||
                 jsonReader.TokenType == JsonTokenType.None ||
                 false))
                throw new Exception("State: Final TokenType.");
        }
        catch (Exception ex)
        {
            exception = ex;
        }

        return new JsonResult(exception, jsonReader.BytesConsumed, jsonReader.CurrentDepth, jsonReader.TokenStartIndex, jsonReader.TokenType);
    }

    readonly struct JsonInput
    {
        public byte[] JsonBytes { get; }
        public bool IsFinalBlock { get; }
        public bool AllowTrailingCommas { get; }
        public JsonCommentHandling CommentHandling { get; }

        public JsonReaderOptions GetJsonReaderOptions() => new JsonReaderOptions() { AllowTrailingCommas = AllowTrailingCommas, CommentHandling = CommentHandling };

        public JsonInput(byte[] jsonBytes, bool isFinalBlock, bool allowTrailingCommas, JsonCommentHandling commentHandling)
        {
            JsonBytes = jsonBytes;
            IsFinalBlock = isFinalBlock;
            AllowTrailingCommas = allowTrailingCommas;
            CommentHandling = commentHandling;
        }
    }

    class JsonResult
    {
        public Exception Exception { get; }
        public long BytesConsumed { get; }
        public int CurrentDepth { get; }
        public long TokenStartIndex { get; }
        public JsonTokenType TokenType { get; }

        public JsonResult(Exception exception, long bytesConsumed, int currentDepth, long tokenStartIndex, JsonTokenType tokenType)
        {
            Exception = exception;
            BytesConsumed = bytesConsumed;
            CurrentDepth = currentDepth;
            TokenStartIndex = tokenStartIndex;
            TokenType = tokenType;
        }

        public override string ToString()
        {
            return $"{nameof(Exception)}: {Exception}, {nameof(BytesConsumed)}: {BytesConsumed}, {nameof(CurrentDepth)}: {CurrentDepth}, {nameof(TokenStartIndex)}: {TokenStartIndex}, {nameof(TokenType)}: {TokenType}";
        }
    }

    public static ReadOnlySequence<T> CreateReadOnlySequence<T>(IEnumerable<Memory<T>> buffers) => SimpleReadOnlySequenceSegment<T>.Create(buffers);

    class SimpleReadOnlySequenceSegment<T> : ReadOnlySequenceSegment<T>
    {
        internal static ReadOnlySequence<T> Create(IEnumerable<Memory<T>> buffers)
        {
            SimpleReadOnlySequenceSegment<T> segment = null;
            SimpleReadOnlySequenceSegment<T> first = null;
            foreach (Memory<T> buffer in buffers)
            {
                var newSegment = new SimpleReadOnlySequenceSegment<T>()
                {
                    Memory = buffer,
                };

                if (segment != null)
                {
                    segment.Next = newSegment;
                    newSegment.RunningIndex = segment.RunningIndex + segment.Memory.Length;
                }
                else
                {
                    first = newSegment;
                }

                segment = newSegment;
            }

            if (first == null)
            {
                first = segment = new SimpleReadOnlySequenceSegment<T>();
            }

            return new ReadOnlySequence<T>(first, 0, segment, segment.Memory.Length);
        }
    }
}