Unify MaxResponseHeadersLength calculations

src/libraries/Common/tests/System/Net/Http/Http2LoopbackConnection.cs

@@ -804,9 +804,9 @@ public async Task SendDefaultResponseAsync(int streamId)
 
         public async Task SendResponseHeadersAsync(int streamId, bool endStream = true, HttpStatusCode statusCode = HttpStatusCode.OK, bool isTrailingHeader = false, bool endHeaders = true, IList<HttpHeaderData> headers = null)
         {
-            // For now, only support headers that fit in a single frame
             byte[] headerBlock = new byte[Frame.MaxFrameLength];
             int bytesGenerated = 0;
+            bool sentAnyFrames = false;
 
             if (!isTrailingHeader)
             {
@@ -818,18 +818,39 @@ public async Task SendResponseHeadersAsync(int streamId, bool endStream = true,
             {
                 foreach (HttpHeaderData headerData in headers)
                 {
+                    int estimatedHeaderLength = headerData.Name.Length + headerData.Value.Length + 20;
+                    Assert.True(estimatedHeaderLength < headerBlock.Length, "A single header exceeds MaxFrameLength");
+
+                    if (bytesGenerated + estimatedHeaderLength > headerBlock.Length)
+                    {
+                        await FlushHeadersAsync(endHeaders: false);
+                    }
+
                     bytesGenerated += HPackEncoder.EncodeHeader(headerData.Name, headerData.Value, headerData.ValueEncoding, headerData.HuffmanEncoded ? HPackFlags.HuffmanEncode : HPackFlags.None, headerBlock.AsSpan(bytesGenerated));
                 }
             }
 
-            FrameFlags flags = endHeaders ? FrameFlags.EndHeaders : FrameFlags.None;
-            if (endStream)
+            await FlushHeadersAsync(endHeaders);
+
+            async Task FlushHeadersAsync(bool endHeaders)
             {
-                flags |= FrameFlags.EndStream;
-            }
+                Memory<byte> headerBytes = headerBlock.AsMemory(0, bytesGenerated);
+                bytesGenerated = 0;
 
-            HeadersFrame headersFrame = new HeadersFrame(headerBlock.AsMemory(0, bytesGenerated), flags, 0, 0, 0, streamId);
-            await WriteFrameAsync(headersFrame).ConfigureAwait(false);
+                FrameFlags flags = endHeaders ? FrameFlags.EndHeaders : FrameFlags.None;
+                if (endStream)
+                {
+                    flags |= FrameFlags.EndStream;
+                }
+
+                Frame frame = sentAnyFrames
+                    ? new ContinuationFrame(headerBytes, flags, 0, 0, 0, streamId)
+                    : new HeadersFrame(headerBytes, flags, 0, 0, 0, streamId);
+
+                sentAnyFrames = true;
+
+                await WriteFrameAsync(frame).ConfigureAwait(false);
+            }
         }
 
         public async Task SendResponseDataAsync(int streamId, ReadOnlyMemory<byte> responseData, bool endStream)

src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.MaxResponseHeadersLength.cs

@@ -49,11 +49,6 @@ public void ValidValue_SetGet_Roundtrips(int validValue)
         [Fact]
         public async Task SetAfterUse_Throws()
         {
-            if (IsWinHttpHandler && UseVersion >= HttpVersion20.Value)
-            {
-                return;
-            }
-
             await LoopbackServerFactory.CreateClientAndServerAsync(async uri =>
             {
                 using HttpClientHandler handler = CreateHttpClientHandler();
@@ -66,124 +61,72 @@ await LoopbackServerFactory.CreateClientAndServerAsync(async uri =>
             server => server.AcceptConnectionSendResponseAndCloseAsync());
         }
 
-        [OuterLoop]
-        [Fact]
-        public async Task InfiniteSingleHeader_ThrowsException()
+        [Theory]
+        [InlineData(1)]
+        [InlineData(15)]
+        public async Task LargeSingleHeader_ThrowsException(int maxResponseHeadersLength)
         {
-            await LoopbackServer.CreateServerAsync(async (server, url) =>
+            using HttpClientHandler handler = CreateHttpClientHandler();
+            handler.MaxResponseHeadersLength = maxResponseHeadersLength;
+
+            await LoopbackServerFactory.CreateClientAndServerAsync(async uri =>
             {
-                using (HttpClientHandler handler = CreateHttpClientHandler())
-                using (HttpClient client = CreateHttpClient(handler))
+                using HttpClient client = CreateHttpClient(handler);
+
+                Exception e = await Assert.ThrowsAsync<HttpRequestException>(() => client.GetAsync(uri));
+                if (!IsWinHttpHandler)
                 {
-                    Task<HttpResponseMessage> getAsync = client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
-                    await server.AcceptConnectionAsync(async connection =>
-                    {
-                        var cts = new CancellationTokenSource();
-                        Task serverTask = Task.Run(async delegate
-                        {
-                            await connection.ReadRequestHeaderAndSendCustomResponseAsync("HTTP/1.1 200 OK\r\nContent-Length: 0\r\nMyInfiniteHeader: ");
-                            try
-                            {
-                                while (!cts.IsCancellationRequested)
-                                {
-                                    await connection.WriteStringAsync(new string('s', 16000));
-                                    await Task.Delay(1);
-                                }
-                            }
-                            catch (Exception ex)
-                            {
-                                _output.WriteLine($"Ignored exception:{Environment.NewLine}{ex}");
-                            }
-                        });
-
-                        Exception e = await Assert.ThrowsAsync<HttpRequestException>(() => getAsync);
-                        cts.Cancel();
-                        if (!IsWinHttpHandler)
-                        {
-                            Assert.Contains((handler.MaxResponseHeadersLength * 1024).ToString(), e.ToString());
-                        }
-                        await serverTask;
-                    });
+                    Assert.Contains((handler.MaxResponseHeadersLength * 1024).ToString(), e.ToString());
                 }
+            },
+            async server =>
+            {
+                await server.HandleRequestAsync(headers: new[] { new HttpHeaderData("Foo", new string('a', handler.MaxResponseHeadersLength * 1024)) });
             });
         }
 
-        [OuterLoop]
-        [Theory, MemberData(nameof(ResponseWithManyHeadersData))]
-        public async Task ThresholdExceeded_ThrowsException(string responseHeaders, int? maxResponseHeadersLength, bool shouldSucceed)
+        [Theory]
+        [InlineData(null, 63 * 1024)]
+        [InlineData(null, 65 * 1024)]
+        [InlineData(1, 100)]
+        [InlineData(1, 1024)]
+        [InlineData(int.MaxValue / 800, 100 * 1024)] // Capped at int.MaxValue
+        public async Task ThresholdExceeded_ThrowsException(int? maxResponseHeadersLength, int headersLengthEstimate)
         {
-            await LoopbackServer.CreateServerAsync(async (server, url) =>
+            await LoopbackServerFactory.CreateClientAndServerAsync(async uri =>
             {
-                using (HttpClientHandler handler = CreateHttpClientHandler())
-                using (HttpClient client = CreateHttpClient(handler))
+                using HttpClientHandler handler = CreateHttpClientHandler();
+
+                if (maxResponseHeadersLength.HasValue)
                 {
-                    if (maxResponseHeadersLength.HasValue)
-                    {
-                        handler.MaxResponseHeadersLength = maxResponseHeadersLength.Value;
-                    }
-                    Task<HttpResponseMessage> getAsync = client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
+                    handler.MaxResponseHeadersLength = maxResponseHeadersLength.Value;
+                }
 
-                    await server.AcceptConnectionAsync(async connection =>
+                using HttpClient client = CreateHttpClient(handler);
+
+                if (headersLengthEstimate < handler.MaxResponseHeadersLength * 1024L)
+                {
+                    await client.GetAsync(uri);
+                }
+                else
+                {
+                    Exception e = await Assert.ThrowsAsync<HttpRequestException>(() => client.GetAsync(uri));
+                    if (!IsWinHttpHandler)
                     {
-                        Task serverTask = connection.ReadRequestHeaderAndSendCustomResponseAsync(responseHeaders);
-
-                        if (shouldSucceed)
-                        {
-                            (await getAsync).Dispose();
-                            await serverTask;
-                        }
-                        else
-                        {
-                            Exception e = await Assert.ThrowsAsync<HttpRequestException>(() => getAsync);
-                            if (!IsWinHttpHandler)
-                            {
-                                Assert.Contains((handler.MaxResponseHeadersLength * 1024).ToString(), e.ToString());
-                            }
-                            try
-                            {
-                                await serverTask;
-                            }
-                            catch (Exception ex)
-                            {
-                                _output.WriteLine($"Ignored exception:{Environment.NewLine}{ex}");
-                            }
-                        }
-                    });
+                        Assert.Contains((handler.MaxResponseHeadersLength * 1024).ToString(), e.ToString());
+                    }
                 }
-            });
-        }
-
-        public static IEnumerable<object[]> ResponseWithManyHeadersData
-        {
-            get
+            },
+            async server =>
             {
-                foreach (int? max in new int?[] { null, 1, 31, 128 })
+                var headers = new List<HttpHeaderData>();
+                for (int i = 0; i <= headersLengthEstimate / 500; i++)
                 {
-                    int actualSize = max.HasValue ? max.Value : 64;
-
-                    yield return new object[] { GenerateLargeResponseHeaders(actualSize * 1024 - 1), max, true }; // Small enough
-                    yield return new object[] { GenerateLargeResponseHeaders(actualSize * 1024), max, true }; // Just right
-                    yield return new object[] { GenerateLargeResponseHeaders(actualSize * 1024 + 1), max, false }; // Too big
+                    headers.Add(new HttpHeaderData($"Custom-{i}", new string('a', 480)));
                 }
-            }
-        }
 
-        private static string GenerateLargeResponseHeaders(int responseHeadersSizeInBytes)
-        {
-            var buffer = new StringBuilder();
-            buffer.Append("HTTP/1.1 200 OK\r\n");
-            buffer.Append("Content-Length: 0\r\n");
-            for (int i = 0; i < 24; i++)
-            {
-                buffer.Append($"Custom-{i:D4}: 1234567890123456789012345\r\n");
-            }
-            buffer.Append($"Custom-24: ");
-            buffer.Append(new string('c', responseHeadersSizeInBytes - (buffer.Length + 4)));
-            buffer.Append("\r\n\r\n");
-
-            string response = buffer.ToString();
-            Assert.Equal(responseHeadersSizeInBytes, response.Length);
-            return response;
+                await server.HandleRequestAsync(headers: headers);
+            });
         }
     }
 }

src/libraries/System.Net.Http.WinHttpHandler/tests/FunctionalTests/PlatformHandlerTest.cs

@@ -333,6 +333,7 @@ public sealed class PlatformHandlerTest_Cookies_Http11_Http2 : HttpClientHandler
         public PlatformHandlerTest_Cookies_Http11_Http2(ITestOutputHelper output) : base(output) { }
     }
 
+    [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsWindows10Version1607OrGreater))]
     public sealed class PlatformHandler_HttpClientHandler_MaxResponseHeadersLength_Http2_Test : HttpClientHandler_MaxResponseHeadersLength_Test
     {
         protected override Version UseVersion => HttpVersion20.Value;

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs

@@ -138,7 +138,7 @@ public Http2Connection(HttpConnectionPool pool, Stream stream)
             _incomingBuffer = new ArrayBuffer(InitialConnectionBufferSize);
             _outgoingBuffer = new ArrayBuffer(InitialConnectionBufferSize);
 
-            _hpackDecoder = new HPackDecoder(maxHeadersLength: pool.Settings._maxResponseHeadersLength * 1024);
+            _hpackDecoder = new HPackDecoder(maxHeadersLength: pool.Settings.MaxResponseHeadersByteLength);
 
             _httpStreams = new Dictionary<int, Http2Stream>();
 

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Stream.cs

@@ -103,7 +103,7 @@ public Http2Stream(HttpRequestMessage request, Http2Connection connection)
 
                 _windowManager = new Http2StreamWindowManager(connection, this);
 
-                _headerBudgetRemaining = connection._pool.Settings._maxResponseHeadersLength * 1024;
+                _headerBudgetRemaining = connection._pool.Settings.MaxResponseHeadersByteLength;
 
                 if (_request.Content == null)
                 {
@@ -589,7 +589,7 @@ private void AdjustHeaderBudget(int amount)
                 _headerBudgetRemaining -= amount;
                 if (_headerBudgetRemaining < 0)
                 {
-                    throw new HttpRequestException(SR.Format(SR.net_http_response_headers_exceeded_length, _connection._pool.Settings._maxResponseHeadersLength * 1024L));
+                    throw new HttpRequestException(SR.Format(SR.net_http_response_headers_exceeded_length, _connection._pool.Settings.MaxResponseHeadersByteLength));
                 }
             }
 

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http3Connection.cs

@@ -391,7 +391,7 @@ public static byte[] BuildSettingsFrame(HttpConnectionSettings settings)
         {
             Span<byte> buffer = stackalloc byte[4 + VariableLengthIntegerHelper.MaximumEncodedLength];
 
-            int integerLength = VariableLengthIntegerHelper.WriteInteger(buffer.Slice(4), settings._maxResponseHeadersLength * 1024L);
+            int integerLength = VariableLengthIntegerHelper.WriteInteger(buffer.Slice(4), settings.MaxResponseHeadersByteLength);
             int payloadLength = 1 + integerLength; // includes the setting ID and the integer value.
             Debug.Assert(payloadLength <= VariableLengthIntegerHelper.OneByteLimit);
 

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http3RequestStream.cs

@@ -38,7 +38,7 @@ internal sealed class Http3RequestStream : IHttpStreamHeadersHandler, IAsyncDisp
         // Header decoding.
         private QPackDecoder _headerDecoder;
         private HeaderState _headerState;
-        private long _headerBudgetRemaining;
+        private int _headerBudgetRemaining;
 
         /// <summary>Reusable array used to get the values for each header being written to the wire.</summary>
         private string[] _headerValues = Array.Empty<string>();
@@ -73,7 +73,7 @@ public Http3RequestStream(HttpRequestMessage request, Http3Connection connection
             _sendBuffer = new ArrayBuffer(initialSize: 64, usePool: true);
             _recvBuffer = new ArrayBuffer(initialSize: 64, usePool: true);
 
-            _headerBudgetRemaining = connection.Pool.Settings._maxResponseHeadersLength * 1024L; // _maxResponseHeadersLength is in KiB.
+            _headerBudgetRemaining = connection.Pool.Settings.MaxResponseHeadersByteLength;
             _headerDecoder = new QPackDecoder(maxHeadersLength: (int)Math.Min(int.MaxValue, _headerBudgetRemaining));
 
             _requestBodyCancellationSource = new CancellationTokenSource();
@@ -843,10 +843,10 @@ private async ValueTask ReadHeadersAsync(long headersLength, CancellationToken c
             if (headersLength > _headerBudgetRemaining)
             {
                 _stream.Abort(QuicAbortDirection.Write, (long)Http3ErrorCode.ExcessiveLoad);
-                throw new HttpRequestException(SR.Format(SR.net_http_response_headers_exceeded_length, _connection.Pool.Settings._maxResponseHeadersLength * 1024L));
+                throw new HttpRequestException(SR.Format(SR.net_http_response_headers_exceeded_length, _connection.Pool.Settings.MaxResponseHeadersByteLength));
             }
 
-            _headerBudgetRemaining -= headersLength;
+            _headerBudgetRemaining -= (int)headersLength;
 
             while (headersLength != 0)
             {

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs

@@ -555,7 +555,7 @@ public async Task<HttpResponseMessage> SendAsyncCore(HttpRequestMessage request,
                 }
 
                 // Start to read response.
-                _allowedReadLineBytes = (int)Math.Min(int.MaxValue, _pool.Settings._maxResponseHeadersLength * 1024L);
+                _allowedReadLineBytes = _pool.Settings.MaxResponseHeadersByteLength;
 
                 // We should not have any buffered data here; if there was, it should have been treated as an error
                 // by the previous request handling.  (Note we do not support HTTP pipelining.)
@@ -1550,7 +1550,7 @@ private bool TryReadNextChunkedLine(bool readingHeader, out ReadOnlySpan<byte> l
             }
 
             string message = readingHeader
-                ? SR.Format(SR.net_http_response_headers_exceeded_length, _pool.Settings._maxResponseHeadersLength * 1024L)
+                ? SR.Format(SR.net_http_response_headers_exceeded_length, _pool.Settings.MaxResponseHeadersByteLength)
                 : SR.net_http_chunk_too_large;
 
             throw new HttpRequestException(message);
@@ -1651,7 +1651,7 @@ private void ThrowIfExceededAllowedReadLineBytes()
         {
             if (_allowedReadLineBytes < 0)
             {
-                throw new HttpRequestException(SR.Format(SR.net_http_response_headers_exceeded_length, _pool.Settings._maxResponseHeadersLength * 1024L));
+                throw new HttpRequestException(SR.Format(SR.net_http_response_headers_exceeded_length, _pool.Settings.MaxResponseHeadersByteLength));
             }
         }
 

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionSettings.cs

@@ -122,6 +122,8 @@ public HttpConnectionSettings CloneAndNormalize()
             return settings;
         }
 
+        public int MaxResponseHeadersByteLength => (int)Math.Min(int.MaxValue, _maxResponseHeadersLength * 1024L);
+
         public bool EnableMultipleHttp2Connections => _enableMultipleHttp2Connections;
 
         private byte[]? _http3SettingsFrame;

src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http3.cs

@@ -41,10 +41,10 @@ private async Task AssertProtocolErrorAsync(long errorCode, Func<Task> task)
         }
 
         [Theory]
-        [InlineData(10)] // 2 bytes settings value.
-        [InlineData(100)] // 4 bytes settings value.
-        [InlineData(10_000_000)] // 8 bytes settings value.
-        public async Task ClientSettingsReceived_Success(int headerSizeLimit)
+        [InlineData(10, 10240)] // 2 bytes settings value.
+        [InlineData(100, 102400)] // 4 bytes settings value.
+        [InlineData(10_000_000, int.MaxValue)] // 8 bytes settings value.
+        public async Task ClientSettingsReceived_Success(int headerSizeLimit, int expectedHeaderSizeLimitBytes)
         {
             using Http3LoopbackServer server = CreateHttp3LoopbackServer();
 
@@ -58,7 +58,7 @@ public async Task ClientSettingsReceived_Success(int headerSizeLimit)
                 await using (requestStream)
                 {
                     Assert.False(settingsStream.CanWrite, "Expected unidirectional control stream.");
-                    Assert.Equal(headerSizeLimit * 1024L, connection.MaxHeaderListSize);
+                    Assert.Equal(expectedHeaderSizeLimitBytes, connection.MaxHeaderListSize);
 
                     await requestStream.ReadRequestDataAsync();
                     await requestStream.SendResponseAsync();