[TRTLLM-14510][feat] BREAKING: serve: default num_serve_frontends to 8 with single-frontend fallback

docs/source/commands/trtllm-serve/trtllm-serve.rst

@@ -79,6 +79,19 @@ The module ``tensorrt_llm/serve/tool_parser/qwen3_coder_parser.py`` implements t
 shared format. ``--reasoning_parser`` separately controls reasoning extraction;
 ``nemotron-v3`` and ``nano-v3`` are aliases for the same Nemotron reasoning parser.
 
+Multiple HTTP Frontends
+-----------------------
+
+At high concurrency a single serving process is host-bound: one asyncio event loop parses every request and writes every streamed chunk, and queuing on that loop rather than GPU time starts to dominate first-token latency. ``trtllm-serve`` therefore runs **8 HTTP frontend processes** against one executor by default (``--num_serve_frontends``, beta). The launcher process owns the engine; the other frontends attach to it and share the serving port via ``SO_REUSEPORT``, so clients still see a single URL.
+
+.. note::
+
+   * Pass ``--num_serve_frontends 1`` (or set ``num_serve_frontends: 1`` in the ``--config`` YAML) to run a single frontend.
+   * The default applies to the OpenAI HTTP server on the default executor path on a fixed port. With ``--port 0`` / ``--report_addr``, ``orchestrator_type`` (``rpc`` / ``ray``), ``enable_resource_governor`` or ``--enable_sleep_mode`` it falls back to one frontend and logs why; an explicit ``--num_serve_frontends`` greater than 1 fails instead. ``--grpc`` also keeps one frontend unless ``--num_serve_frontends`` is given (only ``--grpc-protocol openengine`` supports more).
+   * Each request is handled entirely by the frontend that accepted its connection. Server-wide state lives in the launcher: the other frontends forward ``/metrics``, ``/kv_cache_events``, profiling and the Anthropic batches API to it, and ``/prometheus/metrics`` covers every frontend. Per-process state is not shared: the stateful Responses API store (``store`` / ``previous_response_id``) is disabled, ``/health`` describes the frontend that answered, and each frontend writes its own per-request perf metrics file.
+   * ``SO_REUSEPORT`` would also let a second server of the same user join the port and take part of its traffic, so the launcher refuses a port that another server already uses.
+   * Every extra frontend is a full Python process that imports TensorRT LLM and loads the tokenizer, so budget host memory accordingly.
+
 Inference Endpoints
 -------------------
 

tensorrt_llm/commands/serve.py

@@ -488,6 +488,34 @@ def _diagnose_port_in_use(port: int) -> str:
     return "; ".join(details)
 
 
+def _reserve_multi_frontend_port(host: str, port: int,
+                                 family: socket.AddressFamily) -> socket.socket:
+    """Refuse a port another server uses; return a lock to hold while serving.
+
+    SO_REUSEPORT would let a second server of the same user join the port. A
+    plain bind catches any listener (SO_REUSEADDR ignores TIME_WAIT), and an
+    abstract Unix socket named after the port catches a launcher that has not
+    started listening yet.
+    """
+    lock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    try:
+        lock.bind(f"\0trtllm-serve-{os.geteuid()}-{port}")
+    except OSError as e:
+        lock.close()
+        raise RuntimeError(f"Port {port} is already reserved by another "
+                           f"multi-frontend trtllm-serve: {e}") from e
+    try:
+        with socket.socket(family, socket.SOCK_STREAM) as probe:
+            probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            probe.bind((host, port))
+    except OSError as e:
+        lock.close()
+        raise RuntimeError(
+            f"Failed to bind socket to {host}:{port}: {e}. "
+            f"Port holder(s): {_diagnose_port_in_use(port)}") from e
+    return lock
+
+
 @contextlib.contextmanager
 def _provision_kv_cache_pool(llm_args: dict,
                              owns_engine: bool = True) -> Iterator[None]:
@@ -558,6 +586,38 @@ def _share_prometheus_multiproc_dir(llm, ipc_dir: str) -> None:
             f"Shared PROMETHEUS_MULTIPROC_DIR for all frontends: {path}")
 
 
+def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool,
+                                         port: int, grpc: bool,
+                                         report_addr: Optional[str],
+                                         enable_sleep_mode: bool) -> None:
+    """Fall back to one frontend where the default count cannot run.
+
+    An explicit count (CLI, --config YAML or --set) is left alone, so the
+    guard that owns the conflict still rejects it.
+    """
+    num_frontends = llm_args.get("num_serve_frontends", 1)
+    if requested or num_frontends <= 1:
+        return
+    if grpc:
+        # OpenEngine supports several frontends, but only when asked to.
+        reason = "--grpc"
+    elif port == 0 or report_addr:
+        reason = "--port 0 / --report_addr"
+    elif llm_args.get("orchestrator_type") is not None:
+        reason = f"orchestrator_type={llm_args['orchestrator_type']!r}"
+    elif llm_args.get("enable_resource_governor"):
+        reason = "enable_resource_governor"
+    elif enable_sleep_mode:
+        reason = "--enable_sleep_mode"
+    else:
+        return
+    logger.info(
+        f"num_serve_frontends defaults to {num_frontends} for the OpenAI HTTP "
+        f"server only; running 1 frontend because of {reason}. Pass "
+        "--num_serve_frontends explicitly to override.")
+    llm_args["num_serve_frontends"] = 1
+
+
 def launch_server(
         host: str,
         port: int,
@@ -622,14 +682,17 @@ def launch_server(
             logger.warning(
                 "num_serve_frontends > 1: stateful Responses API storage "
                 "(store/previous_response_id) is disabled; the per-frontend "
-                "in-memory store cannot be shared across frontends.")
+                "in-memory store cannot be shared across frontends. Pass "
+                "--num_serve_frontends 1 to keep it.")
         os.environ["TRTLLM_RESPONSES_API_DISABLE_STORE"] = "1"
 
     addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
                                    socket.SOCK_STREAM)
     address_family = socket.AF_INET6 if all(
         [info[0] == socket.AF_INET6 for info in addr_info]) else socket.AF_INET
-    with socket.socket(address_family, socket.SOCK_STREAM) as s:
+    port_lock = (_reserve_multi_frontend_port(host, port, address_family)
+                 if multi_frontend.is_launcher else contextlib.nullcontext())
+    with port_lock, socket.socket(address_family, socket.SOCK_STREAM) as s:
         # port == 0 lets the kernel pick the port; the caller then needs a way
         # to learn it, either by service discovery or by report_addr.
         assert port > 0 or disagg_cluster_config is not None or report_addr, (
@@ -1111,11 +1174,14 @@ def launch_visual_gen_server(
 @stability_option(
     "--num_serve_frontends",
     type=click.IntRange(min=1, max=MAX_NUM_FRONTENDS),
-    default=1,
+    # LlmArgs keeps 1: only trtllm-serve spawns frontends.
+    default=8,
     help="Number of HTTP or OpenEngine frontend processes serving one "
     "executor; values > 1 share the serving port via "
-    "SO_REUSEPORT (classic IPC executor path only).",
-    status="prototype")
+    "SO_REUSEPORT (classic IPC executor path only). Without an explicit "
+    "value, --grpc and configurations that support a single frontend only "
+    "use 1 and log why.",
+    status="beta")
 @stability_option("--num_input_processor_workers",
                   type=click.IntRange(min=1),
                   default=8,
@@ -1556,6 +1622,15 @@ def _serve_llm():
         if enable_sleep_mode and llm_args.get("sleep_config") is None:
             llm_args["sleep_config"] = SleepConfig()
 
+        _resolve_default_num_serve_frontends(
+            llm_args,
+            requested=("num_serve_frontends" in explicit_cli_keys
+                       or "num_serve_frontends" in llm_args_extra_dict),
+            port=port,
+            grpc=grpc,
+            report_addr=report_addr,
+            enable_sleep_mode=enable_sleep_mode)
+
         _apply_effective_telemetry_config(llm_args, telemetry=telemetry)
         # Preserve requested settings for failures before entering the LLM constructor.
         try:

tests/integration/defs/test_e2e.py

@@ -595,6 +595,13 @@ def test_openai_prometheus(llm_root, llm_venv):
          str(test_root / "_test_openai_prometheus.py")])
 
 
+def test_openai_multi_frontend(llm_root, llm_venv):
+    test_root = unittest_path() / "llmapi" / "apps"
+    llm_venv.run_cmd(
+        ["-m", "pytest",
+         str(test_root / "_test_openai_multi_frontend.py")])
+
+
 def test_openai_lora(llm_root, llm_venv):
     test_root = unittest_path() / "llmapi" / "apps"
     llm_venv.run_cmd(["-m", "pytest", str(test_root / "_test_openai_lora.py")])

tests/integration/test_lists/test-db/l0_b200.yml

@@ -107,6 +107,7 @@ l0_b200:
   - test_e2e.py::test_ptp_quickstart_advanced_mtp[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16]
   - test_e2e.py::test_ptp_quickstart_advanced_mtp_eagle[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16]
   - test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b]
+  - test_e2e.py::test_openai_multi_frontend
   # Covers the GPU cases of tests/unittest/_torch/attention/ on B200, except the paths
   # ignored below; its cpu_only cases run on CPU (l0_cpu). kernels/ has no entry of its own
   # on any list, so apart from kernels/parallel_hw_agnostic/ this entry is what runs it

tests/integration/test_lists/test-db/l0_cpu.yml

@@ -295,6 +295,7 @@ l0_cpu:
   - unittest/llmapi/test_serialization.py
   - unittest/llmapi/test_serve_multi_frontend_helpers.py
   - unittest/llmapi/test_serve_multi_frontend_route_gate.py
+  - unittest/llmapi/test_serve_num_frontends_default.py
   - unittest/llmapi/test_serve_report_addr.py
   - unittest/llmapi/test_serve_sleep_mode.py
   - unittest/llmapi/test_session_prefetcher.py

tests/unittest/api_stability/references/trtllm_serve_cli.yaml

@@ -344,8 +344,8 @@ commands:
         - "--num_postprocess_workers"
       num_serve_frontends:
         type: int
-        default: 1
-        status: prototype
+        default: 8
+        status: beta
         required: false
         multiple: false
         is_flag: false

tests/unittest/llmapi/apps/_test_openai_multi_frontend.py

@@ -0,0 +1,107 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""trtllm-serve with 8 frontend processes on one engine.
+
+Each connection may land on any frontend (SO_REUSEPORT), so every request opens
+its own. Server-wide state must not depend on which frontend answers.
+"""
+
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+import psutil
+import pytest
+import requests
+
+from ..test_llm import get_model_path
+from .openai_server import RemoteOpenAIServer
+
+NUM_FRONTENDS = 8
+NUM_REQUESTS = 64
+NUM_STREAMS = 16
+
+
+@pytest.fixture(scope="module")
+def model_path() -> str:
+    return get_model_path("Qwen3/Qwen3-0.6B")
+
+
+@pytest.fixture(scope="module")
+def server(model_path: str):
+    args = ["--backend", "pytorch", "--num_serve_frontends", str(NUM_FRONTENDS)]
+    extra_config = {"return_perf_metrics": True, "enable_iter_perf_stats": True}
+    with RemoteOpenAIServer(model_path, args, extra_config=extra_config) as server:
+        # /health answers once the first frontend listens.
+        deadline = time.monotonic() + 600
+        while _listeners(server.port) < NUM_FRONTENDS:
+            assert time.monotonic() < deadline, _listeners(server.port)
+            time.sleep(1)
+        yield server
+
+
+def _listeners(port: int) -> int:
+    return sum(
+        1
+        for c in psutil.net_connections("tcp")
+        if c.status == psutil.CONN_LISTEN and c.laddr and c.laddr.port == port
+    )
+
+
+def _finish_reason(server: RemoteOpenAIServer, model: str, i: int, stream: bool):
+    """Run one completion on a connection of its own."""
+    with server.get_client() as client:
+        output = client.completions.create(
+            model=model,
+            prompt=f"Request {i}: the capital of France is",
+            max_tokens=16,
+            stream=stream,
+        )
+        if stream:
+            choices = [chunk.choices[0] for chunk in output if chunk.choices]
+        else:
+            choices = output.choices
+    text = "".join(choice.text for choice in choices)
+    reasons = [choice.finish_reason for choice in choices if choice.finish_reason]
+    return reasons[-1] if text and reasons else None
+
+
+def _success_total(server: RemoteOpenAIServer) -> float:
+    text = requests.get(server.url_for("prometheus", "metrics"), timeout=30).text
+    return sum(
+        float(line.rsplit(" ", 1)[1])
+        for line in text.splitlines()
+        if line.startswith("trtllm_request_success_total")
+    )
+
+
+def test_frontends_share_one_engine(server: RemoteOpenAIServer, model_path: str):
+    with ThreadPoolExecutor(16) as pool:
+        finish_reasons = list(
+            pool.map(
+                lambda i: _finish_reason(server, model_path, i, stream=i >= NUM_REQUESTS),
+                range(NUM_REQUESTS + NUM_STREAMS),
+            )
+        )
+    assert all(finish_reasons), finish_reasons
+
+    # Only the launcher drains the engine's stats (every second), so the first
+    # poll gets all of them, whichever frontend answers.
+    time.sleep(3)
+    polls = [len(requests.get(server.url_for("metrics"), timeout=30).json()) for _ in range(16)]
+    assert polls[0] > 0 and not any(polls[1:]), polls
+
+    # Every scrape counts the requests of all frontends.
+    totals = [_success_total(server) for _ in range(NUM_FRONTENDS)]
+    assert totals == [NUM_REQUESTS + NUM_STREAMS] * NUM_FRONTENDS, totals

tests/unittest/llmapi/apps/openai_server.py

@@ -15,6 +15,19 @@
 from tensorrt_llm.llmapi.disagg_utils import ServerRole
 
 
+def _picks_num_frontends(cli_args: List[str],
+                         extra_config: Optional[dict]) -> bool:
+    """Whether a test sets num_serve_frontends (a --set override wins anyway)."""
+    configs = [extra_config or {}]
+    for arg, value in zip(cli_args, cli_args[1:] + [""]):
+        if arg.split("=")[0] == "--num_serve_frontends":
+            return True
+        if arg in ("--config", "--extra_llm_api_options") and value:
+            with open(value) as f:
+                configs.append(yaml.safe_load(f) or {})
+    return any("num_serve_frontends" in config for config in configs)
+
+
 class RemoteOpenAIServer:
     DUMMY_API_KEY = "tensorrt_llm"
     MAX_SERVER_START_WAIT_S = 7200  # wait for server to start for 7200 seconds (~ 2 hours) for LLM models weight loading
@@ -51,6 +64,10 @@ def __init__(self,
                 f.write(yaml.dump(extra_config))
                 self.extra_config_file = f.name
             args += ["--extra_llm_api_options", self.extra_config_file]
+        # trtllm-serve defaults to several frontend processes; test servers
+        # run one unless the test picks a count.
+        if not _picks_num_frontends(cli_args or [], extra_config):
+            args += ["--num_serve_frontends", "1"]
         launch_cmd = ["trtllm-serve"] + [model] + args
         if llmapi_launch:
             # start server with llmapi-launch on multi nodes