From 7ac66f0dfc214e21e3bdcc65247904676451219c Mon Sep 17 00:00:00 2001 From: Zhenhua Wang Date: Tue, 27 Oct 2026 23:56:41 -0700 Subject: [PATCH 1/2] [None][feat] Classify startup failures, surface their root cause and report them in telemetry Startup failures inside executor workers reach users as "Executor worker returned error" or "MPI worker exited unexpectedly", with the real cause only in earlier log lines. Usage telemetry cannot tell why a start failed either, or whether failures come from one deployment stuck in a restart loop: the exit report carries no cause, and every process gets a new random session ID. - Classify startup failures into coarse categories (config_input, resource, unsupported, model_load, runtime_init, other) and log one ERROR line with the root cause and an actionable hint at the LLM boundary, in the proxy before an abort that may end the process, and at the CLI boundary until an LLM has initialized in the process. - Let TensorRT-LLM raise sites declare the category with one exception class per category that derives only from its builtin, so callers can still catch it as ValueError or RuntimeError. About 50 startup raise sites are converted, a few of them to a different builtin where nothing catches the old one, and QuantAlgo raises UnsupportedError for a name it does not know. Errors that TensorRT-LLM does not raise pass through unchanged and are classified by exception type, error code, a fixed list of message phrases and the raising module; e.g. a --config file that does not exist is invalid input in trtllm-serve, trtllm-bench and trtllm-eval. - Carry the worker's classification, rank and initialization stage to the proxy, including from non-leader ranks, so the final error names the root cause; a "[]" marker carries it in the worker error that is forwarded as text. - Fail fast on certain preconditions before weights are loaded: no visible GPU or fewer GPUs than an LLM-spawned MPI pool needs on this node, a GPU below the documented minimum for the quantization format, and an unregistered model architecture. - Report the category of a startup failure on the usage telemetry exit report, and a bucketed count of consecutive failed starts of the same launch on the exit and initial reports. The count lives in a small local file that only the reporting process reads or writes, never when telemetry is disabled or after a late opt-out. Both reports wait at most 100 ms for that I/O, so a stalled file system cannot hold up shutdown or the initial report. VisualGen sessions report neither field. The event schema goes to 0.8, and the usage notice, the README and the schema reference describe the fields and the file. Signed-off-by: Zhenhua Wang --- README.md | 14 +- tensorrt_llm/_errors.py | 166 +++++ tensorrt_llm/_startup.py | 10 +- tensorrt_llm/_startup_errors.py | 496 +++++++++++++ .../backends/sparse/deepseek_v4/module.py | 3 +- .../sparse/minimax_m3/msa_availability.py | 5 +- .../_torch/custom_ops/torch_custom_ops.py | 3 +- tensorrt_llm/_torch/model_config.py | 3 +- .../models/checkpoints/hf/weight_loader.py | 5 +- .../checkpoints/mistral/config_loader.py | 3 +- tensorrt_llm/_torch/models/modeling_auto.py | 5 +- tensorrt_llm/_torch/models/modeling_dflash.py | 3 +- tensorrt_llm/_torch/models/modeling_utils.py | 11 +- .../_torch/moe/fused_moe/moe_resolution.py | 5 +- tensorrt_llm/_torch/pyexecutor/_util.py | 11 +- .../_torch/pyexecutor/model_loader.py | 26 +- .../_torch/pyexecutor/warmup_timer.py | 4 + tensorrt_llm/bench/benchmark/utils/general.py | 3 +- tensorrt_llm/bench/benchmark/visual_gen.py | 4 + tensorrt_llm/bench/tuning/heuristics.py | 15 +- tensorrt_llm/commands/_telemetry.py | 9 +- tensorrt_llm/commands/eval.py | 3 +- tensorrt_llm/commands/serve.py | 45 +- tensorrt_llm/executor/base_worker.py | 8 +- tensorrt_llm/executor/executor.py | 7 +- tensorrt_llm/executor/postprocessor_hook.py | 6 +- tensorrt_llm/executor/proxy.py | 41 +- tensorrt_llm/executor/worker.py | 23 +- tensorrt_llm/inputs/registry.py | 4 +- tensorrt_llm/llmapi/llm.py | 39 +- tensorrt_llm/llmapi/llm_args.py | 5 +- tensorrt_llm/llmapi/llm_utils.py | 24 +- tensorrt_llm/llmapi/startup_checks.py | 190 +++++ tensorrt_llm/models/quant_config_utils.py | 3 +- tensorrt_llm/quantization/mode.py | 10 + tensorrt_llm/usage/__init__.py | 2 + tensorrt_llm/usage/schema.py | 60 +- tensorrt_llm/usage/schemas/README.md | 95 ++- .../schemas/trtllm_usage_event_schema.json | 47 +- tensorrt_llm/usage/startup_state.py | 212 ++++++ tensorrt_llm/usage/usage_lib.py | 179 ++++- .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../test_quantization_for_cold_page.py | 2 +- .../modeling/test_modeling_out_of_tree.py | 17 +- .../kv_cache/test_kv_cache_budget_split.py | 6 +- .../test_kv_cache_compression_manager.py | 2 +- .../test_indexer_workspace_reserve.py | 2 +- .../executor/test_proxy_fast_death.py | 89 +++ .../inputs/test_input_processor_fallback.py | 43 ++ tests/unittest/llmapi/test_llm_args.py | 8 +- tests/unittest/llmapi/test_llm_telemetry.py | 31 +- tests/unittest/llmapi/test_startup_checks.py | 249 +++++++ tests/unittest/others/test_startup_errors.py | 664 ++++++++++++++++++ tests/unittest/usage/conftest.py | 8 + tests/unittest/usage/test_cli_telemetry.py | 176 ++++- tests/unittest/usage/test_e2e_capture.py | 20 +- tests/unittest/usage/test_reporter.py | 426 ++++++++++- tests/unittest/usage/test_schema.py | 67 +- tests/unittest/usage/test_startup_state.py | 239 +++++++ tests/unittest/usage/test_transport.py | 78 ++ 60 files changed, 3787 insertions(+), 148 deletions(-) create mode 100644 tensorrt_llm/_errors.py create mode 100644 tensorrt_llm/_startup_errors.py create mode 100644 tensorrt_llm/llmapi/startup_checks.py create mode 100644 tensorrt_llm/usage/startup_state.py create mode 100644 tests/unittest/inputs/test_input_processor_fallback.py create mode 100644 tests/unittest/llmapi/test_startup_checks.py create mode 100644 tests/unittest/others/test_startup_errors.py create mode 100644 tests/unittest/usage/test_startup_state.py diff --git a/README.md b/README.md index 76b3dc18daf0..e1e87835f4ed 100644 --- a/README.md +++ b/README.md @@ -336,6 +336,8 @@ deployment, and not linked to users. The data we collect includes: - Disaggregated serving metadata (role and deployment ID) - Process-local LLM lifecycle counts (initialization attempts, successful and active instances, peak concurrency, and initialization failures) - Bounded terminal status for instrumented processes (known exit code or signal, termination category, lifecycle phase, component, and reporting source) +- For a failed startup, its error category: one of `config_input`, `resource`, `unsupported`, `model_load`, `runtime_init`, or `other`, computed locally from the exception +- The number of consecutive failed starts of the same launch, as a coarse bucket (0, 1, 2, 3, 10, 30, 100, 300, or 1000 and more), and whether that local count was available - Selected LLM API configuration values: parallelism, dtype, KV cache, scheduler, CUDA graph, and compile settings - Capture diagnostics for that payload: a schema checksum (for provenance), the count of captured fields, and whether any free-form value was skipped @@ -344,6 +346,14 @@ Telemetry is automatically disabled in CI and test environments. If LLM startup fails, an initial report can still include the available environment and sanitized configuration information, marked as partial context. +To count consecutive failed starts, TensorRT-LLM keeps one small file per launch +under `~/.cache/tensorrt_llm/usage/` (or `$XDG_CACHE_HOME/tensorrt_llm/usage/`, +or the directory in `TRTLLM_USAGE_STATE_DIR`). The file holds only the count and +the time of the last failure. Its name is a local hash of the TensorRT-LLM +version, command line, launcher rank, visible GPUs, and TensorRT-LLM +process-role variables; none of these values is sent. The file is never read or +written when telemetry is disabled. + ### Opting Out of Telemetry Data Collection To disable telemetry data collection, use any of the following methods: @@ -354,8 +364,8 @@ To disable telemetry data collection, use any of the following methods: - **CLI flag**: Use `--no-telemetry` on `trtllm-serve`, `trtllm-bench`, or `trtllm-eval` CLI terminal telemetry may report that an argument or YAML configuration was -invalid. The terminal event records only categorical termination and lifecycle -metadata, never the raw argument or YAML text. Other telemetry events collect +invalid. The terminal event records only categorical termination, lifecycle, and +error-category metadata, never the raw argument or YAML text. Other telemetry events collect the selected, sanitized configuration fields listed above. The environment-variable, file-based, and CLI-flag opt-outs are all applied before CLI and YAML validation. diff --git a/tensorrt_llm/_errors.py b/tensorrt_llm/_errors.py new file mode 100644 index 000000000000..6d2981e8538e --- /dev/null +++ b/tensorrt_llm/_errors.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Error categories, the exception classes that declare them, and standard hints. + +A category says what went wrong, and so who acts on it. A raise site that knows +the category declares it, preferably by raising the category's class: + +- ``ConfigInputError`` (a ``ValueError``): invalid user input; +- ``UnsupportedError`` (a ``ValueError``): a model, GPU, quantization or feature + combination that this version does not support; +- ``ResourceError`` (a ``RuntimeError``): missing or exhausted GPUs, memory, disk or ports; +- ``ModelLoadError`` (a ``RuntimeError``): model files that cannot be read or do not + match the model; +- ``RuntimeInitError`` (a ``RuntimeError``): bringing up the runtime failed: worker + processes, distributed setup or kernel compilation. + +``unsupported`` is a model, GPU, quantization or feature request that the product +could reasonably support later; invalid or misspelled values are ``config_input``. + +Each class derives only from its builtin, so callers can always catch it with +``except ValueError`` or ``except RuntimeError``. The category travels with the +type; the message carries no marker. Any other exception class declares a +category with a ``category`` class attribute. ``"other"`` is never declared: it +is the classifier's fallback for errors that cannot be attributed to the other +categories. + +The reason is a key of ``ERROR_HINTS`` naming the specific cause, e.g. +``"gpu_oom"``; its hint is the next step for the user. Raise sites pick a +standard hint by passing its reason. +""" + +from typing import ClassVar, Literal, get_args + +# Usage telemetry reports the category of a startup failure as the exit report's +# ``errorCategory`` (see tensorrt_llm/usage/schemas/README.md). The names are a +# data contract: renaming one breaks its trend, and adding one also needs a +# telemetry schema update. +ErrorCategory = Literal[ + "config_input", "resource", "unsupported", "model_load", "runtime_init", "other" +] +ERROR_CATEGORIES: tuple[str, ...] = get_args(ErrorCategory) + +_DOCS_SITE = "https://nvidia.github.io/TensorRT-LLM" +# Standard hints keyed by reason. +ERROR_HINTS = { + "config": "Check the LLM arguments and the config file against the documentation.", + "import": "A required Python package is missing or broken; install it in this environment.", + "host_name": "The host name could not be resolved; check --host.", + "hf": ( + "The model could not be fetched from Hugging Face. Check the local path or the repo id " + "and revision; for a gated or private model, accept its license and set HF_TOKEN; " + "without access to the Hub, set HF_ENDPOINT to a mirror or pass a local path." + ), + "gpu_oom": ( + "GPU memory ran out. Lower kv_cache_config.free_gpu_memory_fraction, max_batch_size " + "or max_num_tokens, reduce the CUDA graph batch sizes, or use more GPUs " + "(tensor_parallel_size)." + ), + "host_memory": ( + "The host ran out of memory or file handles. Check the container memory limit, " + "the /dev/shm size (--shm-size) and `ulimit -n`." + ), + "disk": "The disk is full. Free space in the Hugging Face cache (HF_HOME) or move it.", + "port": "The listen address is not available. Use another --port or stop the process using it.", + "no_gpu": ( + "No GPU is usable. Check that the container has GPUs (docker --gpus, or nvidia.com/gpu " + "in Kubernetes), that CUDA_VISIBLE_DEVICES does not hide them, and that the driver " + "supports this CUDA version." + ), + "unsupported": ( + "This model, GPU, quantization or feature combination is not supported by this " + f"TensorRT-LLM version. See {_DOCS_SITE}/models/supported-models.html and " + f"{_DOCS_SITE}/supported-hardware.html." + ), + "model_load": ( + "The model files could not be loaded. Check that the checkpoint is complete, " + "in Hugging Face format and consistent with its config; re-download it if needed." + ), + "runtime_init": ( + "Bringing up the runtime failed (worker processes, multi-GPU communication or kernel " + "compilation). Check the MPI / Slurm launch and the NCCL setup, the CUDA toolkit " + "(nvcc) for kernel compilation, and the host memory and /dev/shm limits." + ), + "distributed": ( + "Multi-GPU initialization failed. Check that every rank has a GPU of its own (two ranks " + "on one GPU fail with NCCL 'invalid usage'), rerun with NCCL_DEBUG=INFO, and check the " + "MPI / Slurm launch and the network setup." + ), + "gpu_busy": ( + "The GPU is busy or unavailable. Check `nvidia-smi` for other processes on it and for " + "an exclusive compute mode." + ), + "jit": ( + "Kernel compilation failed. Check that the CUDA toolkit (nvcc) is installed and " + "that the JIT cache directory is writable." + ), + "internal": ( + "This looks like an internal TensorRT-LLM error. Please report it at " + "https://github.com/NVIDIA/TensorRT-LLM/issues with the full log." + ), + "worker_died": ( + "A worker process exited without reporting an error; check its log above. If it was " + "killed, check host memory (OOM killer, dmesg), the container memory limit and /dev/shm." + ), +} +# A category's reason when nothing more specific is known; "" has no hint. +DEFAULT_REASONS: dict[str, str] = { + "config_input": "config", + "resource": "", + "unsupported": "unsupported", + "model_load": "model_load", + "runtime_init": "runtime_init", + "other": "", +} + + +class _CategorizedError: + """Code shared by the category classes; not an exception type and not for ``except``. + + ``category`` is a class attribute; ``reason`` and ``hint`` are instance + attributes, so they survive pickling across processes. ``str()`` is the + plain message. + + Args: + message: What failed. + reason: Key of ``ERROR_HINTS`` naming the specific cause; defaults to the category's. + hint: Next step for the user; defaults to the reason's standard hint. + """ + + category: ClassVar[ErrorCategory] + + def __init__(self, message: str = "", *, reason: str = "", hint: str = ""): + super().__init__(message) + self.reason = reason or DEFAULT_REASONS[self.category] + self.hint = hint or ERROR_HINTS.get(self.reason, "") + + +class ConfigInputError(_CategorizedError, ValueError): + """Invalid user input: arguments, configuration files or launch parameters.""" + + category = "config_input" + + +class UnsupportedError(_CategorizedError, ValueError): + """A model, GPU, quantization or feature combination that this version does not support.""" + + category = "unsupported" + + +class ResourceError(_CategorizedError, RuntimeError): + """Missing or exhausted resources: GPUs, memory, disk or network ports.""" + + category = "resource" + + +class ModelLoadError(_CategorizedError, RuntimeError): + """Model files that cannot be read or do not match the model.""" + + category = "model_load" + + +class RuntimeInitError(_CategorizedError, RuntimeError): + """Bringing up the runtime failed: worker processes, distributed setup or kernels.""" + + category = "runtime_init" diff --git a/tensorrt_llm/_startup.py b/tensorrt_llm/_startup.py index eb8a73d68ea1..086042c54a6b 100644 --- a/tensorrt_llm/_startup.py +++ b/tensorrt_llm/_startup.py @@ -8,6 +8,7 @@ from contextlib import contextmanager from typing import Iterator +from tensorrt_llm._startup_errors import mark_startup_stage from tensorrt_llm.logger import logger @@ -33,6 +34,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): """Log the complete or failed interval without suppressing exceptions.""" + mark_startup_stage(exc_value, self.name) total = time.perf_counter() - self.started measured = sum(self.timings.values()) parts = ", ".join(f"{name}={seconds:.3f}s" for name, seconds in self.timings.items()) @@ -58,8 +60,9 @@ def phase( ) -> Iterator[None]: """Record outermost elapsed time, including failures, and log status. - Failed work still contributes to startup time; its exception propagates - and its completion marker is explicitly labelled ``failed``. + Failed work still contributes to startup time; its exception propagates, + records this phase as its startup stage unless an inner phase already + did, and its completion marker is explicitly labelled ``failed``. When ``metrics`` is provided, accumulate the same elapsed value under ``metric_name`` (or ``name``), including nested phases. """ @@ -71,6 +74,9 @@ def phase( try: yield completed = True + except BaseException as exc: + mark_startup_stage(exc, f"{self.name}/{name}") + raise finally: elapsed = time.perf_counter() - start self.depth -= 1 diff --git a/tensorrt_llm/_startup_errors.py b/tensorrt_llm/_startup_errors.py new file mode 100644 index 000000000000..a11876bc41ef --- /dev/null +++ b/tensorrt_llm/_startup_errors.py @@ -0,0 +1,496 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Classify startup failures into coarse categories and log one actionable hint. + +The categories, reasons, hints and category exception classes do not depend on +a phase; they are defined in ``tensorrt_llm._errors``. This module is the +startup boundary: it classifies a failure and logs the hint line only where +initialization fails, i.e. in the ``LLM`` constructor, the executor worker and +proxy, and the CLI boundary until an ``LLM`` has initialized in this process. + +Declared categories decide first: a ``category`` attribute (the classes in +``tensorrt_llm._errors``, or any exception class that sets one), the worker's +classification carried by the proxy, and the ``[]`` marker in the text +that ``wrap_worker_startup_error`` forwards for a worker. +Failures that are not declared (builtin, third-party, C++, CUDA and NCCL errors, +and TensorRT-LLM errors raised as plain builtins) are classified by exception +type, errno and CUDA error code, then by the fixed message phrases and raising +modules below. Unmatched errors are ``other``, as are errors that look internal +to TensorRT-LLM, which get the ``internal`` reason and hint. + +The classifier only depends on the standard library. Third-party exceptions +(torch, Hugging Face, pydantic, MPI, ...) are recognized by class name and +module, so importing this module never imports those packages. + +Classification metadata (category, reason, hint, stage, rank, summary) is stored +in the exception's ``__dict__``, so it survives pickling across the worker/proxy +process boundary. +""" + +import dataclasses +import errno +import re +from typing import Optional + +from tensorrt_llm._errors import DEFAULT_REASONS, ERROR_CATEGORIES, ERROR_HINTS +from tensorrt_llm.logger import logger + +# Categories a raise site can declare; "other" is only the classifier's fallback. +_DECLARABLE_CATEGORIES = tuple(c for c in ERROR_CATEGORIES if c != "other") +_MARKER_RE = re.compile(r"\[(" + "|".join(_DECLARABLE_CATEGORIES) + r")\] ") + +_META_ATTR = "_trtllm_startup_error" +_MAX_CHAIN_DEPTH = 8 +_MAX_SUMMARY_CHARS = 300 + +_UNSUPPORTED_RE = re.compile( + r"\bnot (?:yet |currently )?supported\b|\bunsupported\b(?! operand)" + r"|\b(?:does|do) not support\b|\bonly supported (?:on|for|with|by)\b|\bonly supports\b" + r"|\bsupported on sm ?\d" + r"|\brequires? (?:an? |the )?(?:sm ?\d|compute capability|hopper|blackwell)" + r"|\bunknown (?:model )?architecture\b" + r"|\bnot included in this TensorRT-LLM build\b", + re.IGNORECASE, +) +_OOM_RE = re.compile( + r"out of memory|cudaErrorMemoryAllocation|CUDA_ERROR_OUT_OF_MEMORY", re.IGNORECASE +) +_WORKER_DIED_RE = re.compile(r"died during initialization|exited unexpectedly", re.IGNORECASE) +# C++ kernel compilation: DeepGEMM's NVRTC JIT and the trtllm-gen FMHA nvcc probe. +_CPP_JIT_RE = re.compile(r"NVRTC compilation failed|Cannot parse NVCC version", re.IGNORECASE) +_GPU_BUSY_RE = re.compile(r"busy or unavailable|cudaErrorDevicesUnavailable", re.IGNORECASE) +_NCCL_RE = re.compile( + r"\bnccl\b.{0,40}\b(?:error|failure|failed)\b" + r"|\bnccl(?:InternalError|SystemError|UnhandledCudaError|InvalidUsage|RemoteError)\b", + re.IGNORECASE, +) +_MODEL_LOAD_MODULES = ( + "transformers.", + "safetensors", + "tensorrt_llm._torch.models.checkpoints", + "tensorrt_llm._torch.pyexecutor.model_loader", +) +# CLI code opens the files that the user named on the command line, e.g. --config. +_CLI_MODULES = ( + "tensorrt_llm.commands", + "tensorrt_llm.bench.benchmark", + "tensorrt_llm.llmapi.disagg_utils", +) +_JIT_MODULES = ( + "flashinfer", + "triton", + "deep_gemm", + "cutlass", + "tvm_ffi", + "torch.utils.cpp_extension", +) +# Builtin types that, raised inside tensorrt_llm without a declared category, +# usually mean an internal error: "other" with the "internal" reason. +_INTERNAL_ERROR_TYPES = ( + AssertionError, + AttributeError, + IndexError, + KeyError, + NameError, + TypeError, + ZeroDivisionError, +) +# torch.AcceleratorError.error_code values (cudaError_t). +_CUDA_OOM_CODES = {2} +_CUDA_NO_DEVICE_CODES = {35, 100, 803, 804} # insufficient driver, no device, compat issues +_CUDA_BUSY_CODES = {46} # cudaErrorDevicesUnavailable: exclusive mode or in use +_CUDA_NO_KERNEL_IMAGE_CODES = {209} +# Message phrases recognized when no raise site declared the category: text from +# C++, CUDA, NCCL and third-party code, and TensorRT-LLM errors raised as plain +# builtins. New TensorRT-LLM errors declare their category instead. +_MESSAGE_RULES = ( + (_OOM_RE, "resource", "gpu_oom"), + (_GPU_BUSY_RE, "resource", "gpu_busy"), + (_NCCL_RE, "runtime_init", "distributed"), + (_WORKER_DIED_RE, "runtime_init", "worker_died"), + (_CPP_JIT_RE, "runtime_init", "jit"), + (_UNSUPPORTED_RE, "unsupported", "unsupported"), +) + +# Set once an LLM in this process has finished initializing. +_llm_initialized = False + + +def mark_llm_initialized() -> None: + """Record that an ``LLM`` in this process finished initializing. + + From then on the CLI boundary does not classify uncaught exceptions: they are + failures of a running process, not startup failures. A later ``LLM`` that + fails to initialize is still classified by its own boundary. + """ + global _llm_initialized + _llm_initialized = True + + +def llm_initialized() -> bool: + """Whether an ``LLM`` in this process has finished initializing.""" + return _llm_initialized + + +@dataclasses.dataclass(frozen=True) +class StartupErrorInfo: + """Classification of one startup failure. + + Args: + category: Coarse category, one of ``ERROR_CATEGORIES``. + summary: ``": "`` of the deciding exception, on one line. + hint: Actionable next step for the user; may be empty. + stage: Initialization stage that failed (e.g. ``executor_creation/model``); may be empty. + rank: MPI rank that failed, when known. + reason: Key of ``ERROR_HINTS`` naming the specific cause; may be empty. + """ + + category: str + summary: str + hint: str = "" + stage: str = "" + rank: Optional[int] = None + reason: str = "" + + +def _meta(exc: BaseException) -> dict: + meta = exc.__dict__.get(_META_ATTR) + if meta is None: + meta = {} + setattr(exc, _META_ATTR, meta) + return meta + + +def mark_startup_stage(exc: Optional[BaseException], stage: str) -> None: + """Record the initialization stage on ``exc`` unless an inner stage already did.""" + if exc is None or not stage: + return + try: + _meta(exc).setdefault("stage", stage) + except Exception: # noqa: BLE001 - annotating must never replace the original error + pass + + +def _exception_chain(exc: BaseException) -> list[BaseException]: + """Return the cause/context chain, deepest first, including pydantic inner errors.""" + chain: list[BaseException] = [] + seen: set[int] = set() + current: Optional[BaseException] = exc + while current is not None and id(current) not in seen and len(chain) < _MAX_CHAIN_DEPTH: + seen.add(id(current)) + chain.append(current) + # Errors raised inside pydantic validators count as deeper than the ValidationError. + inner = [e for e in _pydantic_inner_errors(current) if id(e) not in seen] + chain.extend(reversed(inner)) + seen.update(id(e) for e in inner) + if current.__cause__ is not None: + current = current.__cause__ + elif not current.__suppress_context__: + current = current.__context__ + else: + current = None + chain.reverse() + return chain + + +def _has_class(exc: BaseException, names: tuple[str, ...], module_prefix: str = "") -> bool: + return any( + cls.__name__ in names and cls.__module__.startswith(module_prefix) + for cls in type(exc).__mro__ + ) + + +def _is_pydantic_validation_error(exc: BaseException) -> bool: + return _has_class(exc, ("ValidationError",), "pydantic") + + +def _pydantic_inner_errors(exc: BaseException) -> list[BaseException]: + if not _is_pydantic_validation_error(exc): + return [] + try: + details = exc.errors() # type: ignore[attr-defined] + except Exception: # noqa: BLE001 - best effort; the outer error still classifies + return [] + return [ + d["ctx"]["error"] + for d in details + if isinstance(d.get("ctx"), dict) and isinstance(d["ctx"].get("error"), BaseException) + ] + + +def _message(exc: BaseException) -> str: + try: + return str(exc) + except Exception: # noqa: BLE001 - a broken __str__ must not break classification + return "" + + +def _summarize(exc: BaseException, hint: str = "") -> str: + """One bounded line; drops the category marker and text that repeats the hint.""" + message = _MARKER_RE.sub("", _message(exc), count=1) + if hint: + message = message.replace(hint, "") + text = " ".join(message.split()) + if len(text) > _MAX_SUMMARY_CHARS: + text = text[: _MAX_SUMMARY_CHARS - 3] + "..." + name = type(exc).__name__ + return f"{name}: {text}" if text else name + + +def _raise_module(exc: BaseException) -> str: + tb = exc.__traceback__ + if tb is None: + return "" + while tb.tb_next is not None: + tb = tb.tb_next + return tb.tb_frame.f_globals.get("__name__", "") + + +# (category, reason, hint) decided by a rule. +_Hit = tuple[str, str, str] + + +def _hit(category: str, reason: str) -> _Hit: + return category, reason, ERROR_HINTS.get(reason, "") + + +def _declared_rule(exc: BaseException) -> Optional[_Hit]: + # Any exception class declares its category with a ``category`` attribute. + category = getattr(exc, "category", None) + if isinstance(category, str) and category in _DECLARABLE_CATEGORIES: + reason = getattr(exc, "reason", None) + if not isinstance(reason, str) or not reason: + reason = DEFAULT_REASONS[category] + hint = getattr(exc, "hint", None) + if not isinstance(hint, str) or not hint: + hint = ERROR_HINTS.get(reason, "") + return category, reason, hint + # The worker's classification, carried by the proxy's wrapper. + meta = exc.__dict__.get(_META_ATTR) or {} + if meta.get("category"): + return meta["category"], meta.get("reason", ""), meta.get("hint", "") + # The worker's classification, forwarded as text only (wrap_worker_startup_error). + match = _MARKER_RE.search(_message(exc)) + if match is None: + return None + category = match.group(1) + # The marker decides the category; a recognized phrase can still add a specific hint. + phrase = _phrase_rule(exc) + if phrase is not None and phrase[0] == category: + return phrase + return _hit(category, DEFAULT_REASONS[category]) + + +def _typed_rule(exc: BaseException) -> Optional[_Hit]: + # Resources. + if _has_class(exc, ("OutOfMemoryError",), "torch"): + return _hit("resource", "gpu_oom") + if isinstance(exc, MemoryError): + return _hit("resource", "host_memory") + if _has_class(exc, ("gaierror",), "socket"): + return _hit("config_input", "host_name") + if isinstance(exc, OSError) and exc.errno is not None: + if exc.errno in (errno.EADDRINUSE, errno.EADDRNOTAVAIL): + return _hit("resource", "port") + if exc.errno == errno.ENOSPC: + return _hit("resource", "disk") + if exc.errno in (errno.ENOMEM, errno.EMFILE): + return _hit("resource", "host_memory") + if _has_class(exc, ("AcceleratorError",), "torch"): + code = getattr(exc, "error_code", None) + if code in _CUDA_OOM_CODES: + return _hit("resource", "gpu_oom") + if code in _CUDA_NO_DEVICE_CODES: + return _hit("resource", "no_gpu") + if code in _CUDA_BUSY_CODES: + return _hit("resource", "gpu_busy") + if code in _CUDA_NO_KERNEL_IMAGE_CODES: + return _hit("unsupported", "unsupported") + return _hit("other", "internal") + + # User input. A file missing from a repo that was found is a model-file problem; + # LocalEntryNotFoundError (offline, not cached) subclasses EntryNotFoundError. + if _has_class(exc, ("EntryNotFoundError",), "huggingface_hub") and not _has_class( + exc, ("LocalEntryNotFoundError",), "huggingface_hub" + ): + return _hit("model_load", "model_load") + if _has_class( + exc, + ( + "HfHubHTTPError", + "GatedRepoError", + "RepositoryNotFoundError", + "RevisionNotFoundError", + "HFValidationError", + "LocalEntryNotFoundError", + "OfflineModeIsEnabled", + ), + "huggingface_hub", + ) or any( + cls.__module__.split(".")[0] in ("requests", "httpx", "urllib3") + for cls in type(exc).__mro__ + ): + return _hit("config_input", "hf") + if _is_pydantic_validation_error(exc): + # A validator that rejects a feature combination is a product gap, not a typo. + if any(_UNSUPPORTED_RE.search(_message(e)) for e in _pydantic_inner_errors(exc)): + return _hit("unsupported", "unsupported") + return _hit("config_input", "config") + if _has_class(exc, ("YAMLError",), "yaml") or _has_class(exc, ("ClickException",), "click"): + return _hit("config_input", "config") + if _has_class(exc, ("ConfigOverrideError",), "tensorrt_llm"): + return _hit("config_input", "config") + if isinstance(exc, ImportError): + return _hit("config_input", "import") + + # Product support. + if isinstance(exc, NotImplementedError): + return _hit("unsupported", "unsupported") + + # Model files. + if _has_class(exc, ("SafetensorError",)): + return _hit("model_load", "model_load") + + # Runtime initialization. + if _has_class(exc, ("Exception",), "mpi4py") or _has_class( + exc, ("DistBackendError", "DistNetworkError", "DistStoreError"), "torch" + ): + return _hit("runtime_init", "distributed") + if _has_class(exc, ("CompilationError", "OutOfResources"), "triton") or _has_class( + exc, ("CalledProcessError",), "subprocess" + ): + return _hit("runtime_init", "jit") + return None + + +def _phrase_rule(exc: BaseException) -> Optional[_Hit]: + message = _message(exc) + for pattern, category, reason in _MESSAGE_RULES: + if pattern.search(message): + return _hit(category, reason) + return None + + +def _heuristic_rule(exc: BaseException) -> Optional[_Hit]: + phrase = _phrase_rule(exc) + if phrase is not None: + return phrase + module = _raise_module(exc) + if module.startswith(_MODEL_LOAD_MODULES): + return _hit("model_load", "model_load") + if isinstance(exc, FileNotFoundError) and module.startswith(_CLI_MODULES): + return _hit("config_input", "config") + if module.startswith(_JIT_MODULES): + return _hit("runtime_init", "jit") + if isinstance(exc, KeyError) and module.startswith("tensorrt_llm._torch.models"): + return _hit("model_load", "model_load") + if isinstance(exc, _INTERNAL_ERROR_TYPES) and module.startswith("tensorrt_llm"): + return _hit("other", "internal") + return None + + +def classify_startup_error(exc: BaseException, rank: Optional[int] = None) -> StartupErrorInfo: + """Classify ``exc`` by walking its cause chain from the deepest exception outward. + + Three passes, each taking the first match: declared categories (a + ``category`` attribute, the worker's classification carried by the proxy or + by the ``[]`` marker in its forwarded text), typed rules (exception + types, errno, CUDA error codes), then local-only heuristics (message phrases + and raising module). + Internal-looking errors are ``other`` with the ``internal`` reason; unmatched + errors are ``other`` with no reason. Never raises. + + Args: + exc: The failure to classify. + rank: MPI rank to record when the caller knows it, e.g. in a worker. + """ + try: + info = _classify(exc) + except Exception: # noqa: BLE001 - classification must never replace the original error + info = StartupErrorInfo("other", type(exc).__name__) + return info if rank is None else dataclasses.replace(info, rank=rank) + + +def _classify(exc: BaseException) -> StartupErrorInfo: + chain = _exception_chain(exc) + decided: Optional[tuple[BaseException, str, str, str]] = None + for rule in (_declared_rule, _typed_rule, _heuristic_rule): + for e in chain: + hit = rule(e) + if hit is not None: + decided = (e, *hit) + break + if decided is not None: + break + if decided is None: + decided = (chain[0], "other", "", "") + decider, category, reason, hint = decided + + metas = [e.__dict__.get(_META_ATTR) or {} for e in chain] + stage = next((m["stage"] for m in metas if m.get("stage")), "") + rank = next((m["rank"] for m in metas if m.get("rank") is not None), None) + summary = (decider.__dict__.get(_META_ATTR) or {}).get("summary") or _summarize(decider, hint) + return StartupErrorInfo(category, summary, hint, stage, rank, reason) + + +def wrap_worker_startup_error(message: str, info: Optional[StartupErrorInfo]) -> RuntimeError: + """Build the error reporting a worker's initialization failure, with its classification. + + The error holds only strings and ints, so it pickles across processes. Its + message alone names the rank, stage, ``[]`` marker and root cause, + so the worker's classification also survives channels that forward only + text, such as a non-leader rank under ``trtllm-llmapi-launch``. + + Args: + message: Fixed prefix, e.g. ``"Executor worker returned error"``. + info: Classification computed in the worker process, or None if unavailable. + """ + if info is None: + return RuntimeError(message) + where = "" if info.rank is None else f" on rank {info.rank}" + if info.stage: + where += f" during {info.stage}" + summary = info.summary + if info.category in _DECLARABLE_CATEGORIES: + summary = f"[{info.category}] {summary}" + error = RuntimeError(f"{message}{where}: {summary}") + _meta(error).update(dataclasses.asdict(info)) + return error + + +def is_worker_startup_error(exc: BaseException) -> bool: + """Whether ``exc`` came from ``wrap_worker_startup_error`` with a worker classification.""" + return "summary" in (exc.__dict__.get(_META_ATTR) or {}) + + +def format_startup_error(info: StartupErrorInfo, what: str) -> str: + """Render the single ERROR line for a classified startup failure.""" + where = "" if info.rank is None else f" on rank {info.rank}" + if info.stage: + where += f" during {info.stage}" + line = f"{what} [{info.category}]{where}: {info.summary}" + return f"{line} Hint: {info.hint}" if info.hint else line + + +def log_startup_error(exc: BaseException, what: str) -> Optional[StartupErrorInfo]: + """Classify ``exc`` and log one ERROR line, unless it was already logged. + + Never raises, so callers can use it right before re-raising ``exc``. + + Args: + exc: The exception about to propagate. + what: Leading words of the line, e.g. ``"LLM initialization failed"``. + + Returns: + The classification, or None if nothing was logged. + """ + try: + if any((e.__dict__.get(_META_ATTR) or {}).get("logged") for e in _exception_chain(exc)): + return None + info = classify_startup_error(exc) + _meta(exc)["logged"] = True + logger.error(format_startup_error(info, what)) + return info + except Exception: # noqa: BLE001 - diagnostics must never mask the original error + return None diff --git a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/module.py b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/module.py index dcbd878d613d..c577e1f90785 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/module.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/deepseek_v4/module.py @@ -10,6 +10,7 @@ import torch from torch import nn +from tensorrt_llm._errors import UnsupportedError from tensorrt_llm._torch.attention.backends.interface import AttentionInputType, AttentionMetadata from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention from tensorrt_llm._torch.attention.rotary_embedding import RotaryEmbedding @@ -66,7 +67,7 @@ def _has_dsv4_indexer(self) -> bool: def _get_validated_sm_version() -> int: sm_version = get_sm_version() if sm_version < 90: - raise RuntimeError(f"DeepSeek-V4 requires Hopper or newer GPUs, got SM{sm_version}") + raise UnsupportedError(f"DeepSeek-V4 requires Hopper or newer GPUs, got SM{sm_version}") return sm_version diff --git a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_availability.py b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_availability.py index f76443e2e1ee..2028381ba4a6 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_availability.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/minimax_m3/msa_availability.py @@ -11,6 +11,7 @@ from __future__ import annotations +from tensorrt_llm._errors import UnsupportedError from tensorrt_llm._utils import get_sm_version, is_sm_100f from .kernels.msa_utils import msa_package_available @@ -22,7 +23,7 @@ def ensure_msa_available() -> None: - """Raise RuntimeError if the MSA sparse attention path cannot run here.""" + """Raise UnsupportedError or RuntimeError if the MSA sparse attention path cannot run here.""" # Function-local: msa_indexer reaches the trtllm attention classes through # this package's init, which a module-scope import here would cycle with. from .msa_indexer import cutedsl_score_runner @@ -34,7 +35,7 @@ def ensure_msa_available() -> None: ) if not is_sm_100f(): sm_version = get_sm_version() - raise RuntimeError( + raise UnsupportedError( "MiniMax-M3 MSA sparse attention requires an SM100 or SM103 device, " f"but the current device reports SM version {sm_version}." ) diff --git a/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py b/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py index 672c9b9f20e8..0562139a43d6 100644 --- a/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py @@ -27,6 +27,7 @@ import tensorrt_llm.quantization.utils.fp4_utils as fp4_utils from tensorrt_llm import deep_gemm +from tensorrt_llm._errors import UnsupportedError from tensorrt_llm._torch.distributed.allreduce_helper import \ CustomAllReduceHelper from tensorrt_llm._utils import get_sm_version, is_sm_100f, is_sm_120f @@ -396,7 +397,7 @@ def fused_moe( except RuntimeError as e: error_msg = str(e) if "DeepGEMM only supports Hopper" in error_msg: - raise RuntimeError( + raise UnsupportedError( f"{error_msg}" "Note: This is the Cutlass backend with DeepGemm JIT path. " "For Blackwell (SM100+) support, please use the DEEPGEMM backend instead." diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 5615d44c10c6..5fbf7259e7bb 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -29,6 +29,7 @@ import transformers from transformers.utils import HF_MODULES_CACHE +from tensorrt_llm._errors import ModelLoadError from tensorrt_llm._torch.locality_domain.policy import LocalityDomainPolicy from tensorrt_llm._torch.pyexecutor.config_utils import ( get_glm5_next_layer_masks, get_glm5_next_num_attention_layers, @@ -717,7 +718,7 @@ def _build_modelopt_quant_config(json_quant_configs, checkpoint_dir, kv_quant_rhs := quant_config.kv_cache_quant_algo) is not None: if kv_quant_lhs != kv_quant_rhs: - raise RuntimeError( + raise ModelLoadError( f"The kvcache config in 'quant_cfg.json', {kv_quant_lhs}," f"is different from 'hf_quant_config.json', {kv_quant_rhs}!" ) diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/weight_loader.py b/tensorrt_llm/_torch/models/checkpoints/hf/weight_loader.py index 4069d16c5ca9..2c403c5b0a1a 100644 --- a/tensorrt_llm/_torch/models/checkpoints/hf/weight_loader.py +++ b/tensorrt_llm/_torch/models/checkpoints/hf/weight_loader.py @@ -31,6 +31,7 @@ import tqdm from mpi4py import MPI as _MPI +from tensorrt_llm._errors import ModelLoadError from tensorrt_llm._torch.mmap_utils import populate_file_pages from tensorrt_llm._torch.models.checkpoints.base_weight_loader import ( BaseWeightLoader, ConsumableWeightsDict) @@ -435,7 +436,7 @@ def _load_lazy_safetensors( """ weight_files = sorted(glob.glob(f"{checkpoint_dir}/*.safetensors")) if not weight_files: - raise RuntimeError(f"No safetensors files in {checkpoint_dir}.") + raise ModelLoadError(f"No safetensors files in {checkpoint_dir}.") # Same sharded-vs-consolidated selection as the eager path below: # when both flavors are present, keep only the requested one. filtered_weight_files = [ @@ -998,7 +999,7 @@ def _load_weights_native(self, weight_files, self._load_bin_or_path_file, "Loading bin weights in parallel")) - raise RuntimeError(f"No weight files found in {checkpoint_dir}.") + raise ModelLoadError(f"No weight files found in {checkpoint_dir}.") def _prefetch_and_load( self, diff --git a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py index 8ddff08d81f1..c0946e32e1bc 100644 --- a/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py +++ b/tensorrt_llm/_torch/models/checkpoints/mistral/config_loader.py @@ -20,6 +20,7 @@ import torch from transformers import PretrainedConfig, WhisperConfig +from tensorrt_llm._errors import UnsupportedError from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.checkpoints.base_config_loader import BaseConfigLoader from tensorrt_llm._torch.models.modeling_utils import register_config_loader @@ -205,7 +206,7 @@ def _remap_mistral_quantization_args(config: dict) -> dict: "activation_scheme": "dynamic" if is_dynamic else "static", } else: - raise ValueError(f"Found unknown quantization='{quantization}' in config") + raise UnsupportedError(f"Found unknown quantization='{quantization}' in config") return config diff --git a/tensorrt_llm/_torch/models/modeling_auto.py b/tensorrt_llm/_torch/models/modeling_auto.py index 370fa5626590..9f953d60d5f2 100644 --- a/tensorrt_llm/_torch/models/modeling_auto.py +++ b/tensorrt_llm/_torch/models/modeling_auto.py @@ -5,6 +5,7 @@ from tensorrt_llm.logger import logger +from ..._errors import UnsupportedError from .._experimental.modeling_v2._router_index import (ModelingV2Mode, NoTarget, resolved_mode) from ..model_config import ModelConfig @@ -64,14 +65,14 @@ def from_config( model_arch = config.pretrained_config.architectures[0] vision_encoder_info = get_registered_vision_encoder(model_arch) if vision_encoder_info is None: - raise ValueError( + raise UnsupportedError( f"Unknown architecture for AutoModelForMultimodalEncoder: {model_arch}" ) vision_encoder_cls, vlm_base_model = vision_encoder_info return vision_encoder_cls(config, vlm_base_model) cls = AutoModelForCausalLM._resolve_class(config) if cls is None: - raise ValueError( + raise UnsupportedError( f"Unknown architecture for AutoModelForCausalLM: {config.pretrained_config.architectures[0]}" ) if issubclass(cls, DecoderModelForCausalLM): diff --git a/tensorrt_llm/_torch/models/modeling_dflash.py b/tensorrt_llm/_torch/models/modeling_dflash.py index 459b90ed8560..c09a460b63f7 100644 --- a/tensorrt_llm/_torch/models/modeling_dflash.py +++ b/tensorrt_llm/_torch/models/modeling_dflash.py @@ -10,6 +10,7 @@ from torch import nn from transformers import PretrainedConfig +from tensorrt_llm._errors import ConfigInputError, UnsupportedError from tensorrt_llm.llmapi.llm_args import is_dflash2_checkpoint, is_lilicorr_checkpoint from tensorrt_llm.logger import logger @@ -597,7 +598,7 @@ def __init__(self, draft_config, *, dflash_attention_backend: str = "AUTO"): pretrained_cfg = draft_config.pretrained_config try: DraftModelClass, _ = get_model_architecture(pretrained_cfg) - except RuntimeError: + except (RuntimeError, ConfigInputError, UnsupportedError): model_type = pretrained_cfg.model_type arch_name = "".join(w.capitalize() for w in model_type.split("_")) + "ForCausalLM" logger.info( diff --git a/tensorrt_llm/_torch/models/modeling_utils.py b/tensorrt_llm/_torch/models/modeling_utils.py index 5e71c5da8da1..f28059fa7196 100755 --- a/tensorrt_llm/_torch/models/modeling_utils.py +++ b/tensorrt_llm/_torch/models/modeling_utils.py @@ -18,6 +18,8 @@ from torch.utils._pytree import tree_any_only from tqdm import tqdm +from tensorrt_llm._errors import (ConfigInputError, ModelLoadError, + UnsupportedError) from tensorrt_llm._torch.peft.lora.loaders import HfLoraLoader from tensorrt_llm._utils import local_mpi_rank from tensorrt_llm.checkpoint_format import CheckpointFormat @@ -1498,16 +1500,17 @@ def get_model_architecture( model_config.architectures) > 0: cls = get_registered_model_class(model_config.architectures[0]) else: - raise RuntimeError("Model architecture is not provided.") + raise ModelLoadError("Model architecture is not provided.") if cls is None: arch = model_config.architectures[0] if arch in _GEMMA4_ARCHITECTURES: - raise RuntimeError( + raise ConfigInputError( f"Gemma4 model support requires transformers>=5.5.0, " f"please upgrade: pip install 'transformers>=5.5.0' " - f"(architecture: {arch}).") - raise RuntimeError(f"Unknown model architecture: {arch}") + f"(architecture: {arch}).", + reason="import") + raise UnsupportedError(f"Unknown model architecture: {arch}") return cls, model_config.architectures[0] diff --git a/tensorrt_llm/_torch/moe/fused_moe/moe_resolution.py b/tensorrt_llm/_torch/moe/fused_moe/moe_resolution.py index ddd4fde3a2e7..96885aafa6f5 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/moe_resolution.py +++ b/tensorrt_llm/_torch/moe/fused_moe/moe_resolution.py @@ -22,6 +22,7 @@ import torch +from tensorrt_llm._errors import UnsupportedError from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.peft.lora.validation import has_moe_lora_targets from tensorrt_llm._torch.utils import ActivationType @@ -633,7 +634,7 @@ def resolve_moe_impl( location = "" if layer_idx is None else f" [layer_idx={layer_idx}]" if report.degraded and not allow_degradation: - raise ValueError( + raise UnsupportedError( f"MoE backend {requested} was requested with degradation disallowed " f"but cannot serve this layer{location}. {report.describe()}" f" Each candidate's reason: {report.describe_rejections()}" @@ -675,7 +676,7 @@ def impl_class_for(report: MoEResolutionReport) -> MoEImplClass: if report.winner is None: # describe() prints reason codes only. With nothing left to run, the # operator needs the details too -- that is all the error can offer. - raise ValueError( + raise UnsupportedError( f"no MoE implementation can serve this layer. {report.describe()}" f" Each candidate's reason: {report.describe_rejections()}" ) diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index ba20671e2b23..3d0b0d3a864d 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -22,6 +22,7 @@ import tensorrt_llm import tensorrt_llm.bindings.executor as trtllm +from tensorrt_llm._errors import ResourceError, UnsupportedError from tensorrt_llm._utils import (confidential_compute_enabled, get_sm_version, is_sm_100f, prefer_pinned, str_dtype_to_binding, torch_dtype_to_str) @@ -697,7 +698,7 @@ def get_indexer_mqa_logits_workspace_reserve(kv_cache_memory: int, if kv_cache_memory <= 0 or workspace_memory <= 0: return 0 if workspace_memory >= kv_cache_memory: - raise ValueError( + raise ResourceError( f"The DeepSeek-V4 Indexer MQA-logits workspace ({workspace_memory} bytes) " f"needs the whole KV cache budget ({kv_cache_memory} bytes). Lower " "TLLM_INDEXER_MQA_LOGITS_ELEM_BUDGET or max_num_tokens, or raise " @@ -3090,7 +3091,7 @@ def _split_kv_cache_budget_for_draft( When the split is *infeasible* (the combined fixed cost exhausts the budget while either manager has a per-token cost, or exceeds it) the shortfall is fatal: both managers need their fixed state resident in - GPU memory, so the run would OOM. It raises ``ValueError`` rather than + GPU memory, so the run would OOM. It raises ``ResourceError`` rather than silently producing an unusable config. A defensive degrade-to-zero path for non-GPU budgets remains so the draft never silently inherits the full budget and double-allocates it. @@ -3133,7 +3134,7 @@ def _split_kv_cache_budget_for_draft( # fatal: both managers need their fixed state resident in GPU # memory, so the run would OOM. Fail fast with actionable # guidance rather than producing an unusable zero-budget draft. - raise ValueError( + raise ResourceError( f"KV cache GPU budget ({total_budget / GB:.2f} GiB) is " f"insufficient after the combined fixed cost " f"({intercept_total / GB:.2f} GiB, e.g. SWA or mamba state) " @@ -4268,11 +4269,11 @@ def validate_kv_cache_compression_compatibility( """Reject unsupported KV-cache compression feature combinations.""" if config.algorithm == "quantization_for_cold_page": if config.quant == "nvfp4" and not is_sm_100f(): - raise RuntimeError( + raise UnsupportedError( "NVFP4 cold-page quantization requires an SM100-family device " "(SM100, SM103 or SM107).") elif config.algorithm == "triattention" and not is_sm_100f(): - raise RuntimeError( + raise UnsupportedError( "TriAttention requires an SM100-family device (SM100 or SM103).") if kv_cache_config.enable_block_reuse and not config.supports_block_reuse(): diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index e2ce6abc30e3..6618b6402c6a 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -14,6 +14,7 @@ import torch +from tensorrt_llm._errors import ConfigInputError, UnsupportedError from tensorrt_llm._torch.models.checkpoints.base_checkpoint_loader import ( AutoCheckpointMapper, BaseCheckpointLoader) from tensorrt_llm._torch.peft.lora.config import LoraConfig @@ -169,7 +170,7 @@ def _validate_and_adjust_mamba_snapshot_config(config: ModelConfig, or state_config.additional_snapshot_offsets_from_end) if (has_additional_snapshots and kv_cache_config.use_kv_cache_manager_v2 is not True): - raise ValueError( + raise ConfigInputError( "Mamba additional snapshot offsets require " "kv_cache_config.use_kv_cache_manager_v2=True after resolving " "the model configuration.") @@ -308,13 +309,14 @@ def validate_fp4_mla_config(model_config: ModelConfig, and model_config.quant_config.quant_mode.has_fp4_kv_cache()): return if not supports_fp4_mla_attention(model_config): - raise ValueError("Dense FP4 MLA requires the TRTLLM attention backend.") + raise UnsupportedError( + "Dense FP4 MLA requires the TRTLLM attention backend.") mapping = model_config.mapping if mapping.cp_size != 1 and not mapping.has_cp_helix(): - raise ValueError( + raise UnsupportedError( "FP4 MLA supports context parallelism only through Helix.") if llm_args.kv_cache_config.use_kv_cache_manager_v2 is False: - raise ValueError("FP4 MLA requires use_kv_cache_manager_v2=True.") + raise ConfigInputError("FP4 MLA requires use_kv_cache_manager_v2=True.") spec_config = model_config.spec_config if spec_config is not None and spec_config.spec_dec_mode.use_one_engine(): @@ -339,7 +341,7 @@ def validate_encoder_decoder_kv_cache_config(model_config: ModelConfig, """ if model_config.is_encoder_decoder: if kv_cache_config.cross_kv_cache_fraction is None: - raise ValueError( + raise ConfigInputError( "Encoder-decoder models require kv_cache_config.cross_kv_cache_fraction to be set." ) if (kv_cache_config.use_kv_cache_manager_v2 is True @@ -354,7 +356,7 @@ def validate_encoder_decoder_kv_cache_config(model_config: ModelConfig, return if kv_cache_config.cross_kv_cache_fraction is not None: - raise ValueError( + raise ConfigInputError( "kv_cache_config.cross_kv_cache_fraction should only be set for encoder-decoder models." ) @@ -366,22 +368,22 @@ def validate_encoder_decoder_tp_scope(model_config: ModelConfig) -> None: mapping = model_config.mapping if mapping.enable_attention_dp: - raise ValueError( + raise UnsupportedError( "Encoder-decoder models do not support attention DP yet. " "Set enable_attention_dp=False.") if mapping.cp_size > 1: - raise ValueError( + raise UnsupportedError( "Encoder-decoder models do not support context parallelism yet. " "Set context_parallel_size=1.") if mapping.pp_size > 1: - raise ValueError( + raise UnsupportedError( "Encoder-decoder models do not support pipeline parallelism yet. " "Set pipeline_parallel_size=1.") if mapping.tp_size > 1 and model_config.attn_backend != "TRTLLM": - raise ValueError( + raise UnsupportedError( "Encoder-decoder tensor parallelism currently supports " f"attn_backend='TRTLLM' only, but got " f"attn_backend='{model_config.attn_backend}'.") @@ -1189,7 +1191,7 @@ def load( locality_domain_policy = getattr(config, "locality_domain_policy", None) if (load_format == LoadFormat.GMS and getattr(locality_domain_policy, "enabled", False)): - raise ValueError( + raise ConfigInputError( "LoadFormat.GMS is incompatible with locality domain localized weights. " "GMS shares registered parameters, while locality domain execution " "releases those parameters after creating partition-local " @@ -1228,7 +1230,7 @@ def load( and checkpoint_loader.checkpoint_format != CheckpointFormat.MX) if requires_standard_hf_loading and not uses_standard_hf_loader: - raise ValueError( + raise UnsupportedError( "Host-resident model weights currently require the standard " "Hugging Face AUTO loader; DUMMY, VISION_ONLY, GMS, and MX " "loaders cannot materialize them") diff --git a/tensorrt_llm/_torch/pyexecutor/warmup_timer.py b/tensorrt_llm/_torch/pyexecutor/warmup_timer.py index 2fedebd6a557..75adbc8c1332 100644 --- a/tensorrt_llm/_torch/pyexecutor/warmup_timer.py +++ b/tensorrt_llm/_torch/pyexecutor/warmup_timer.py @@ -8,6 +8,7 @@ from collections.abc import Iterator from contextlib import contextmanager +from tensorrt_llm._startup_errors import mark_startup_stage from tensorrt_llm.logger import logger @@ -66,6 +67,9 @@ def phase( try: yield completed = True + except BaseException as exc: + mark_startup_stage(exc, f"warmup/{name}") + raise finally: elapsed = time.perf_counter() - start if record: diff --git a/tensorrt_llm/bench/benchmark/utils/general.py b/tensorrt_llm/bench/benchmark/utils/general.py index cf567b89a68b..d2a133a9be40 100755 --- a/tensorrt_llm/bench/benchmark/utils/general.py +++ b/tensorrt_llm/bench/benchmark/utils/general.py @@ -6,6 +6,7 @@ import yaml +from tensorrt_llm._errors import ConfigInputError from tensorrt_llm._torch.pyexecutor.model_loader import \ validate_and_set_kv_cache_quant from tensorrt_llm.bench.dataclasses.general import (DatasetMetadata, @@ -48,7 +49,7 @@ def get_settings(params: dict, dataset_metadata: DatasetMetadata, model: str, with open(extra_llm_api_options, 'r') as f: llm_args_dict = yaml.safe_load(f) if not isinstance(llm_args_dict, dict): - raise TypeError( + raise ConfigInputError( f"extra_llm_api_options must contain a YAML mapping, " f"got {type(llm_args_dict)}") kv_cache_config = llm_args_dict.get("kv_cache_config", { diff --git a/tensorrt_llm/bench/benchmark/visual_gen.py b/tensorrt_llm/bench/benchmark/visual_gen.py index bc29527ad865..5d77c456115d 100644 --- a/tensorrt_llm/bench/benchmark/visual_gen.py +++ b/tensorrt_llm/bench/benchmark/visual_gen.py @@ -40,6 +40,7 @@ ) from tensorrt_llm.bench.dataclasses.general import BenchmarkEnvironment from tensorrt_llm.logger import logger +from tensorrt_llm.usage.usage_lib import _mark_visual_gen def _parse_size(size_str: str) -> tuple[Optional[int], Optional[int]]: @@ -195,6 +196,9 @@ def visual_gen_command( metric_percentiles: str, ) -> None: """Benchmark VisualGen (image/video generation) models offline.""" + # VisualGen reports no startup error category or startup state. + _mark_visual_gen() + import yaml from tensorrt_llm.visual_gen import VisualGen, VisualGenParams diff --git a/tensorrt_llm/bench/tuning/heuristics.py b/tensorrt_llm/bench/tuning/heuristics.py index 967bb3b71b7d..31827468fb18 100755 --- a/tensorrt_llm/bench/tuning/heuristics.py +++ b/tensorrt_llm/bench/tuning/heuristics.py @@ -3,6 +3,7 @@ import torch +from tensorrt_llm._errors import ResourceError from tensorrt_llm._utils import str_dtype_to_torch from tensorrt_llm.bench.tuning.dataclasses import ( ModelConfig, @@ -15,6 +16,8 @@ from .utils import get_device_memory +_NOT_ENOUGH_GPU_MEMORY_HINT = "Use more GPUs (--tp / --pp), or a smaller or quantized checkpoint." + BYTES_PER_ELEM = { QuantAlgo.NO_QUANT: 2.0, QuantAlgo.FP8: 1.0, @@ -155,7 +158,7 @@ def calc_engine_setting( # Functional and performance if total_gpu_memory < engine_size: - raise RuntimeError( + raise ResourceError( f"The model requires at least: {engine_size:.2f} GB, " f"the total GPU memory of {total_gpu_memory:.2f} is insufficient.\n" "----------------------------------------------------------\n" @@ -172,12 +175,14 @@ def calc_engine_setting( f"Number of Tensor Parallel Shards: {tp_size}\n" f"Number of Pipeline Parallel Stages: {pp_size}\n" f"KV Cache GPU Memory Fraction: {kv_cache_gpu_mem_fraction}\n" - "----------------------------------------------------------\n" + "----------------------------------------------------------\n", + hint=_NOT_ENOUGH_GPU_MEMORY_HINT, ) if kv_cache_max_requests < 1: - raise RuntimeError( - "The amount of KV cache memory is insufficient to " - "run this model. Please try with more GPUs." + raise ResourceError( + "The amount of KV cache memory is insufficient to run this model. " + "Please try with more GPUs.", + hint=_NOT_ENOUGH_GPU_MEMORY_HINT, ) warning_gpu_count = pp_size if enable_attention_dp else n_gpus if cache_memory / warning_gpu_count < 10.0: diff --git a/tensorrt_llm/commands/_telemetry.py b/tensorrt_llm/commands/_telemetry.py index 81288359e5d2..c6ee379e650d 100644 --- a/tensorrt_llm/commands/_telemetry.py +++ b/tensorrt_llm/commands/_telemetry.py @@ -25,6 +25,7 @@ import yaml import tensorrt_llm.usage as usage +from tensorrt_llm._startup_errors import classify_startup_error, llm_initialized, log_startup_error from tensorrt_llm.usage.config import TelemetryConfig, UsageContext from tensorrt_llm.usage.schema import TerminationComponent @@ -202,6 +203,9 @@ def report(outcome: usage.TerminalOutcome) -> None: termination_kind = "clean" else: termination_kind = "exception" + cause = exc.__cause__ or (None if exc.__suppress_context__ else exc.__context__) + if cause is not None and not llm_initialized(): + usage.record_startup_error_category(classify_startup_error(cause).category) report( usage.TerminalOutcome( termination_kind=termination_kind, @@ -222,7 +226,10 @@ def report(outcome: usage.TerminalOutcome) -> None: ) ) raise - except Exception: + except Exception as exc: + if not llm_initialized(): + info = log_startup_error(exc, "Command failed") or classify_startup_error(exc) + usage.record_startup_error_category(info.category) signal_number = usage.get_observed_signal() report( usage.TerminalOutcome( diff --git a/tensorrt_llm/commands/eval.py b/tensorrt_llm/commands/eval.py index 4ed1efb6faea..2a01d0602ba4 100644 --- a/tensorrt_llm/commands/eval.py +++ b/tensorrt_llm/commands/eval.py @@ -20,6 +20,7 @@ import tensorrt_llm.profiler as profiler from .. import LLM as PyTorchLLM +from .._errors import ConfigInputError from ..evaluate import (AALCR, AIME2025, AIME2026, GSM8K, HLE, MMLU, MMMU, ArenaHard, CnnDailymail, CoVoST2, GPQADiamond, GPQAExtended, GPQAMain, GPQANemoSkills, GSM8KInferenceX, @@ -227,7 +228,7 @@ def main(ctx, model: str, tokenizer: Optional[str], if llm_args_extra_dict is None: llm_args_extra_dict = {} elif not isinstance(llm_args_extra_dict, dict): - raise ValueError("Configuration file root must be a mapping.") + raise ConfigInputError("Configuration file root must be a mapping.") apply_raw_config_telemetry_opt_out( llm_args_extra_dict, usage_context=_telemetry_config.UsageContext.CLI_EVAL, diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index d24c45b8bcde..4d4d137e6941 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -44,6 +44,7 @@ from tensorrt_llm import LLM as PyTorchLLM from tensorrt_llm import MultimodalEncoder +from tensorrt_llm._errors import ConfigInputError from tensorrt_llm._utils import mpi_rank, set_prometheus_multiproc_dir from tensorrt_llm.commands import _telemetry as _command_telemetry from tensorrt_llm.commands._config_overrides import (ConfigOverride, @@ -68,6 +69,7 @@ from tensorrt_llm.llmapi.mpi_session import find_free_ipc_addr from tensorrt_llm.llmapi.reasoning_parser import (ReasoningParserFactory, resolve_auto_reasoning_parser) +from tensorrt_llm.llmapi.startup_checks import no_visible_gpu_error from tensorrt_llm.logger import logger, severity_map from tensorrt_llm.mapping import CpType from tensorrt_llm.serve import OpenAIDisaggServer, OpenAIServer @@ -87,6 +89,7 @@ from tensorrt_llm.usage import (get_observed_signal, record_termination_observation, set_lifecycle_phase, set_usage_context) +from tensorrt_llm.usage.usage_lib import _mark_visual_gen if TYPE_CHECKING: # Type-only: the visual_gen tree is imported lazily inside the VisualGen @@ -291,7 +294,7 @@ def get_llm_args( if gpus_per_node is None: gpus_per_node = device_count() if gpus_per_node == 0: - raise ValueError("No GPU devices found on the node") + raise no_visible_gpu_error() # TODO: This manual cp_type conversion can be removed once cp_config # is refactored to a typed Pydantic model with enum coercion @@ -1617,6 +1620,10 @@ def serve( MODEL: model name or Hugging Face checkpoint path """ + is_visual_gen = enable_visual_gen or visual_gen_args is not None + if is_visual_gen: + # VisualGen reports no startup error category or startup state. + _mark_visual_gen() logger.set_level(log_level) if enable_sleep_mode and backend != "pytorch": raise click.UsageError( @@ -1700,8 +1707,9 @@ def serve( explicit_cli_keys = collect_explicit_cli_keys( exclude=("extra_llm_api_options", "config", "config_overrides")) - is_visual_gen = (enable_visual_gen or visual_gen_args is not None - or get_is_diffusion_only_model(model)) + if not is_visual_gen and get_is_diffusion_only_model(model): + is_visual_gen = True + _mark_visual_gen() # Honor YAML telemetry opt-out before --set parsing can raise an error. raw_llm_args_extra_dict = {} if not is_visual_gen and extra_llm_api_options is not None: @@ -1710,7 +1718,7 @@ def serve( if raw_llm_args_extra_dict is None: raw_llm_args_extra_dict = {} elif not isinstance(raw_llm_args_extra_dict, dict): - raise ValueError("Configuration file root must be a mapping.") + raise ConfigInputError("Configuration file root must be a mapping.") if not is_visual_gen: _command_telemetry.apply_raw_config_telemetry_opt_out( raw_llm_args_extra_dict, @@ -1879,13 +1887,13 @@ def _serve_llm(): } for name, value in unsupported_args.items(): if value is not None: - raise ValueError( + raise ConfigInputError( f"Argument '{name}' is not supported when running in gRPC mode. " f"The gRPC server is designed for use with external routers that handle " f"these features (e.g., tool parsing, chat templates).") if grpc_protocol == "smg": if find_spec("smg_grpc_proto") is None: - raise ValueError( + raise ConfigInputError( "gRPC serving with the SMG protocol requires the optional " "'smg-grpc-proto' package. Install it with: " 'pip install "tensorrt_llm[grpc-smg]"') @@ -2113,7 +2121,7 @@ def serve_encoder( if encoder_args_extra_dict is None: encoder_args_extra_dict = {} elif not isinstance(encoder_args_extra_dict, dict): - raise ValueError("Configuration file root must be a mapping.") + raise ConfigInputError("Configuration file root must be a mapping.") _command_telemetry.apply_raw_config_telemetry_opt_out( encoder_args_extra_dict, usage_context=_telemetry_config.UsageContext.CLI_SERVE, @@ -2246,7 +2254,7 @@ def serve_embedding( if extra_dict is None: extra_dict = {} elif not isinstance(extra_dict, dict): - raise ValueError("Configuration file root must be a mapping.") + raise ConfigInputError("Configuration file root must be a mapping.") _command_telemetry.apply_raw_config_telemetry_opt_out( extra_dict, usage_context=_telemetry_config.UsageContext.CLI_SERVE, @@ -3153,14 +3161,31 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, ) +def _visual_gen_requested(args: Sequence[str]) -> bool: + """Detect the explicit VisualGen flags of ``serve`` before Click parses them.""" + for arg in args: + if arg == "--": + break + if arg.split("=", 1)[0] in ("--enable_visual_gen", "--visual_gen_args"): + return True + return False + + class DefaultGroup(_command_telemetry.TelemetryGroup): """Custom Click group to allow default command behavior""" def resolve_command(self, ctx, args): # If the first argument is not a recognized subcommand, assume "serve" if args and args[0] not in self.commands: - return "serve", self.commands["serve"], args - return super().resolve_command(ctx, args) + result = "serve", self.commands["serve"], args + else: + result = super().resolve_command(ctx, args) + if result[0] == "serve" and _visual_gen_requested(result[2]): + # Marked before Click parses the serve options, so a VisualGen + # launch rejected by Click also reports no startup error category + # or startup state. + _mark_visual_gen() + return result main = DefaultGroup( diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index c4183c39799a..d82a4f809e8a 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -43,6 +43,7 @@ configure_moe_launch_queues, uses_collective_shutdown) from ..llmapi.llm_args import BaseLlmArgs, ExecutorMemoryType +from ..llmapi.startup_checks import no_visible_gpu_error from ..llmapi.tokenizer import TokenizerBase from ..llmapi.tracer import global_tracer from ..llmapi.utils import _SyncQueue, configure_cpu_affinity, logger_debug @@ -164,7 +165,12 @@ def resource_governor_queue(self): return self._resource_governor_queue def _configure_device(self): - device_id = self.global_rank % torch.cuda.device_count() + # Ranks launched on other nodes (mpirun, Slurm, trtllm-llmapi-launch) + # are not covered by the GPU check in the LLM's own process. + device_count = torch.cuda.device_count() + if device_count == 0: + raise no_visible_gpu_error() + device_id = self.global_rank % device_count # Fail before loading the model: missing optimized kernels for this GPU otherwise # surface much later as unrelated errors such as running out of memory. check_cuda_architecture_supported(device_id) diff --git a/tensorrt_llm/executor/executor.py b/tensorrt_llm/executor/executor.py index 0519899987a3..9946d246a6c2 100644 --- a/tensorrt_llm/executor/executor.py +++ b/tensorrt_llm/executor/executor.py @@ -24,6 +24,7 @@ from tensorrt_llm.kv_hints import KvHintsInput from tensorrt_llm.logger import logger, set_level +from .._errors import ConfigInputError from .._utils import mpi_world_size from ..conversation_params import ConversationParams from ..disaggregated_params import DisaggregatedParams @@ -581,10 +582,12 @@ def create( world_size = mpi_world_size() if world_size > 1 and world_size < model_world_size: - raise RuntimeError( + raise ConfigInputError( "Cannot instantiate an executor requiring " f"{model_world_size} ranks, while currently running " - f"on {world_size} ranks.") + f"on {world_size} ranks. Launch tensor_parallel_size x " + "pipeline_parallel_size x context_parallel_size MPI ranks " + "(srun -n / mpirun -np), or lower the parallel sizes.") postproc_worker_config = postproc_worker_config or PostprocWorkerConfig( ) diff --git a/tensorrt_llm/executor/postprocessor_hook.py b/tensorrt_llm/executor/postprocessor_hook.py index 953c4362189c..be04a7d33a08 100644 --- a/tensorrt_llm/executor/postprocessor_hook.py +++ b/tensorrt_llm/executor/postprocessor_hook.py @@ -18,6 +18,8 @@ import logging from typing import List, Optional, Protocol, runtime_checkable +from .._errors import ConfigInputError + __all__ = [ "PostProcessorHookAction", "PostProcessorHookChunk", @@ -50,13 +52,13 @@ def load_post_processor_hook(import_path: str) -> "PostProcessorHook": hook_class = getattr(module, class_name) hook = hook_class() except (ValueError, ImportError, AttributeError, TypeError) as e: - raise ValueError( + raise ConfigInputError( f"Failed to load post-processor hook '{import_path}': {e}. " "Expected format: 'module.path.ClassName' resolving to a " "no-arg-constructible callable class." ) from e if not callable(hook): - raise ValueError( + raise ConfigInputError( f"Failed to load post-processor hook '{import_path}': resolved " f"object is not callable (got {type(hook).__name__}); expected an " "instance implementing __call__(chunk)." diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index 3e0f3f919ecc..5a3d7c3fdcde 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -31,7 +31,12 @@ import zmq import zmq.asyncio +from tensorrt_llm._errors import RuntimeInitError from tensorrt_llm._startup import _StartupTimer +from tensorrt_llm._startup_errors import (StartupErrorInfo, + is_worker_startup_error, + log_startup_error, + wrap_worker_startup_error) from tensorrt_llm.logger import logger from .._utils import customized_gc_thresholds, mpi_rank, nvtx_range_debug @@ -971,6 +976,10 @@ def _fail_initialization(self, abort itself raises, so the engine-dead bookkeeping (and the original initialization error) cannot be lost to a teardown failure. """ + # The abort can end this process (MPI_Abort) before the error reaches + # the LLM boundary, so the classified hint is logged first. + error.__cause__ = cause + log_startup_error(error, "LLM initialization failed") root_cause = cause if cause is not None else error try: self._abort_owned_session(root_cause) @@ -993,9 +1002,10 @@ def _detect_worker_death_during_init(self) -> Optional[BaseException]: """ dead_worker = self._worker_process_monitor.find_dead_worker() if dead_worker is not None: - return RuntimeError("MPI worker rank " - f"{dead_worker.rank} (pid {dead_worker.pid}) " - "exited unexpectedly") + return RuntimeInitError( + f"MPI worker rank {dead_worker.rank} (pid {dead_worker.pid}) " + "exited unexpectedly", + reason="worker_died") for fut in self.mpi_futures: if fut.done(): # exception() raises CancelledError on a cancelled future, @@ -1003,7 +1013,8 @@ def _detect_worker_death_during_init(self) -> Optional[BaseException]: # teardown; a cancelled worker future still means this rank # can never come up, so report it as a death instead. exc = fut.exception() if not fut.cancelled() else None - return exc or RuntimeError("MPI worker exited unexpectedly") + return exc or RuntimeInitError("MPI worker exited unexpectedly", + reason="worker_died") check = getattr(self.mpi_session, "check_worker_error", None) if check is None: return None @@ -1071,9 +1082,13 @@ def mpi_done_callback(future: concurrent.futures.Future): if ready_signal != GenerationExecutorProxy.READY_SIGNAL: logger.error( f"Executor worker initialization error: {error_trace}") + # The worker classified the error before pickling dropped its + # cause chain; carry that onto the error raised here. + info = status[2] if len(status) > 2 and isinstance( + status[2], StartupErrorInfo) else None self._fail_initialization( - RuntimeError("Executor worker returned error"), - ready_signal) + wrap_worker_startup_error("Executor worker returned error", + info), ready_signal) # Register the fast-death callback only after the world reported # ready: on an already-completed future, add_done_callback() runs the @@ -1115,13 +1130,17 @@ def _wait_for_executor_workers_ready(self) -> tuple: death = self._detect_worker_death_during_init() if death is not None: + # A non-leader rank that fails here has no status queue, so its + # peers stay blocked in the init collective still holding their + # share of the weights. Raising alone leaks them until job end + # and makes the next blocking shutdown() hang instead of + # reporting this failure. + if is_worker_startup_error(death): + # The rank raised its classified error, which already names + # the rank, stage and root cause. + self._fail_initialization(death) message = f"Executor worker died during initialization: {death}" logger.error(message) - # A non-leader rank that fails here returns without notifying - # anyone, so its peers stay blocked in the init collective - # still holding their share of the weights. Raising alone - # leaks them until job end and makes the next blocking - # shutdown() hang instead of reporting this failure. self._fail_initialization(RuntimeError(message), death) self._handle_background_error() diff --git a/tensorrt_llm/executor/worker.py b/tensorrt_llm/executor/worker.py index 9b1ef7f031e8..d9aaec5af59e 100644 --- a/tensorrt_llm/executor/worker.py +++ b/tensorrt_llm/executor/worker.py @@ -28,6 +28,7 @@ from tensorrt_llm.logger import logger +from .._startup_errors import classify_startup_error, wrap_worker_startup_error from .._utils import mpi_comm, mpi_rank, print_all_stacks from ..llmapi._load_balance_env import (LAUNCH_QUEUES_ENV, configure_moe_launch_queues, @@ -606,14 +607,24 @@ def notify_proxy_threads_to_quit(): rpc_addr=rpc_addr, hmac_key=hmac_key) except Exception as e: - logger.error(f"Failed to initialize executor on rank {mpi_rank()}: {e}") + # Classify here: the cause chain and traceback are lost once the + # exception is pickled to the proxy. + info = classify_startup_error(e, rank=mpi_rank()) + stage = f" during {info.stage}" if info.stage else "" + logger.error(f"Failed to initialize executor on rank {mpi_rank()}" + f"{stage}: {info.summary}") logger.error(traceback.format_exc()) logger_debug(f"error: {traceback.format_exc()}", "red") - if is_leader: - # Send error message with confirmation - error_msg = (e, traceback.format_exc()) - if not worker_init_status_queue.notify_with_retry(error_msg): - logger.error("Failed to deliver error message to proxy") + if not is_leader: + # Only the leader has a status queue. The proxy sees a non-leader's + # failure through the rank's MPI future, or through the launcher, + # which forwards only the type and message of a raised exception. + raise wrap_worker_startup_error("Executor worker returned error", + info) from None + # Send error message with confirmation + error_msg = (e, traceback.format_exc(), info) + if not worker_init_status_queue.notify_with_retry(error_msg): + logger.error("Failed to deliver error message to proxy") return # Optionally disable GC (default: not disabled) diff --git a/tensorrt_llm/inputs/registry.py b/tensorrt_llm/inputs/registry.py index e21faaf3ccb6..71cfa6cefd23 100644 --- a/tensorrt_llm/inputs/registry.py +++ b/tensorrt_llm/inputs/registry.py @@ -34,6 +34,7 @@ from tensorrt_llm.checkpoint_format import CheckpointFormat +from .._errors import ConfigInputError, UnsupportedError from ..logger import logger from ..sampling_params import SamplingParams from .content_format import ContentFormat @@ -1224,7 +1225,8 @@ def create_input_processor( model_cls, _ = get_model_architecture(config) input_processor_cls = INPUT_PROCESSOR_REGISTRY._input_processors_cls_by_model_type \ .get(model_cls) - except RuntimeError: # unregistered model + except (RuntimeError, ConfigInputError, + UnsupportedError): # unregistered model logger.info("Unregistered model, using DefaultInputProcessor") input_processor_cls = None if input_processor_cls is not None: diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 195a490e1507..bedc8eb0d251 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -38,6 +38,9 @@ from tensorrt_llm.llmapi import tracing from tensorrt_llm.metrics.enums import MetricNames +from .._errors import ConfigInputError +from .._startup_errors import (classify_startup_error, log_startup_error, + mark_llm_initialized) from .._utils import nvtx_range_debug from ..bindings import steady_clock_now from ..conversation_params import ConversationParams @@ -65,11 +68,14 @@ validate_token_encoder_bucket_config) from .llm_utils import CachedModelLoader, KvCacheRetentionConfig, ModelLoader from .mpi_session import MpiPoolSession, external_mpi_comm_available +from .startup_checks import (check_architecture_registered, + check_quantization_compute_capability, + check_visible_gpus) from .thinking_budget import add_thinking_budget_logits_processor from .tokenizer import TokenizerBase # TODO[chunweiy]: move the following symbols back to utils scope, and remove the following import -from .utils import (append_docstring, exception_handler, get_device_count, - logger_debug, set_api_status) +from .utils import (append_docstring, exception_handler, logger_debug, + set_api_status) _RUNTIME_MEMORY_TYPES = tuple( memory_type for memory_type in ExecutorMemoryType @@ -414,7 +420,7 @@ def __init__(self, kwargs["orchestrator_type"] = "ray" else: - raise ValueError( + raise ConfigInputError( f"Unknown backend: {backend!r}. Supported backends are " "'pytorch'.") @@ -487,11 +493,14 @@ def __init__(self, self._is_attached_frontend = os.getenv( "TLLM_EXECUTOR_ATTACH_INFO") is not None if self.args.parallel_config.is_multi_gpu and not self._is_attached_frontend: - if os.getenv("RAY_LOCAL_WORLD_SIZE") is None and get_device_count( - ) < self.args.parallel_config.world_size_per_node: - raise RuntimeError( - f"Only {get_device_count()} GPUs are available, but {self.args.parallel_config.world_size} are required." - ) + if os.getenv("RAY_LOCAL_WORLD_SIZE") is None: + # An LLM that creates its own MPI pool spawns every rank on this + # node; Ray and external launches may spread them over nodes. + check_visible_gpus( + self.args.parallel_config, + all_ranks_local=self.args.orchestrator_type != "ray" + and self.mpi_session is None + and not get_spawn_proxy_process_env()) logger.info( f'start MpiSession with {self.args.parallel_config.world_size} workers' @@ -1898,10 +1907,14 @@ def __init__(self, tokenizer_revision, backend=backend, **kwargs) - except Exception: + except Exception as exc: + info = log_startup_error(exc, "LLM initialization failed") if self._usage_attempt_tracked and _usage is not None: - _usage.record_llm_initialization_failure() + # An already logged failure (e.g. by the proxy) classifies the same. + _usage.record_llm_initialization_failure( + (info or classify_startup_error(exc)).category) raise + mark_llm_initialized() try: if not self._usage_attempt_tracked and _usage is not None: @@ -2110,6 +2123,10 @@ def _build_model(self): self._generation_config = self._try_load_generation_config() self._generation_config_explicit_values = self._try_load_generation_config_explicit_values( ) + # The checkpoint's quantization config is merged by now; fail before + # any worker loads weights. + check_quantization_compute_capability(self.args) + check_architecture_registered(self.args, self._hf_model_config) # Multimodal special handling: # 1. Default load_tokenizer may fail because MM has different tokenizer configuration. Hence we initialize it inside input processor @@ -2194,7 +2211,7 @@ def _validate_args_for_torch_backend(self, kwargs: dict) -> None: unsupported_args = [key for key in kwargs if key not in accepted_keys] if unsupported_args: - raise ValueError( + raise ConfigInputError( f"The following arguments are not supported by the PyTorch backend: {unsupported_args}." ) diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 6a0c263fe6ce..ce443721f1b9 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -49,6 +49,7 @@ LoraConfig, get_default_trtllm_modules_to_hf_modules) from tensorrt_llm.bindings.internal.batch_manager import LinearCacheType +from .._errors import ConfigInputError from .._utils import (_str_to_torch_dtype_dict, is_sm_100f, mpi_rank, prefer_pinned) @@ -6617,7 +6618,7 @@ def from_yaml(cls, yaml_path: Union[str, Path]): if config_dict is None: config_dict = {} elif not isinstance(config_dict, dict): - raise ValueError("Configuration file root must be a mapping.") + raise ConfigInputError("Configuration file root must be a mapping.") return cls(**config_dict) @field_validator("gpus_per_node", mode='before') @@ -8869,7 +8870,7 @@ def update_llm_args_with_extra_options( if llm_args_dict is None: llm_args_dict = {} elif not isinstance(llm_args_dict, dict): - raise ValueError("Configuration file root must be a mapping.") + raise ConfigInputError("Configuration file root must be a mapping.") llm_args = update_llm_args_with_extra_dict( llm_args, llm_args_dict, diff --git a/tensorrt_llm/llmapi/llm_utils.py b/tensorrt_llm/llmapi/llm_utils.py index 7475548b170b..09fda5f35066 100644 --- a/tensorrt_llm/llmapi/llm_utils.py +++ b/tensorrt_llm/llmapi/llm_utils.py @@ -8,6 +8,7 @@ import transformers +from .._errors import ConfigInputError, ModelLoadError, UnsupportedError from .._utils import global_mpi_rank, local_mpi_rank, mpi_rank # yapf: disable from ..bindings.executor import (CapacitySchedulerPolicy, ContextChunkingPolicy, @@ -74,7 +75,8 @@ def _apply_modelopt_quant_config(self, hf_quant_config: Dict[str, Any], quant_config = self.llm_args.quant_config hf_quant_algo = hf_quant_config.pop("quant_algo", None) if hf_quant_algo is None: - raise ValueError("Pre-quantized checkpoint must have quant_algo.") + raise ModelLoadError( + "Pre-quantized checkpoint must have quant_algo.") reject_checkpoint_nvfp4_plus(hf_quant_algo, "The checkpoint quantization config") hf_quant_algo = QuantAlgo(hf_quant_algo) @@ -83,9 +85,10 @@ def _apply_modelopt_quant_config(self, hf_quant_config: Dict[str, Any], f"Setting quant_algo={hf_quant_algo} from HF quant config.") quant_config.quant_algo = hf_quant_algo elif quant_config.quant_algo != hf_quant_algo: - raise ValueError( - f"Specified quant_algo={quant_config.quant_algo}, conflicting with quant_algo={hf_quant_algo} from HF quant config." - ) + raise ConfigInputError( + f"Specified quant_algo={quant_config.quant_algo}, conflicting with quant_algo={hf_quant_algo} from HF quant config.", + hint="Remove quant_config.quant_algo; the quantization of a " + "pre-quantized checkpoint is read from the checkpoint.") hf_kv_cache_quant_algo = hf_quant_config.pop("kv_cache_quant_algo", None) @@ -103,14 +106,15 @@ def _apply_modelopt_quant_config(self, hf_quant_config: Dict[str, Any], ) quant_config.kv_cache_quant_algo = hf_kv_cache_quant_algo elif quant_config.kv_cache_quant_algo != hf_kv_cache_quant_algo: - raise ValueError( - f"Specified kv_cache_quant_algo={quant_config.kv_cache_quant_algo}, conflicting with kv_cache_quant_algo={hf_kv_cache_quant_algo} from HF quant config." - ) + raise ConfigInputError( + f"Specified kv_cache_quant_algo={quant_config.kv_cache_quant_algo}, conflicting with kv_cache_quant_algo={hf_kv_cache_quant_algo} from HF quant config.", + hint="Set the KV cache format with kv_cache_config.dtype, " + "which overrides the checkpoint's.") else: if quant_config.kv_cache_quant_algo not in [ None, QuantAlgo.FP8, QuantAlgo.NVFP4 ]: - raise ValueError( + raise UnsupportedError( f"Only kv_cache_quant_algo={QuantAlgo.FP8} or {QuantAlgo.NVFP4} is allowed for pre-quantized checkpoint, got {quant_config.kv_cache_quant_algo}." ) @@ -173,7 +177,7 @@ def _update_from_hf_quant_config(self) -> bool: "falling back to config.json or model_kwargs for global " "quantization.") else: - raise ValueError( + raise ModelLoadError( "Pre-quantized checkpoint must have quant_algo.") else: self._apply_modelopt_quant_config(normalized, @@ -201,7 +205,7 @@ def _update_from_hf_quant_config(self) -> bool: ) if requires_global_quant_config_fallback and hf_quant_config is None: - raise ValueError( + raise ModelLoadError( "hf_quant_config.json does not set a global quant_algo and no " "quantization_config fallback was found.") diff --git a/tensorrt_llm/llmapi/startup_checks.py b/tensorrt_llm/llmapi/startup_checks.py new file mode 100644 index 000000000000..867f3c100add --- /dev/null +++ b/tensorrt_llm/llmapi/startup_checks.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Cheap, deterministic checks that fail before any weights are loaded. + +Only certain lower bounds are checked here. Model-, kernel- or feature-specific +compatibility is left to the existing errors raised where the support is +implemented. +""" + +import math +import os +from typing import Optional + +import torch +from transformers import PretrainedConfig + +from tensorrt_llm._errors import _DOCS_SITE, ResourceError, UnsupportedError +from tensorrt_llm._utils import get_sm_version +from tensorrt_llm.llmapi.llm_args import TorchLlmArgs, _ParallelConfig +from tensorrt_llm.llmapi.utils import get_device_count +from tensorrt_llm.quantization.mode import QuantAlgo + +# Lowest architecture marked as supported in the "Hardware Support Matrix" of +# docs/source/features/quantization.md. Formats missing from the matrix are not +# checked, and neither are KV cache formats: attention kernels dequantize the KV +# cache before the MMA, so its support depends on the kernel, not on a lower bound. +_MIN_SM_BY_QUANT_ALGO = { + QuantAlgo.FP8: 89, + QuantAlgo.FP8_BLOCK_SCALES: 90, + QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN: 90, + QuantAlgo.W4A8_AWQ: 89, + QuantAlgo.NVFP4: 100, + QuantAlgo.W4A8_MXFP4_MXFP8: 100, +} +# The opt-in Marlin NVFP4 GEMM runs NVFP4 linear layers on Ada and Hopper. +_MIN_SM_NVFP4_MARLIN = 89 + + +def _visible_devices_env() -> str: + return ", ".join( + f"{name}={os.environ.get(name, '')}" + for name in ("CUDA_VISIBLE_DEVICES", "NVIDIA_VISIBLE_DEVICES") + ) + + +def _nvml_facts() -> str: + try: + import pynvml + + pynvml.nvmlInit() + try: + count = pynvml.nvmlDeviceGetCount() + driver = pynvml.nvmlSystemGetDriverVersion() + cuda = pynvml.nvmlSystemGetCudaDriverVersion() + finally: + pynvml.nvmlShutdown() + except Exception as e: # noqa: BLE001 - facts are best effort + # nvmlInit fails without the driver libraries, e.g. in a container started without GPUs. + return f"NVML unavailable ({type(e).__name__})" + if isinstance(driver, bytes): + driver = driver.decode() + return ( + f"NVML sees {count} GPU(s), driver {driver} supports CUDA up to " + f"{cuda // 1000}.{cuda % 1000 // 10}" + ) + + +def no_visible_gpu_error() -> ResourceError: + """Build the error for "PyTorch sees no GPU", listing the observed facts and likely causes.""" + cuda = torch.version.cuda or "unknown" + causes = ( + "Possible causes: (1) the container was started without GPUs (docker --gpus, or " + "nvidia.com/gpu in Kubernetes); (2) CUDA_VISIBLE_DEVICES hides all GPUs; " + f"(3) the NVIDIA driver is too old for CUDA {cuda}: upgrade it, or install the " + "matching cuda-compat package." + ) + return ResourceError( + "No GPU is visible to PyTorch (torch.cuda.device_count() == 0). " + f"{_visible_devices_env()}; {_nvml_facts()}; this build uses CUDA {cuda}.\n{causes}", + reason="no_gpu", + hint=causes, + ) + + +def check_visible_gpus(parallel_config: _ParallelConfig, all_ranks_local: bool) -> None: + """Fail when this node shows no GPU, or fewer GPUs than its share of the world needs. + + Args: + parallel_config: The LLM's parallel configuration. + all_ranks_local: Whether every rank is spawned on this node, as the LLM's own + MPI pool does. Otherwise (mpirun / Slurm launches) the world is assumed to + span ``ceil(world_size / gpus_per_node)`` nodes. + """ + visible = get_device_count() + if visible == 0: + raise no_visible_gpu_error() + world_size = parallel_config.world_size + if all_ranks_local: + needed, placement = world_size, "on this node" + else: + nodes = math.ceil(world_size / parallel_config.gpus_per_node) + needed, placement = parallel_config.world_size_per_node, f"over {nodes} node(s)" + if visible >= needed: + return + raise ResourceError( + f"Only {visible} GPUs are available, but {needed} are required on this node: " + "tensor_parallel_size x pipeline_parallel_size x context_parallel_size = " + f"{parallel_config.tp_size} x {parallel_config.pp_size} x " + f"{parallel_config.cp_size} = {world_size} ranks {placement} " + f"({_visible_devices_env()}).", + hint=( + "Make more GPUs visible (CUDA_VISIBLE_DEVICES, docker --gpus) " + "or lower the parallel sizes." + ), + ) + + +def check_quantization_compute_capability(llm_args: TorchLlmArgs) -> None: + """Fail when GPU 0 is older than the lowest architecture supporting the quantization format. + + Must run after the checkpoint's quantization config has been merged into ``llm_args``. + + Args: + llm_args: The resolved LLM arguments. + """ + sm = get_sm_version() + if sm <= 0: + return + algo = llm_args.quant_config.quant_algo + min_sm = _MIN_SM_BY_QUANT_ALGO.get(algo) + if algo == QuantAlgo.NVFP4 and "marlin" in llm_args.nvfp4_gemm_config.allowed_backends: + min_sm = _MIN_SM_NVFP4_MARLIN + if min_sm is None or sm >= min_sm: + return + message = ( + f"Quantization {algo} requires SM{min_sm} or newer, but GPU 0 " + f"({torch.cuda.get_device_name(0)}) is SM{sm}." + ) + if algo == QuantAlgo.NVFP4 and sm >= _MIN_SM_NVFP4_MARLIN: + message += ( + " On Ada and Hopper GPUs, NVFP4 linear layers can use the opt-in Marlin " + "backend: add 'marlin' to nvfp4_gemm_config.allowed_backends." + ) + raise UnsupportedError( + message, + hint=( + "Use a checkpoint quantized for this GPU, or a newer GPU. See the hardware " + f"support matrix at {_DOCS_SITE}/features/quantization.html." + ), + ) + + +def check_architecture_registered( + llm_args: TorchLlmArgs, pretrained_config: Optional[PretrainedConfig] +) -> None: + """Fail when the Hugging Face checkpoint's architecture has no registered implementation. + + `model_kwargs["architectures"]`, when set, is checked instead of the checkpoint's, + since it overrides the config the engine builds from. A custom `checkpoint_loader` + is not checked: its `load_config()` decides the architecture. + + Args: + llm_args: The resolved LLM arguments. + pretrained_config: The checkpoint's Hugging Face config, or None if it was not loaded. + """ + if llm_args.checkpoint_format not in (None, "HF") or llm_args.checkpoint_loader is not None: + return + if llm_args.mm_encoder_only or llm_args.encode_only: + return + architectures = (llm_args.model_kwargs or {}).get("architectures") or ( + getattr(pretrained_config, "architectures", None) or [] + ) + if not architectures: + return + # The static index lists every built-in architecture without importing the model + # zoo; MODEL_CLASS_MAPPING adds custom models registered in this process. + from tensorrt_llm._torch.models._arch_index import MODEL_ARCH_TO_MODULE + from tensorrt_llm._torch.models.modeling_utils import MODEL_CLASS_MAPPING + + if any(arch in MODEL_CLASS_MAPPING or arch in MODEL_ARCH_TO_MODULE for arch in architectures): + return + raise UnsupportedError( + f"Unknown model architecture: {architectures[0]}. TensorRT-LLM has no " + f"implementation registered for the checkpoint's architectures {architectures}.", + hint=( + f"See the supported models at {_DOCS_SITE}/models/supported-models.html. A custom " + "model must be registered with register_auto_model before the LLM is created." + ), + ) diff --git a/tensorrt_llm/models/quant_config_utils.py b/tensorrt_llm/models/quant_config_utils.py index 58340ce69122..15e5ada02321 100644 --- a/tensorrt_llm/models/quant_config_utils.py +++ b/tensorrt_llm/models/quant_config_utils.py @@ -15,6 +15,7 @@ from typing import Any, Mapping +from tensorrt_llm._errors import ConfigInputError from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization.mode import QuantAlgo @@ -49,7 +50,7 @@ def update_quant_config_from_compressed_tensors( if quant_config.kv_cache_quant_algo in (None, QuantAlgo.FP8): quant_config.kv_cache_quant_algo = QuantAlgo.FP8 else: - raise ValueError( + raise ConfigInputError( f"Specified kv_cache_quant_algo={quant_config.kv_cache_quant_algo}, " "conflicting with FP8 KV cache from HF quant config." ) diff --git a/tensorrt_llm/quantization/mode.py b/tensorrt_llm/quantization/mode.py index fdc74f1f3cfb..9bb7788ffa0f 100644 --- a/tensorrt_llm/quantization/mode.py +++ b/tensorrt_llm/quantization/mode.py @@ -17,6 +17,7 @@ from strenum import StrEnum +from .._errors import UnsupportedError from .._utils import BaseEnumMeta @@ -51,6 +52,15 @@ class QuantAlgo(StrEnum, metaclass=BaseEnumMeta): NVFP4_PLUS = auto() NO_QUANT = auto() + @classmethod + def _missing_(cls, value): + # An unknown name, e.g. from a checkpoint quantized with a newer format. + # UnsupportedError is a ValueError, which Enum raises by default. + raise UnsupportedError( + f"{value!r} is not a valid {cls.__qualname__}", + hint="Upgrade TensorRT-LLM, or use a quantization format that this " + "version supports.") + QUANT_ALGO_LIST = list(set(QuantAlgo) - {QuantAlgo.INT8}) KV_CACHE_QUANT_ALGO_LIST = [QuantAlgo.FP8, QuantAlgo.INT8, QuantAlgo.NVFP4] diff --git a/tensorrt_llm/usage/__init__.py b/tensorrt_llm/usage/__init__.py index 6c0c77b1ac16..9364638bbf04 100644 --- a/tensorrt_llm/usage/__init__.py +++ b/tensorrt_llm/usage/__init__.py @@ -41,6 +41,7 @@ record_llm_initialized = _usage_lib.record_llm_initialized record_llm_shutdown = _usage_lib.record_llm_shutdown record_observed_signal = _usage_lib.record_observed_signal +record_startup_error_category = _usage_lib.record_startup_error_category record_termination_observation = _usage_lib.record_termination_observation set_lifecycle_phase = _usage_lib.set_lifecycle_phase set_usage_context = _usage_lib.set_usage_context @@ -60,6 +61,7 @@ "record_llm_initialized", "record_llm_shutdown", "record_observed_signal", + "record_startup_error_category", "record_termination_observation", "report_exit", "report_usage", diff --git a/tensorrt_llm/usage/schema.py b/tensorrt_llm/usage/schema.py index a097fbdcf11e..3c705d15cedd 100644 --- a/tensorrt_llm/usage/schema.py +++ b/tensorrt_llm/usage/schema.py @@ -39,7 +39,7 @@ CLIENT_ID = "616561816355034" EVENT_PROTOCOL = "1.6" -EVENT_SCHEMA_VER = "0.7" +EVENT_SCHEMA_VER = "0.8" EVENT_SYS_VER = "trtllm-telemetry/1.0" CLIENT_TYPE = "Native" CLIENT_VARIANT = "Release" @@ -100,6 +100,11 @@ class _LlmCounterSnapshot(BaseModel): model_config = {"populate_by_name": True} +# Bucket lower bounds of the local consecutive startup failure count. +PriorStartupFailures = Literal[0, 1, 2, 3, 10, 30, 100, 300, 1000] +StartupStateStatus = Literal["", "persisted", "new", "unavailable"] + + class TrtllmInitialReport(_LlmCounterSnapshot): """TRT-LLM initial report event parameters. @@ -187,6 +192,24 @@ class TrtllmInitialReport(_LlmCounterSnapshot): description="Identifier used to correlate a disaggregated deployment.", ) + # Local consecutive startup failure count of this launch + prior_startup_failures: PriorStartupFailures = Field( + default=0, + alias="priorStartupFailures", + description=( + "Consecutive failed starts of the same launch before this session, as a " + "bucket lower bound." + ), + ) + startup_state_status: StartupStateStatus = Field( + default="", + alias="startupStateStatus", + description=( + "persisted if the local startup state was read, new if it was missing or " + "corrupt, unavailable if it could not be read or written; empty if not consulted." + ), + ) + # Legacy feature summary and sanitized opt-in LLM API config capture. features_json: str = Field(default="{}", alias="featuresJson") llm_api_config_json: str = Field(default="{}", alias="llmApiConfigJson") @@ -255,6 +278,17 @@ class TrtllmHeartbeat(_LlmCounterSnapshot): "unknown", ] ReportingSource = Literal["self", "supervisor", "executor_proxy"] +# Category of a startup failure, from ``tensorrt_llm._errors.ERROR_CATEGORIES``; +# "" when the session did not fail during startup. +ErrorCategory = Literal[ + "", + "config_input", + "resource", + "unsupported", + "model_load", + "runtime_init", + "other", +] class TrtllmExitReport(_LlmCounterSnapshot): @@ -322,6 +356,30 @@ class TrtllmExitReport(_LlmCounterSnapshot): alias="deploymentId", description="Identifier used to correlate a disaggregated deployment.", ) + error_category: ErrorCategory = Field( + default="", + alias="errorCategory", + description=( + "Category of the startup failure that ended the session; empty if the " + "session did not fail during startup." + ), + ) + prior_startup_failures: PriorStartupFailures = Field( + default=0, + alias="priorStartupFailures", + description=( + "Consecutive failed starts of the same launch before this session, as a " + "bucket lower bound." + ), + ) + startup_state_status: StartupStateStatus = Field( + default="", + alias="startupStateStatus", + description=( + "persisted if the local startup state was read, new if it was missing or " + "corrupt, unavailable if it could not be read or written; empty if not consulted." + ), + ) model_config = {"populate_by_name": True} diff --git a/tensorrt_llm/usage/schemas/README.md b/tensorrt_llm/usage/schemas/README.md index 715da3f9eb12..f47f1bc7266d 100644 --- a/tensorrt_llm/usage/schemas/README.md +++ b/tensorrt_llm/usage/schemas/README.md @@ -1,13 +1,15 @@ # TRT-LLM Telemetry Schema Reference -Schema version: **0.7** | Client ID: `616561816355034` | Protocol: GXT Event Protocol v1.6 +Schema version: **0.8** | Client ID: `616561816355034` | Protocol: GXT Event Protocol v1.6 ## Overview TRT-LLM collects anonymous, session-level deployment telemetry to understand how the library is used in production (GPU types, parallelism configs, model architectures). No PII, model weights, prompts, outputs, model paths, tokenizer -paths, or raw free-form configuration strings are collected. +paths, or raw free-form configuration strings are collected. The only state kept +across sessions is a local count of consecutive failed starts of the same launch; +see [Startup state](#startup-state). **Opt-out** (any one of these disables telemetry): - `TRTLLM_NO_USAGE_STATS=1` @@ -31,7 +33,7 @@ these top-level fields in Kibana alongside the event parameters. | `clientType` | string | Always `"Native"`. | | `clientVer` | string | TRT-LLM version, e.g. `"1.3.0rc9"`. | | `eventProtocol` | string | Always `"1.6"`. | -| `eventSchemaVer` | string | Schema version, currently `"0.7"`. | +| `eventSchemaVer` | string | Schema version, currently `"0.8"`. | | `eventSysVer` | string | Always `"trtllm-telemetry/1.0"`. | | `sessionId` | string | Unique hex UUID per telemetry session. Use this to correlate initial, heartbeat, and terminal events. | | `sentTs` | string | ISO 8601 UTC timestamp of when the payload was sent. | @@ -153,6 +155,19 @@ heartbeat, and the terminal report. Cumulative counters saturate at uint32 max; | `maxConcurrentLlmInstances` | PositiveInt | Maximum active objects observed in the process session. | | `llmInitializationFailures` | PositiveInt | Initialization attempts that raised a handled Python exception. | +#### Startup state fields + +Present on the initial report and the terminal report; see +[Startup state](#startup-state). + +| Field | Type | Description | +|-------|------|-------------| +| `priorStartupFailures` | PositiveInt | Consecutive failed starts of the same launch before this session, as a bucket lower bound: `0`, `1`, `2`, `3`, `10`, `30`, `100`, `300`, or `1000`. | +| `startupStateStatus` | enum | `persisted` if the local count was read, `new` if it was missing or corrupt, `unavailable` if it could not be read or written in time, or `""` if it was not consulted. | + +A partial initial report carries the values of its failed start. Count +successful starts only from initial reports with `llmInstancesCreated >= 1`. + ### `trtllm_heartbeat` Sent periodically (default: every 600s) to track session duration until process @@ -184,6 +199,9 @@ outcome. Missing terminal events remain unknown; they are not confirmed crashes. | `ingressPoint` | ShortString | Entry point copied onto the terminal event so terminal-only early failures remain attributable. | | `disaggRole` | ShortString | Disaggregated role (`context`, `generation`, `coordinator`, `server_coordinator`, or compatible legacy `ctx0`/`gen0`), or empty when unavailable/not applicable. | | `deploymentId` | ShortString | Optional shared disaggregated deployment UUID. | +| `errorCategory` | enum | Category of the startup failure that ended the session, or `""` if the session did not fail during startup. See [Error category values](#error-category-values). | +| `priorStartupFailures` | PositiveInt | See [Startup state fields](#startup-state-fields). | +| `startupStateStatus` | enum | See [Startup state fields](#startup-state-fields). | Every terminal report also contains the five aggregate LLM lifecycle counters above. Delivery is best-effort and waits no more than 0.5 seconds; the local @@ -194,7 +212,8 @@ normalized to shell-style `exitCode=137` with `signalNumber=9`. Terminal fields are deliberately bounded and categorical. Exception messages, stack traces, commands, paths, model identifiers, and configuration text are -not collected. +not collected. `errorCategory` is computed locally from the exception; only the +category value is sent. V1 signal coverage is boundary-specific. The generic CLI boundary observes Ctrl+C/SIGINT, while Uvicorn serving and explicitly instrumented disaggregated @@ -202,6 +221,73 @@ boundaries observe both SIGINT and SIGTERM. A default SIGTERM delivered to another CLI, such as `trtllm-bench` or `trtllm-eval`, may terminate the process before it can send a terminal report. +#### Error category values + +`errorCategory` uses the categories of `tensorrt_llm/_errors.py`. TRT-LLM raise +sites declare them; other errors are classified by exception type, error code, +and a fixed list of message phrases. + +| Value | Meaning | Examples | +|-------|---------|----------| +| `config_input` | The user's input or setup is wrong. | Invalid CLI option, config file, or key; model path not found; no Hugging Face token or license not accepted | +| `resource` | A required resource is missing or exhausted. | No usable GPU, fewer GPUs than ranks, GPU out of memory, port in use, disk full | +| `unsupported` | The product does not support the request. | Unregistered model architecture, GPU too old for the quantization format | +| `model_load` | The model files are broken or incompatible. | Missing or mismatched weights, corrupted checkpoint | +| `runtime_init` | Bringing up the runtime failed during startup. | Worker process exited during startup, MPI or NCCL initialization, kernel compilation | +| `other` | Not attributable to the categories above. | Errors without a declared category, including internal errors; unrecognized third-party errors; exceptions from user code | + +The category is set only when the session ends during startup (before any LLM +instance was created and before serving) with an exception, or when an LLM API +process exits after a failed `LLM()`. It is `""` for clean exits, signals, +timeouts, failures after startup, and child failures observed by a supervisor, +because the child reports its own. Executor workers classify their own failure +and pass the result to the reporting process. VisualGen sessions do not set it; +a `trtllm-serve` VisualGen launch that fails before the command has determined +that it serves VisualGen, such as on a CLI usage error, is reported like an LLM +launch. + +#### Startup state + +The reporting process keeps a local count of consecutive failed starts of the +same launch, so that dashboards can recognize restart loops without any +identifier. Each report carries the count it found, as a bucket. + +- **Launch**: a local hash of the TensorRT-LLM version, command line, + `CUDA_VISIBLE_DEVICES`, `NVIDIA_VISIBLE_DEVICES`, launcher rank, and + TensorRT-LLM process-role variables. Instances that differ in any of these keep + separate counts, and an upgrade starts a new count; copies of the same launch on + hosts that share the state directory share one count. None of these values is + sent. +- **File**: `startup-.json` under `TRTLLM_USAGE_STATE_DIR`, else + `$XDG_CACHE_HOME/tensorrt_llm/usage/`, else `~/.cache/tensorrt_llm/usage/`, + holding only `{"v": 1, "streak": , "last_failure": }`. + The directory is created with mode 0700 and the file with mode 0600; writes + replace the file atomically. Files not updated for 30 days are removed when a + new one is created. +- **Rules**: a failed start (an exception or worker failure during startup, or + an LLM API exit after a failed `LLM()`) adds one; creating an LLM instance or + reaching serving resets the count to 0; other outcomes, such as signals, leave + it unchanged. A count whose last failure is more than 24 hours old is reported + as 0. Successful CLI probes such as `--help` do not touch the file. +- **When**: only in the process that sends telemetry, after the opt-out + decision, and at most once per process: on success in the background reporter, + otherwise when the terminal report is built. Either way the file I/O runs on a + helper thread, and the report waits for it at most 0.1 seconds (on the exit + path, out of its 0.5-second bound); if it has not finished, the report is sent + with `priorStartupFailures` `0` and `startupStateStatus` `unavailable`. It is + never read or written when telemetry is disabled; after a late opt-out in the + same process, no new read or write starts. VisualGen sessions do not consult + it (`""`). +- **Limits**: starts killed without any report (for example by SIGKILL) are not + counted. Restarts that lose the file, such as container restarts without a + persistent home directory, report `new`; Kubernetes users can point + `TRTLLM_USAGE_STATE_DIR` at an `emptyDir` volume. Processes that never create + an LLM or reach serving, such as `disaggregated_mpi_worker` leaders, rely on + the 24-hour expiry. + +Records from clients before schema 0.8 have none of these fields. Treat them as +not collected, separate from `""`, `other`, and `0`. + ## Type Reference | Type | JSON type | Constraints | @@ -309,6 +395,7 @@ over time. | `TRTLLM_USAGE_STATS_SERVER` | `https://events.gfe.nvidia.com/v1.1/events/json` | Override the GXT endpoint URL. Use for staging. | | `TRTLLM_USAGE_HEARTBEAT_INTERVAL` | `600` | Heartbeat interval in seconds. | | `TRTLLM_USAGE_FORCE_ENABLED` | `0` | Set to `1` to force-enable telemetry in CI/test environments. | +| `TRTLLM_USAGE_STATE_DIR` | `~/.cache/tensorrt_llm/usage` | Directory of the local consecutive startup failure count. See [Startup state](#startup-state). | | `TRTLLM_DISAGG_ROLE` | unset | Disaggregated serving role (`context`, `generation`, `coordinator`, `server_coordinator`, or compatible legacy values such as `ctx0`/`gen0`). | | `TRTLLM_DISAGG_DEPLOYMENT_ID` | unset | Shared deployment UUID across disaggregated workers. Invalid values are reported as an empty ID. | diff --git a/tensorrt_llm/usage/schemas/trtllm_usage_event_schema.json b/tensorrt_llm/usage/schemas/trtllm_usage_event_schema.json index 81ef2e9d4f74..cb3ba1542326 100644 --- a/tensorrt_llm/usage/schemas/trtllm_usage_event_schema.json +++ b/tensorrt_llm/usage/schemas/trtllm_usage_event_schema.json @@ -12,7 +12,7 @@ ], "$schema": "http://json-schema.org/draft-07/schema#", "schemaMeta": { - "schemaVersion": "0.7", + "schemaVersion": "0.8", "clientId": "616561816355034", "clientName": "TrtllmTelemetry", "definitionVersion": "2.0" @@ -143,6 +143,18 @@ }, "llmInitializationFailures": { "$ref": "#/definitions/types/PositiveInt" + }, + "priorStartupFailures": { + "$ref": "#/definitions/types/PositiveInt" + }, + "startupStateStatus": { + "type": "string", + "enum": [ + "", + "persisted", + "new", + "unavailable" + ] } }, "required": [ @@ -176,7 +188,9 @@ "llmInstancesCreated", "activeLlmInstances", "maxConcurrentLlmInstances", - "llmInitializationFailures" + "llmInitializationFailures", + "priorStartupFailures", + "startupStateStatus" ] }, "trtllm_heartbeat": { @@ -312,6 +326,30 @@ }, "llmInitializationFailures": { "$ref": "#/definitions/types/PositiveInt" + }, + "errorCategory": { + "type": "string", + "enum": [ + "", + "config_input", + "resource", + "unsupported", + "model_load", + "runtime_init", + "other" + ] + }, + "priorStartupFailures": { + "$ref": "#/definitions/types/PositiveInt" + }, + "startupStateStatus": { + "type": "string", + "enum": [ + "", + "persisted", + "new", + "unavailable" + ] } }, "required": [ @@ -329,7 +367,10 @@ "llmInstancesCreated", "activeLlmInstances", "maxConcurrentLlmInstances", - "llmInitializationFailures" + "llmInitializationFailures", + "errorCategory", + "priorStartupFailures", + "startupStateStatus" ] } } diff --git a/tensorrt_llm/usage/startup_state.py b/tensorrt_llm/usage/startup_state.py new file mode 100644 index 000000000000..7f5415c2c477 --- /dev/null +++ b/tensorrt_llm/usage/startup_state.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Local count of consecutive failed starts of one launch, for usage telemetry. + +A launch is the same command of the same TensorRT-LLM version, with the same +launcher rank, visible GPUs and TensorRT-LLM process role. Each launch keeps one +small JSON file with its number of consecutive startup failures and the time of +the last one. The file name is a local hash of those values; none of them is +sent, only a bucket of the count. + +The usage telemetry reporter reads and writes the file at most once per process, +and only when telemetry is enabled. Functions here never raise: file system +errors become the ``unavailable`` status. Standard library only. +""" + +import hashlib +import json +import os +import sys +import tempfile +import time +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Literal, Optional + +STATE_DIR_ENV = "TRTLLM_USAGE_STATE_DIR" +# Reported counts are the largest of these lower bounds not above the count. +PRIOR_STARTUP_FAILURE_BUCKETS = (0, 1, 2, 3, 10, 30, 100, 300, 1000) +StartupStateAction = Literal["increment", "reset", "read"] + +_STATE_VERSION = 1 +_KEY_DOMAIN = b"trtllm-startup-state-v1\0" +_KEY_HEX_CHARS = 16 +_FILE_PREFIX = "startup-" +_FILE_SUFFIX = ".json" +_TMP_PREFIX = ".startup-" +_TMP_SUFFIX = ".tmp" +_MAX_STATE_BYTES = 4096 +_MAX_STREAK = 1_000_000 +_STREAK_EXPIRY_SECONDS = 24 * 3600 +_PRUNE_AGE_SECONDS = 30 * 24 * 3600 +# Variables that tell apart launches of the same command: the launcher rank, +# the visible GPUs and TensorRT-LLM's process roles. +_KEY_ENV_VARS = ( + "SLURM_PROCID", + "OMPI_COMM_WORLD_RANK", + "PMI_RANK", + "PMIX_RANK", + "CUDA_VISIBLE_DEVICES", + "NVIDIA_VISIBLE_DEVICES", + "TLLM_DISAGG_INSTANCE_IDX", + "TRTLLM_DISAGG_WORKER_PROCESS_ID", + "TLLM_EXECUTOR_FRONTEND_ID", +) + + +def launch_key( + version: str, + argv: Optional[Sequence[str]] = None, + environ: Optional[Mapping[str, str]] = None, +) -> str: + """Return the hex key naming this launch's state file, or "" if unavailable. + + Hashes the TensorRT-LLM ``version``, the command line and ``_KEY_ENV_VARS``; + touches no file. + """ + try: + environ = os.environ if environ is None else environ + identity = { + "version": version, # an upgrade starts a new count + "argv": list(sys.argv if argv is None else argv), + "env": {name: environ.get(name, "") for name in _KEY_ENV_VARS}, + } + encoded = json.dumps(identity, sort_keys=True).encode("utf-8") + return hashlib.sha256(_KEY_DOMAIN + encoded).hexdigest()[:_KEY_HEX_CHARS] + except Exception: + return "" + + +def bucket(streak: int) -> int: + """Return the reported bucket lower bound for a consecutive failure count.""" + return max(b for b in PRIOR_STARTUP_FAILURE_BUCKETS if b <= max(streak, 0)) + + +def state_dir(environ: Optional[Mapping[str, str]] = None) -> Optional[Path]: + """Return the state directory, or None if no home directory is available. + + ``$TRTLLM_USAGE_STATE_DIR``, else ``$XDG_CACHE_HOME/tensorrt_llm/usage``, else + ``~/.cache/tensorrt_llm/usage``. + """ + environ = os.environ if environ is None else environ + override = environ.get(STATE_DIR_ENV) + if override: + return Path(override) + cache_home = environ.get("XDG_CACHE_HOME") + if cache_home and os.path.isabs(cache_home): + return Path(cache_home) / "tensorrt_llm" / "usage" + try: + return Path.home() / ".cache" / "tensorrt_llm" / "usage" + except (RuntimeError, KeyError): + return None + + +def consult( + key: str, + action: StartupStateAction, + *, + now: Optional[float] = None, + environ: Optional[Mapping[str, str]] = None, +) -> tuple[int, str]: + """Read this launch's count, apply ``action``, and return ``(prior, status)``. + + ``prior`` is the count of consecutive failed starts before this process; a + count whose last failure is more than 24 hours old is 0. ``status`` is + ``persisted`` if the file was read, ``new`` if it was missing or corrupt, and + ``unavailable`` if it could not be read or written. ``increment`` adds this + process's failed start, ``reset`` records a successful start, and ``read`` + leaves the file unchanged. + """ + try: + directory = state_dir(environ) if key else None + if directory is None: + return 0, "unavailable" + now = time.time() if now is None else now + path = directory / f"{_FILE_PREFIX}{key}{_FILE_SUFFIX}" + prior, status = _read(path, now) + if action == "read" or status == "unavailable": + return prior, status + if action == "increment": + state = { + "v": _STATE_VERSION, + "streak": min(prior + 1, _MAX_STREAK), + "last_failure": int(now), + } + else: + state = {"v": _STATE_VERSION, "streak": 0, "last_failure": 0} + if not _write(directory, path, state): + return prior, "unavailable" + if status == "new": + _prune(directory, now, keep=path.name) + return prior, status + except Exception: + return 0, "unavailable" + + +def _read(path: Path, now: float) -> tuple[int, str]: + try: + with open(path, "rb") as stream: + raw = stream.read(_MAX_STATE_BYTES + 1) + except FileNotFoundError: + return 0, "new" + except OSError: + return 0, "unavailable" + try: + state = json.loads(raw) if len(raw) <= _MAX_STATE_BYTES else None + streak = state["streak"] + last_failure = state["last_failure"] + valid = ( + state["v"] == _STATE_VERSION + and type(streak) is int + and streak >= 0 + and type(last_failure) in (int, float) + ) + except (TypeError, ValueError, KeyError): + valid = False + if not valid: + return 0, "new" + if now - last_failure > _STREAK_EXPIRY_SECONDS: + streak = 0 + return streak, "persisted" + + +def _write(directory: Path, path: Path, state: dict) -> bool: + """Replace the file atomically; the directory is created private to the user.""" + tmp = None + try: + directory.mkdir(mode=0o700, parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=directory, prefix=_TMP_PREFIX, suffix=_TMP_SUFFIX) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(state, stream) + os.replace(tmp, path) + return True + except OSError: + if tmp is not None: + try: + os.unlink(tmp) + except OSError: + pass + return False + + +def _prune(directory: Path, now: float, keep: str) -> None: + """Remove this module's files that were not updated for 30 days.""" + try: + with os.scandir(directory) as entries: + for entry in entries: + name = entry.name + ours = (name.startswith(_FILE_PREFIX) and name.endswith(_FILE_SUFFIX)) or ( + name.startswith(_TMP_PREFIX) and name.endswith(_TMP_SUFFIX) + ) + if not ours or name == keep: + continue + try: + if ( + entry.is_file(follow_symlinks=False) + and now - entry.stat(follow_symlinks=False).st_mtime > _PRUNE_AGE_SECONDS + ): + os.unlink(entry.path) + except OSError: + pass + except OSError: + pass diff --git a/tensorrt_llm/usage/usage_lib.py b/tensorrt_llm/usage/usage_lib.py index 0872231086d0..444f0bee97b1 100644 --- a/tensorrt_llm/usage/usage_lib.py +++ b/tensorrt_llm/usage/usage_lib.py @@ -33,6 +33,8 @@ TRTLLM_USAGE_HEARTBEAT_INTERVAL: Heartbeat interval in seconds (default 600). TRTLLM_USAGE_FORCE_ENABLED: Set to "1" to force-enable telemetry even in CI/test environments (e.g., for staging deployments run via CI). + TRTLLM_USAGE_STATE_DIR: Directory of the local consecutive startup failure + count (default ~/.cache/tensorrt_llm/usage). CI/Test auto-detection: Telemetry is automatically disabled when running in CI environments or @@ -57,9 +59,9 @@ import uuid from dataclasses import dataclass from pathlib import Path -from typing import Any, Callable, Dict, Optional, TypeVar +from typing import Any, Callable, Dict, Optional, TypeVar, get_args -from tensorrt_llm.usage import schema +from tensorrt_llm.usage import schema, startup_state from tensorrt_llm.usage._startup import REPORT_CONTEXT, bounded_gpu_fields, requested_fields from tensorrt_llm.usage.architecture_allowlist import PUBLIC_MODEL_ARCHITECTURES from tensorrt_llm.usage.config import UsageContext @@ -79,9 +81,12 @@ _DEFAULT_ENDPOINT = "https://events.gfe.nvidia.com/v1.1/events/json" _HTTP_TIMEOUT = 2.0 _TERMINAL_FLUSH_TIMEOUT = 0.5 +_STARTUP_STATE_TIMEOUT = 0.1 _ALLOWED_USAGE_CONTEXTS = frozenset(context.value for context in UsageContext) _ARCHITECTURE_HASH_DOMAIN = b"trtllm-architecture-class-v1\0" _ARCHITECTURE_HASH_PREFIX = "sha256:" +_ERROR_CATEGORIES = frozenset(get_args(schema.ErrorCategory)) - {""} +_STARTUP_PHASES = frozenset(("cli_parsing", "config_validation", "model_initialization")) _T = TypeVar("_T") @@ -251,7 +256,9 @@ def _get_heartbeat_interval() -> int: "TRT-LLM collects anonymous usage data to help improve the product. " "This data cannot be traced back to any individual user. " "No user-identifying information, persistent identifiers, or prompts " - "are collected. To disable, set TRTLLM_NO_USAGE_STATS=1, " + "are collected. A small local file counts consecutive failed starts of " + "the same launch; only a coarse count is sent. " + "To disable, set TRTLLM_NO_USAGE_STATS=1, " "TELEMETRY_DISABLED=true, or pass " "TelemetryConfig(disabled=True). " "See https://github.com/NVIDIA/TensorRT-LLM for details." @@ -618,6 +625,11 @@ def _clamp_str(value: str, max_len: int) -> str: return value[:max_len] if len(value) > max_len else value +def _allowed_error_category(error_category: str) -> str: + """Map anything outside the reported categories to ``other``.""" + return error_category if error_category in _ERROR_CATEGORIES else "other" + + def _background_reporter( llm_args: Any, pretrained_config: Any, @@ -659,6 +671,14 @@ def _background_reporter( # Use the latest process counters after the collection work above. event_snapshot = _event_snapshot(usage_context) + # This reporter starts after an LLM initialized: record a successful start. + # Bounded like the exit path, so a stalled file system cannot hold back + # the initial report and heartbeats. + prior_failures, state_status = _bounded_startup_state( + session, + "reset" if event_snapshot["llmInstancesCreated"] > 0 else "read", + time.monotonic() + _STARTUP_STATE_TIMEOUT, + ) # --- Build initial report event --- # All fields are required by the SMS schema. Use empty string / 0 @@ -697,6 +717,8 @@ def _background_reporter( featuresJson=features_json, llmApiConfigJson=llm_api_config_json, llmApiConfigMetaJson=llm_api_config_meta_json, + priorStartupFailures=prior_failures, + startupStateStatus=state_status, **_session_event_fields(event_snapshot), ) @@ -782,6 +804,13 @@ def __init__( self.active_llm_instances = 0 self.max_concurrent_llm_instances = 0 self.llm_initialization_failures = 0 + self.error_category = "" + self.visual_gen = False + # Fixed at session creation from version, argv and selected environment variables only. + self.startup_state_key = startup_state.launch_key(self.trtllm_version) + self.startup_state: Optional[tuple[int, str]] = None + # Held across state-file I/O, so it never blocks the session lock. + self.startup_state_lock = threading.Lock() self.disabled = False self.initial_reported = False @@ -855,11 +884,31 @@ def record_llm_initialization_attempt(self) -> bool: self._refresh_metadata_unlocked() return True - def record_llm_initialization_failure(self) -> None: - """Increment the initialization failure counter.""" + def record_llm_initialization_failure(self, error_category: str = "") -> None: + """Increment the initialization failure counter and keep the failure's category.""" with self.lock: if not self.disabled and not self.terminal_reported: self.llm_initialization_failures = self._increment(self.llm_initialization_failures) + if error_category: + self.error_category = _allowed_error_category(error_category) + + def record_startup_error_category(self, error_category: str) -> None: + """Keep the category of a startup failure unless an LLM attempt recorded one.""" + with self.lock: + if ( + not self.disabled + and not self.terminal_reported + and not self.llm_instances_created + and not self.error_category + and error_category + ): + self.error_category = _allowed_error_category(error_category) + + def mark_visual_gen(self) -> None: + """Exclude this VisualGen session from startup failure reporting.""" + with self.lock: + if not self.disabled and not self.terminal_reported: + self.visual_gen = True def record_llm_initialized(self) -> bool: """Record a successfully constructed and active LLM object.""" @@ -873,6 +922,7 @@ def record_llm_initialized(self) -> bool: self.active_llm_instances, ) self.lifecycle_phase = "serving" + self.error_category = "" return True def record_llm_shutdown(self) -> None: @@ -930,6 +980,8 @@ def _snapshot_unlocked(self) -> dict[str, Any]: "lifecyclePhase": self.lifecycle_phase, "component": self.component, "observedSignal": self.observed_signal, + "errorCategory": self.error_category, + "visualGen": self.visual_gen, } def snapshot(self) -> dict[str, Any]: @@ -942,6 +994,27 @@ def get_observed_signal(self) -> int: with self.lock: return self.observed_signal + def consult_startup_state(self, action: startup_state.StartupStateAction) -> tuple[int, str]: + """Apply ``action`` to the local startup state once and return its fields. + + Returns the bucketed prior failure count and the state status. The first + call does the file I/O; later calls return its result, so a process reads + and writes the state file at most once. Disabled and VisualGen sessions + never touch it. + """ + with self.startup_state_lock: + if self.startup_state is None: + # Under the session lock: a consult that has not passed this check + # when disable() returns never touches the file. + with self.lock: + skip = self.disabled or self.visual_gen + if skip: + self.startup_state = (0, "") + else: + prior, status = startup_state.consult(self.startup_state_key, action) + self.startup_state = (startup_state.bucket(prior), status) + return self.startup_state + def claim_terminal( self, outcome: TerminalOutcome, lifecycle_phase: Optional[schema.LifecyclePhase] = None ) -> Optional[tuple[dict[str, Any], TerminalOutcome]]: @@ -1089,6 +1162,8 @@ def _empty_event_snapshot(usage_context: str = "") -> dict[str, Any]: "lifecyclePhase": "unknown", "component": "unknown", "observedSignal": 0, + "errorCategory": "", + "visualGen": False, } @@ -1287,9 +1362,17 @@ def record_llm_initialization_attempt( ) -def record_llm_initialization_failure() -> None: - """Record a handled Python exception from an LLM constructor.""" - _session_call(lambda session: session.record_llm_initialization_failure(), None) +def record_llm_initialization_failure(error_category: str = "") -> None: + """Record a handled Python exception from an LLM constructor and its category.""" + _session_call(lambda session: session.record_llm_initialization_failure(error_category), None) + + +def record_startup_error_category(error_category: str) -> None: + """Record the category of an uncaught startup failure at a process boundary. + + A category recorded by the failed LLM constructor takes precedence. + """ + _session_call(lambda session: session.record_startup_error_category(error_category), None) def _mark_llm_startup() -> None: @@ -1297,6 +1380,11 @@ def _mark_llm_startup() -> None: _session_call(lambda session: session.mark_llm_startup(), None) +def _mark_visual_gen() -> None: + """Mark the session as VisualGen: no error category or startup state is reported.""" + _session_call(lambda session: session.mark_visual_gen(), None) + + def _capture_startup_context( *, requested: Optional[dict] = None, @@ -1475,6 +1563,10 @@ def _send_terminal(pending: _PendingTerminal) -> None: else "unavailable", ) event_fields = _session_event_fields(terminal["parameters"]) + event_fields.update( + priorStartupFailures=terminal["parameters"]["priorStartupFailures"], + startupStateStatus=terminal["parameters"]["startupStateStatus"], + ) defaults = dict(tensorParallelSize=0, pipelineParallelSize=0, contextParallelSize=0) initial = schema.TrtllmInitialReport( **(defaults | fields), @@ -1495,6 +1587,75 @@ def _send_terminal(pending: _PendingTerminal) -> None: _send_if_session_active(pending.session, payload) +def _bounded_startup_state( + session: _TelemetrySession, action: startup_state.StartupStateAction, deadline: float +) -> tuple[int, str]: + """Spend at most 100 ms, and nothing past ``deadline``, on the local startup state. + + The file I/O, and any wait for another thread's I/O under the state lock, + run on a helper thread. If it has not finished in time, for example on a + stalled file system, the state is reported as ``unavailable``. + """ + # Set once under startup_state_lock; a reporter's earlier result needs no I/O. + cached = session.startup_state + if cached is not None: + return cached + result: list[tuple[int, str]] = [] + try: + thread = threading.Thread( + target=lambda: result.append(session.consult_startup_state(action)), + daemon=True, + name="trtllm-usage-state", + ) + thread.start() + except Exception: + # Python 3.12.0 and 3.12.1 cannot start threads from an atexit callback. + return 0, "unavailable" + # A timed-out consult cannot be cancelled; a late write is still atomic. + thread.join(min(_STARTUP_STATE_TIMEOUT, max(0.0, deadline - time.monotonic()))) + return result[0] if result else (0, "unavailable") + + +def _exit_startup_fields( + session: _TelemetrySession, + snapshot: dict[str, Any], + outcome: TerminalOutcome, + deadline: float, +) -> dict[str, Any]: + """Classify the exit for startup failure reporting and update the local count. + + A process that created an LLM or reached serving started successfully: the + count is reset. An exception or worker failure during startup, or an LLM API + exit after a failed ``LLM()``, is a failed start: the count is incremented + and, unless a supervisor observed a child, the failure's category is + reported. Other outcomes, such as signals, only read the count. + """ + if snapshot["visualGen"]: + return {"errorCategory": "", "priorStartupFailures": 0, "startupStateStatus": ""} + kind = outcome.termination_kind + phase = snapshot["lifecyclePhase"] + failed_start = (kind in ("exception", "worker_failure") and phase in _STARTUP_PHASES) or ( + kind == "unknown" + and phase == "model_initialization" + and snapshot["llmInitializationFailures"] > 0 + ) + error_category = "" + if snapshot["llmInstancesCreated"] > 0 or phase == "serving": + action = "reset" + elif failed_start: + action = "increment" + if outcome.reporting_source == "self" and kind in ("exception", "unknown"): + error_category = snapshot["errorCategory"] or "other" + else: + action = "read" + prior, status = _bounded_startup_state(session, action, deadline) + return { + "errorCategory": error_category, + "priorStartupFailures": prior, + "startupStateStatus": status, + } + + def report_exit( outcome: TerminalOutcome, *, @@ -1544,6 +1705,7 @@ def report_exit( ): return True + startup_fields = _exit_startup_fields(session, snapshot, outcome, deadline) _show_usage_notification() exit_code_known = outcome.exit_code_known is True @@ -1566,6 +1728,7 @@ def report_exit( component=outcome.component or snapshot["component"], reportingSource=outcome.reporting_source, **_session_event_fields(snapshot), + **startup_fields, ) payload = schema.build_gxt_payload( event=event, diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 83e786c3af7a..da2c875cb19d 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -345,6 +345,7 @@ l0_cpu: - unittest/llmapi/test_serve_report_addr.py - unittest/llmapi/test_serve_sleep_mode.py - unittest/llmapi/test_session_prefetcher.py + - unittest/llmapi/test_startup_checks.py - unittest/llmapi/test_tokenizer_aliases.py - unittest/llmapi/test_tokenizer_decode.py - unittest/llmapi/test_trtllm_serve_config_schemas.py diff --git a/tests/unittest/_torch/kv_cache_compression/test_quantization_for_cold_page.py b/tests/unittest/_torch/kv_cache_compression/test_quantization_for_cold_page.py index eb874bf220f6..e2a328e87217 100644 --- a/tests/unittest/_torch/kv_cache_compression/test_quantization_for_cold_page.py +++ b/tests/unittest/_torch/kv_cache_compression/test_quantization_for_cold_page.py @@ -1266,7 +1266,7 @@ def test_fp8_runtime_uses_modelopt_nvfp4_scales(tmp_path): def test_runtime_admission_is_checked_before_manager_creation(monkeypatch) -> None: monkeypatch.setattr(util_mod, "is_sm_100f", lambda: False) - with pytest.raises(RuntimeError, match="requires an SM100-family device"): + with pytest.raises(ValueError, match="requires an SM100-family device"): util_mod.create_kv_cache_compression_manager( ColdPageQuantizationCompressionConfig(), model_engine=_factory_model_engine(), diff --git a/tests/unittest/_torch/modeling/test_modeling_out_of_tree.py b/tests/unittest/_torch/modeling/test_modeling_out_of_tree.py index 0916fbbe3e99..7c92d6d87404 100644 --- a/tests/unittest/_torch/modeling/test_modeling_out_of_tree.py +++ b/tests/unittest/_torch/modeling/test_modeling_out_of_tree.py @@ -1,7 +1,5 @@ -import re from contextlib import nullcontext from pathlib import Path -from typing import cast import pytest @@ -82,9 +80,11 @@ def test_llm_api( monkeypatch.syspath_prepend(oot_path) import modeling_opt # noqa - with (nullcontext() if import_oot_code else - pytest.raises(RuntimeError, - match=".*Executor worker returned error.*")) as ctx: + # Without the out-of-tree code the architecture is unregistered, which + # the LLM rejects before spawning any worker. + with (nullcontext() if import_oot_code else pytest.raises( + ValueError, + match="Unknown model architecture: OPTForCausalLM")): with LLM( model=str(model_dir), kv_cache_config=KvCacheConfig(free_gpu_memory_fraction=0.2), @@ -95,13 +95,6 @@ def test_llm_api( for output, ref in zip(outputs, references): assert similar(output.outputs[0].text, ref) - if not import_oot_code: - exc_val = cast(pytest.ExceptionInfo, ctx).value - assert re.match( - ".*Unknown architecture for AutoModelForCausalLM: OPTForCausalLM.*", - str(exc_val.__cause__), - ) is not None - @pytest.mark.parametrize("import_oot_code", [False, True]) def test_serve( self, diff --git a/tests/unittest/_torch/pyexecutor/kv_cache/test_kv_cache_budget_split.py b/tests/unittest/_torch/pyexecutor/kv_cache/test_kv_cache_budget_split.py index 3fceaa1ada95..6c31577797ca 100644 --- a/tests/unittest/_torch/pyexecutor/kv_cache/test_kv_cache_budget_split.py +++ b/tests/unittest/_torch/pyexecutor/kv_cache/test_kv_cache_budget_split.py @@ -658,7 +658,7 @@ def test_exact_fixed_budget_rejects_a_linear_manager(self): total_kv_intercept=200, ) - with pytest.raises(ValueError, match="GPU budget"): + with pytest.raises(RuntimeError, match="GPU budget"): c._split_kv_cache_budget_for_draft("max_gpu_total_bytes") @@ -1008,7 +1008,7 @@ def test_gpu_split_infeasible_raises(self): total_kv_intercept=2 * GB, ) - with pytest.raises(ValueError, match="GPU budget"): + with pytest.raises(RuntimeError, match="GPU budget"): c._split_kv_cache_budget_for_draft("max_gpu_total_bytes") def test_gpu_raise_does_not_block_subsequent_host_split(self): @@ -1816,7 +1816,7 @@ def test_floor_exceeding_slope_budget_is_infeasible(self): # The slope budget is 1000 - 400 = 600, below the 700-byte one-mirror # quota. Flooring anyway would hand the draft 700 and the target 300, # below its 400-byte intercept. - with pytest.raises(ValueError, match="insufficient"): + with pytest.raises(RuntimeError, match="insufficient"): c._split_kv_cache_budget_for_draft("max_gpu_total_bytes") def test_draft_gpu_budget_cap_moves_the_excess_to_the_target(self, monkeypatch): diff --git a/tests/unittest/_torch/pyexecutor/kv_cache/test_kv_cache_compression_manager.py b/tests/unittest/_torch/pyexecutor/kv_cache/test_kv_cache_compression_manager.py index 672cb7bb98e3..8062e2858c9f 100644 --- a/tests/unittest/_torch/pyexecutor/kv_cache/test_kv_cache_compression_manager.py +++ b/tests/unittest/_torch/pyexecutor/kv_cache/test_kv_cache_compression_manager.py @@ -333,7 +333,7 @@ def test_triattention_requires_sm100_family(self): cfg.algorithm = "triattention" with ( patch.object(util_mod, "is_sm_100f", return_value=False), - pytest.raises(RuntimeError, match="SM100-family"), + pytest.raises(ValueError, match="SM100-family"), ): util_mod.validate_kv_cache_compression_compatibility( cfg, diff --git a/tests/unittest/_torch/pyexecutor/test_indexer_workspace_reserve.py b/tests/unittest/_torch/pyexecutor/test_indexer_workspace_reserve.py index d5c0cf7cd290..257a9150c315 100644 --- a/tests/unittest/_torch/pyexecutor/test_indexer_workspace_reserve.py +++ b/tests/unittest/_torch/pyexecutor/test_indexer_workspace_reserve.py @@ -47,5 +47,5 @@ def test_workspace_reserve_covers_full_cap(kv_cache_memory, workspace_memory, ex @pytest.mark.parametrize("kv_cache_memory", [2, 4]) def test_workspace_reserve_rejects_exhausting_the_kv_budget(kv_cache_memory): - with pytest.raises(ValueError, match="whole KV cache budget"): + with pytest.raises(RuntimeError, match="whole KV cache budget"): get_indexer_mqa_logits_workspace_reserve(kv_cache_memory, 4) diff --git a/tests/unittest/executor/test_proxy_fast_death.py b/tests/unittest/executor/test_proxy_fast_death.py index 6a138c17eea3..bfcf1d8e1f47 100644 --- a/tests/unittest/executor/test_proxy_fast_death.py +++ b/tests/unittest/executor/test_proxy_fast_death.py @@ -23,6 +23,13 @@ import pytest +from tensorrt_llm._startup_errors import ( + StartupErrorInfo, + classify_startup_error, + is_worker_startup_error, + log_startup_error, + wrap_worker_startup_error, +) from tensorrt_llm.executor import EngineDeadError from tensorrt_llm.executor import proxy as proxy_module from tensorrt_llm.executor import worker as worker_module @@ -318,6 +325,59 @@ def make_ipc_queue(*args, name, **kwargs): assert events == [("notify", GenerationExecutorProxy.WORKER_PROCESS_IDENTITIES_SIGNAL)] +def test_non_leader_worker_raises_its_classified_error(monkeypatch): + """A non-leader has no status queue, so it reports through its MPI future.""" + + class _FakeComm: + def barrier(self): + pass + + def allgather(self, captured_identity): + return [captured_identity] + + class _OomWorker: + def __init__(self, *args, **kwargs): + raise RuntimeError("CUDA Runtime API error: cudaErrorMemoryAllocation") + + monkeypatch.setattr(worker_module, "mpi_comm", lambda: _FakeComm()) + monkeypatch.setattr(worker_module, "mpi_rank", lambda: 1) + monkeypatch.setattr(worker_module, "capture_worker_process_identity", lambda rank: None) + monkeypatch.setattr(worker_module, "set_mpi_session_cpp", lambda comm: None) + + with pytest.raises(RuntimeError, match="^Executor worker returned error on rank 1: ") as e: + worker_module.worker_main( + worker_queues=_Mock(frontend_result_queue_addrs=None), + log_level=worker_module.logger.level, + worker_cls=_OomWorker, + ready_signal=GenerationExecutorProxy.READY_SIGNAL, + ) + + assert is_worker_startup_error(e.value) + assert classify_startup_error(e.value).category == "resource" + + +def test_non_leader_classified_error_during_init_is_raised_as_is(): + info = StartupErrorInfo( + "resource", + "RuntimeError: cudaErrorMemoryAllocation", + "Lower it.", + "executor_creation/model", + 1, + ) + error = wrap_worker_startup_error("Executor worker returned error", info) + proxy = _proxy_awaiting_worker_init(owns_session=True, future_exception=error) + + with pytest.raises(RuntimeError) as excinfo: + proxy._start_executor_workers({}) + + assert excinfo.value is error + assert str(error).startswith( + "Executor worker returned error on rank 1 during executor_creation/model" + ) + proxy.mpi_session.shutdown_abort.assert_called_once() + assert proxy._engine_dead is True + + def test_result_step_raises_on_engine_dead(): res = GenerationResult.__new__(GenerationResult) res.queue = _queue.Queue() @@ -984,6 +1044,35 @@ def test_non_ready_status_during_init_aborts_then_marks_dead(): assert isinstance(r.queue.get_nowait(), EngineDeadError) +def test_non_ready_status_reports_worker_rank_stage_and_root_cause(): + """The raised error names the rank, stage and root cause the worker classified. + + Pickling drops the worker exception's cause chain and traceback, so the + worker sends its classification along and the proxy error carries it. + """ + err = KeyError("lm_head.weight") + info = StartupErrorInfo( + "model_load", + "KeyError: 'lm_head.weight'", + "Check the checkpoint.", + "executor_creation/model", + 0, + ) + proxy = _proxy_awaiting_worker_init(owns_session=True, ready_status=(err, "trace", info)) + + with pytest.raises(RuntimeError) as excinfo: + proxy._start_executor_workers({}) + + assert str(excinfo.value) == ( + "Executor worker returned error on rank 0 during executor_creation/model: " + "[model_load] KeyError: 'lm_head.weight'" + ) + assert excinfo.value.__cause__ is err + assert classify_startup_error(excinfo.value) == info + # The hint was logged before the abort, which can end the process. + assert log_startup_error(excinfo.value, "LLM initialization failed") is None + + def test_non_ready_status_during_init_does_not_abort_borrowed_session(): proxy = _proxy_awaiting_worker_init( owns_session=False, ready_status=(ValueError("worker init failed"), "trace") diff --git a/tests/unittest/inputs/test_input_processor_fallback.py b/tests/unittest/inputs/test_input_processor_fallback.py new file mode 100644 index 000000000000..bd2a015aaf4d --- /dev/null +++ b/tests/unittest/inputs/test_input_processor_fallback.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""create_input_processor uses DefaultInputProcessor for an unregistered architecture.""" + +import json +from types import SimpleNamespace + +import pytest + +from tensorrt_llm._errors import ConfigInputError, UnsupportedError +from tensorrt_llm._torch.models import modeling_utils +from tensorrt_llm.inputs.registry import DefaultInputProcessor, create_input_processor + +pytestmark = pytest.mark.cpu_only + +_LLAMA_CONFIG = { + "model_type": "llama", + "hidden_size": 64, + "intermediate_size": 128, + "num_attention_heads": 4, + "num_key_value_heads": 4, + "num_hidden_layers": 1, + "vocab_size": 128, +} + + +@pytest.mark.parametrize( + "arch, error", + [ + ("FooBarForCausalLM", UnsupportedError), + # Gemma4 has no registered class when transformers is older than it requires. + ("Gemma4ForCausalLM", ConfigInputError), + ], +) +def test_unregistered_architecture_uses_default_input_processor(tmp_path, monkeypatch, arch, error): + monkeypatch.setattr(modeling_utils, "get_registered_model_class", lambda arch: None) + with pytest.raises(error): + modeling_utils.get_model_architecture(SimpleNamespace(architectures=[arch])) + + (tmp_path / "config.json").write_text(json.dumps({**_LLAMA_CONFIG, "architectures": [arch]})) + processor = create_input_processor(str(tmp_path), tokenizer=None) + + assert type(processor) is DefaultInputProcessor diff --git a/tests/unittest/llmapi/test_llm_args.py b/tests/unittest/llmapi/test_llm_args.py index 5a1513211863..6a67dfca5c3b 100644 --- a/tests/unittest/llmapi/test_llm_args.py +++ b/tests/unittest/llmapi/test_llm_args.py @@ -2188,8 +2188,12 @@ def test_vision_gpu_normalize_reaches_input_processor( setattr(llm, method, lambda *args, **kwargs: None) module = "mm_encoder" if encoder_only else "llm" with patch.object(BaseLLM, "_build_model"), patch( - f"tensorrt_llm.llmapi.{module}.create_input_processor", - side_effect=ValueError("input processor boundary")) as factory: + "tensorrt_llm.llmapi.llm.check_quantization_compute_capability" + ), patch( + "tensorrt_llm.llmapi.llm.check_architecture_registered"), patch( + f"tensorrt_llm.llmapi.{module}.create_input_processor", + side_effect=ValueError( + "input processor boundary")) as factory: with pytest.raises(ValueError, match="input processor boundary"): llm._build_model() kwargs = factory.call_args.kwargs diff --git a/tests/unittest/llmapi/test_llm_telemetry.py b/tests/unittest/llmapi/test_llm_telemetry.py index aff2f9b2640c..e1dab1b27549 100644 --- a/tests/unittest/llmapi/test_llm_telemetry.py +++ b/tests/unittest/llmapi/test_llm_telemetry.py @@ -31,6 +31,7 @@ from tensorrt_llm import LLM as LLM_torch from tensorrt_llm import MultimodalEncoder +from tensorrt_llm._errors import ConfigInputError, ResourceError from tensorrt_llm.llmapi import KvCacheConfig, llm_args from tensorrt_llm.llmapi.llm import BaseLLM from tensorrt_llm.usage import usage_lib @@ -42,8 +43,9 @@ @pytest.fixture -def enable_telemetry(monkeypatch): +def enable_telemetry(monkeypatch, tmp_path): """Enable telemetry for lifecycle tests in this sibling test package.""" + monkeypatch.setenv("TRTLLM_USAGE_STATE_DIR", str(tmp_path / "usage-state")) monkeypatch.delenv("TRTLLM_NO_USAGE_STATS", raising=False) monkeypatch.delenv("DO_NOT_TRACK", raising=False) monkeypatch.delenv("TELEMETRY_DISABLED", raising=False) @@ -129,6 +131,33 @@ def test_initialization_exception_is_counted_and_preserved(self, enable_telemetr assert usage_lib._REPORTER_STARTED is False assert snapshot["activeLlmInstances"] == 0 + @pytest.mark.parametrize( + ("error", "category"), + [ + (ConfigInputError("bad argument"), "config_input"), + (ResourceError("not enough GPUs"), "resource"), + (ValueError("expected test failure"), "other"), + ], + ) + def test_initialization_failure_records_its_category(self, enable_telemetry, error, category): + """The failed attempt's category is kept until an LLM initializes.""" + with ( + patch.object(BaseLLM, "__init__", side_effect=error), + patch.object(BaseLLM, "_start_usage_reporting"), + pytest.raises(type(error)), + ): + LLM_torch(model="unused") + assert usage_lib._SESSION.snapshot()["errorCategory"] == category + + with ( + patch.object(BaseLLM, "__init__", return_value=None), + patch.object(BaseLLM, "_start_usage_reporting"), + ): + llm = LLM_torch(model="unused") + assert usage_lib._SESSION.snapshot()["errorCategory"] == "" + with patch.object(BaseLLM, "_shutdown_resources"): + llm.shutdown() + def test_failure_before_base_constructor_is_counted(self, enable_telemetry): """The public LLM boundary includes subclass argument validation.""" with pytest.raises(ValueError): diff --git a/tests/unittest/llmapi/test_startup_checks.py b/tests/unittest/llmapi/test_startup_checks.py new file mode 100644 index 000000000000..02f61ad38948 --- /dev/null +++ b/tests/unittest/llmapi/test_startup_checks.py @@ -0,0 +1,249 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Preflight checks that run before weights are loaded (no GPU; devices are faked).""" + +import sys +from types import SimpleNamespace + +import pytest +import torch + +from tensorrt_llm._errors import ResourceError +from tensorrt_llm._startup_errors import classify_startup_error +from tensorrt_llm.executor.base_worker import BaseWorker +from tensorrt_llm.llmapi import startup_checks +from tensorrt_llm.llmapi.llm_args import Nvfp4GemmConfig, _ParallelConfig +from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.quantization.mode import QuantAlgo + +pytestmark = pytest.mark.cpu_only + + +@pytest.fixture +def fake_gpus(monkeypatch): + """Fake the visible device count, NVML and GPU 0's compute capability.""" + + def set_gpus(count: int = 1, sm: int = 90): + monkeypatch.setattr(startup_checks, "get_device_count", lambda: count) + monkeypatch.setattr(startup_checks, "get_sm_version", lambda: sm) + monkeypatch.setattr(torch.cuda, "get_device_name", lambda device=None: "Fake GPU") + monkeypatch.setattr( + startup_checks, + "_nvml_facts", + lambda: "NVML sees 4 GPU(s), driver 570.00 supports CUDA up to 12.8", + ) + + return set_gpus + + +def test_no_visible_gpu_lists_facts_and_causes(fake_gpus, monkeypatch): + fake_gpus(count=0) + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "") + + with pytest.raises(RuntimeError) as excinfo: + startup_checks.check_visible_gpus(_ParallelConfig(), all_ranks_local=True) + + message = str(excinfo.value) + assert message.startswith("No GPU is visible to PyTorch") + assert "CUDA_VISIBLE_DEVICES=," in message + assert "NVML sees 4 GPU(s), driver 570.00 supports CUDA up to 12.8" in message + assert "(3) the NVIDIA driver is too old" in message + info = classify_startup_error(excinfo.value) + assert (info.category, info.reason) == ("resource", "no_gpu") + assert info.hint.startswith("Possible causes:") + + +def test_no_visible_gpu_error_is_a_runtime_error(fake_gpus): + fake_gpus(count=0) + error = startup_checks.no_visible_gpu_error() + assert isinstance(error, ResourceError) + assert isinstance(error, RuntimeError) + + +def test_worker_without_visible_gpu_raises_resource_error(fake_gpus, monkeypatch): + fake_gpus(count=0) + monkeypatch.setattr(torch.cuda, "device_count", lambda: 0) + + with pytest.raises(ResourceError, match="No GPU is visible to PyTorch"): + BaseWorker._configure_device(SimpleNamespace(global_rank=0)) + + +def test_nvml_facts_report_unavailable(monkeypatch): + monkeypatch.setitem(sys.modules, "pynvml", None) # import fails + assert startup_checks._nvml_facts() == "NVML unavailable (ModuleNotFoundError)" + + +def test_too_few_gpus_for_local_ranks(fake_gpus): + # gpus_per_node defaults to the visible count, which alone would read as a + # 2-node world; ranks spawned by the LLM's own pool all land on this node. + fake_gpus(count=1) + + with pytest.raises(RuntimeError, match=r"Only 1 GPUs are available, but 2 are required") as e: + startup_checks.check_visible_gpus( + _ParallelConfig(tp_size=2, gpus_per_node=1), all_ranks_local=True + ) + + assert "2 x 1 x 1 = 2 ranks on this node" in str(e.value) + assert classify_startup_error(e.value).category == "resource" + + +def test_too_few_gpus_for_this_nodes_share(fake_gpus): + fake_gpus(count=2) + + with pytest.raises(RuntimeError, match=r"Only 2 GPUs are available, but 4 are required") as e: + startup_checks.check_visible_gpus( + _ParallelConfig(tp_size=8, pp_size=2, gpus_per_node=4), all_ranks_local=False + ) + + assert "8 x 2 x 1 = 16 ranks over 4 node(s)" in str(e.value) + + +@pytest.mark.parametrize( + "count, tp_size, gpus_per_node, all_ranks_local", + [ + (1, 1, 8, True), + (4, 4, 8, True), + (8, 4, 8, True), + # Launched across nodes: this node only needs its share. + (1, 2, 1, False), + (4, 8, 4, False), + ], +) +def test_enough_gpus_pass(fake_gpus, count, tp_size, gpus_per_node, all_ranks_local): + fake_gpus(count=count) + startup_checks.check_visible_gpus( + _ParallelConfig(tp_size=tp_size, gpus_per_node=gpus_per_node), all_ranks_local + ) + + +def _quant_args(quant_algo=None, kv_cache_quant_algo=None, nvfp4_backends=None): + nvfp4 = ( + Nvfp4GemmConfig() + if nvfp4_backends is None + else Nvfp4GemmConfig(allowed_backends=nvfp4_backends) + ) + return SimpleNamespace( + quant_config=QuantConfig(quant_algo=quant_algo, kv_cache_quant_algo=kv_cache_quant_algo), + nvfp4_gemm_config=nvfp4, + ) + + +@pytest.mark.parametrize( + "sm, quant_algo, kv_cache_quant_algo, backends", + [ + (80, QuantAlgo.FP8, None, None), + (89, QuantAlgo.FP8_BLOCK_SCALES, None, None), + (89, QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN, None, None), + (90, QuantAlgo.NVFP4, None, None), + (80, QuantAlgo.NVFP4, None, ["marlin"]), + (90, QuantAlgo.W4A8_MXFP4_MXFP8, None, None), + ], +) +def test_quantization_below_minimum_is_unsupported( + fake_gpus, sm, quant_algo, kv_cache_quant_algo, backends +): + fake_gpus(sm=sm) + + with pytest.raises(ValueError, match=r"requires SM\d+ or newer") as e: + startup_checks.check_quantization_compute_capability( + _quant_args(quant_algo, kv_cache_quant_algo, backends) + ) + + assert f"(Fake GPU) is SM{sm}." in str(e.value) + info = classify_startup_error(e.value) + assert info.category == "unsupported" + assert "quantization.html" in info.hint + + +@pytest.mark.parametrize( + "sm, quant_algo, kv_cache_quant_algo, backends", + [ + (89, QuantAlgo.FP8, None, None), + (90, QuantAlgo.FP8_BLOCK_SCALES, None, None), + (100, QuantAlgo.NVFP4, None, None), + (90, QuantAlgo.NVFP4, None, ["cutlass", "marlin"]), + (120, QuantAlgo.NVFP4, QuantAlgo.FP8, None), + # KV cache formats are not checked: kernels dequantize them before the MMA. + (80, None, QuantAlgo.FP8, None), + (90, None, QuantAlgo.NVFP4, None), + # Formats outside the documented matrix are not checked. + (80, QuantAlgo.W4A16_AWQ, None, None), + (80, QuantAlgo.MIXED_PRECISION, None, None), + (80, None, None, None), + ], +) +def test_quantization_at_or_above_minimum_passes( + fake_gpus, sm, quant_algo, kv_cache_quant_algo, backends +): + fake_gpus(sm=sm) + startup_checks.check_quantization_compute_capability( + _quant_args(quant_algo, kv_cache_quant_algo, backends) + ) + + +def test_nvfp4_on_hopper_suggests_marlin(fake_gpus): + fake_gpus(sm=90) + with pytest.raises(ValueError, match="add 'marlin' to nvfp4_gemm_config.allowed_backends"): + startup_checks.check_quantization_compute_capability(_quant_args(QuantAlgo.NVFP4)) + + +def test_quantization_check_skipped_without_gpu(fake_gpus): + fake_gpus(sm=-1) + startup_checks.check_quantization_compute_capability(_quant_args(QuantAlgo.NVFP4)) + + +def _arch_args( + checkpoint_format=None, + mm_encoder_only=False, + encode_only=False, + model_kwargs=None, + checkpoint_loader=None, +): + return SimpleNamespace( + checkpoint_format=checkpoint_format, + mm_encoder_only=mm_encoder_only, + encode_only=encode_only, + model_kwargs=model_kwargs, + checkpoint_loader=checkpoint_loader, + ) + + +@pytest.mark.parametrize( + "args, architectures", + [ + (_arch_args(), ["FooBarForCausalLM"]), + # model_kwargs overrides the checkpoint's architectures. + (_arch_args(model_kwargs={"architectures": ["FooBarForCausalLM"]}), ["LlamaForCausalLM"]), + ], +) +def test_unregistered_architecture_is_unsupported(args, architectures): + config = SimpleNamespace(architectures=architectures) + + with pytest.raises(ValueError, match="Unknown model architecture: FooBarForCausalLM") as e: + startup_checks.check_architecture_registered(args, config) + + info = classify_startup_error(e.value) + assert info.category == "unsupported" + assert "supported-models.html" in info.hint + + +@pytest.mark.parametrize( + "args, architectures", + [ + (_arch_args(), ["LlamaForCausalLM"]), + (_arch_args(), None), + (_arch_args(), []), + (_arch_args(checkpoint_format="mistral"), ["FooBarForCausalLM"]), + (_arch_args(mm_encoder_only=True), ["FooBarForCausalLM"]), + (_arch_args(encode_only=True), ["FooBarForCausalLM"]), + (_arch_args(model_kwargs={"architectures": ["LlamaForCausalLM"]}), ["FooBarForCausalLM"]), + (_arch_args(model_kwargs={"num_hidden_layers": 2}), ["LlamaForCausalLM"]), + (_arch_args(checkpoint_loader=object()), ["FooBarForCausalLM"]), + ], +) +def test_architecture_check_passes_or_is_skipped(args, architectures): + startup_checks.check_architecture_registered(args, SimpleNamespace(architectures=architectures)) + + +def test_architecture_check_skipped_without_config(): + startup_checks.check_architecture_registered(_arch_args(), None) diff --git a/tests/unittest/others/test_startup_errors.py b/tests/unittest/others/test_startup_errors.py new file mode 100644 index 000000000000..0bcde85a4cd7 --- /dev/null +++ b/tests/unittest/others/test_startup_errors.py @@ -0,0 +1,664 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Error categories, startup error classification, stage marking and hint logging (no GPU).""" + +import errno +import pickle +import subprocess +from unittest import mock + +import click +import pydantic +import pytest +import torch +import yaml + +from tensorrt_llm import _errors as errors +from tensorrt_llm import _startup_errors as se +from tensorrt_llm._startup import _StartupTimer +from tensorrt_llm._torch.pyexecutor.warmup_timer import _WarmupTimer + +pytestmark = pytest.mark.cpu_only + + +def _fake_exception(name: str, module: str, base: type = Exception, **attrs) -> BaseException: + """An instance of a stand-in for a third-party exception class, matched by name/module.""" + cls = type(name, (base,), {"__module__": module}) + exc = cls(f"{name} message") + for key, value in attrs.items(): + setattr(exc, key, value) + return exc + + +def _raise_from_module(module: str, exc: BaseException) -> BaseException: + """Raise ``exc`` from a frame whose module is ``module`` and return it with its traceback.""" + namespace = {"__name__": module} + exec(compile("def fail(exc):\n raise exc\n", f"<{module}>", "exec"), namespace) + try: + namespace["fail"](exc) + except BaseException as raised: # noqa: BLE001 - returning the raised exception + return raised + raise AssertionError("unreachable") + + +def _chained(outer: BaseException, inner: BaseException) -> BaseException: + try: + try: + raise inner + except BaseException as e: + raise outer from e + except BaseException as raised: # noqa: BLE001 - returning the raised exception + return raised + + +def _context(outer: BaseException, inner: BaseException) -> BaseException: + try: + try: + raise inner + except BaseException: + raise outer + except BaseException as raised: # noqa: BLE001 - returning the raised exception + return raised + + +class _StrictModel(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + size: int = 1 + + @pydantic.field_validator("size") + @classmethod + def _check_size(cls, value: int) -> int: + if value == 7: + raise ValueError("size=7 is not supported with paged context") + if value < 0: + raise ValueError("size must be non-negative") + return value + + +def _validation_error(**kwargs) -> pydantic.ValidationError: + with pytest.raises(pydantic.ValidationError) as excinfo: + _StrictModel(**kwargs) + return excinfo.value + + +@pytest.mark.parametrize( + "make_exc, category", + [ + # config_input + (lambda: _validation_error(unknown_key=1), "config_input"), + (lambda: _validation_error(size=-1), "config_input"), + (lambda: yaml.YAMLError("bad yaml"), "config_input"), + (lambda: click.BadParameter("bad flag"), "config_input"), + (lambda: ModuleNotFoundError("No module named 'grpc'"), "config_input"), + (lambda: _fake_exception("GatedRepoError", "huggingface_hub.errors"), "config_input"), + ( + lambda: _fake_exception("RepositoryNotFoundError", "huggingface_hub.errors"), + "config_input", + ), + ( + lambda: _fake_exception("LocalEntryNotFoundError", "huggingface_hub.errors"), + "config_input", + ), + (lambda: _fake_exception("ConnectionError", "requests.exceptions"), "config_input"), + # resource + (lambda: torch.OutOfMemoryError("CUDA out of memory"), "resource"), + (lambda: MemoryError(), "resource"), + (lambda: OSError(errno.EADDRINUSE, "Address already in use"), "resource"), + (lambda: OSError(errno.ENOSPC, "No space left on device"), "resource"), + (lambda: _fake_exception("AcceleratorError", "torch", error_code=2), "resource"), + (lambda: _fake_exception("AcceleratorError", "torch", error_code=35), "resource"), + (lambda: RuntimeError("[TensorRT-LLM][ERROR] CUDA_ERROR_OUT_OF_MEMORY"), "resource"), + (lambda: _fake_exception("AcceleratorError", "torch", error_code=46), "resource"), + ( + lambda: RuntimeError("CUDA error: CUDA-capable device(s) is/are busy or unavailable"), + "resource", + ), + # unsupported + (lambda: NotImplementedError("fp8 lm_head"), "unsupported"), + (lambda: ValueError("MTP is not supported with beam search"), "unsupported"), + (lambda: RuntimeError("Unknown model architecture: FooForCausalLM"), "unsupported"), + (lambda: ValueError("FP4 MLA does not support context parallelism."), "unsupported"), + ( + lambda: RuntimeError("DeepSeek-V4 requires Hopper or newer GPUs, got SM89"), + "unsupported", + ), + (lambda: RuntimeError("TriAttention requires an SM100-family device."), "unsupported"), + (lambda: ValueError("This kernel is supported on SM90 only."), "unsupported"), + (lambda: _fake_exception("AcceleratorError", "torch", error_code=209), "unsupported"), + (lambda: _validation_error(size=7), "unsupported"), + # model_load + (lambda: _fake_exception("SafetensorError", "builtins"), "model_load"), + (lambda: _fake_exception("EntryNotFoundError", "huggingface_hub.errors"), "model_load"), + ( + lambda: _raise_from_module( + "tensorrt_llm._torch.models.modeling_fake", KeyError("model.layers.0.weight") + ), + "model_load", + ), + ( + lambda: _raise_from_module("transformers.configuration_utils", ValueError("bad")), + "model_load", + ), + # runtime_init + (lambda: _fake_exception("Exception", "mpi4py.MPI"), "runtime_init"), + ( + lambda: RuntimeError("NCCL error in: comm.cpp:42, unhandled system error"), + "runtime_init", + ), + (lambda: subprocess.CalledProcessError(1, ["nvcc"]), "runtime_init"), + (lambda: _fake_exception("CompilationError", "triton.compiler.errors"), "runtime_init"), + ( + lambda: RuntimeError( + "Executor worker died during initialization: MPI worker exited unexpectedly" + ), + "runtime_init", + ), + # other + (lambda: RuntimeError("something odd"), "other"), + ( + lambda: _raise_from_module( + "tensorrt_llm._torch.pyexecutor.fake", AttributeError("'NoneType' has no 'x'") + ), + "other", + ), + (lambda: _fake_exception("AcceleratorError", "torch", error_code=700), "other"), + (lambda: _raise_from_module("my_plugin", ZeroDivisionError("division by zero")), "other"), + ], +) +def test_category(make_exc, category): + info = se.classify_startup_error(make_exc()) + assert info.category == category + assert info.category in errors.ERROR_CATEGORIES + if category != "other": + assert info.hint + + +@pytest.mark.parametrize( + "message, hint_key", + [ + ("CUDA error: CUDA-capable device(s) is/are busy or unavailable", "gpu_busy"), + ( + "Failed, NCCL error opUtils.cpp:179 'invalid usage (run with NCCL_DEBUG=WARN)'", + "distributed", + ), + ], +) +def test_gpu_contention_hints(message, hint_key): + assert se.classify_startup_error(RuntimeError(message)).hint == errors.ERROR_HINTS[hint_key] + + +@pytest.mark.parametrize( + "message", + [ + "[TensorRT-LLM][ERROR] NVRTC compilation failed", + "[TensorRT-LLM][ERROR] Assertion failed: Cannot parse NVCC version", + ], +) +def test_cpp_kernel_compilation_failure_is_jit(message): + info = se.classify_startup_error(RuntimeError(message)) + assert (info.category, info.reason) == ("runtime_init", "jit") + + +def test_only_supports_phrase_is_unsupported(): + # DeepGEMM's C++ SM gate, as raised before the custom op re-raises it as UnsupportedError. + exc = RuntimeError( + "DeepGEMM only supports Hopper (SM90) architectures, but current device compute " + "capability is 100." + ) + info = se.classify_startup_error(exc) + assert (info.category, info.reason) == ("unsupported", "unsupported") + + +def test_gpu_architecture_missing_from_build_is_unsupported(): + # The worker's BuildInfo.check_cuda_architecture_supported for a GPU the build left out. + exc = RuntimeError( + "GPU 0 (NVIDIA RTX PRO 6000 Blackwell Server Edition, compute capability 12.0) is not " + 'included in this TensorRT-LLM build, which was built for CUDA architectures "100". ' + 'Rebuild TensorRT-LLM with 120 included, e.g. with --cuda_architectures "100;120".' + ) + info = se.classify_startup_error(exc) + assert (info.category, info.reason) == ("unsupported", "unsupported") + assert info.hint == errors.ERROR_HINTS["unsupported"] + + +def test_too_few_mpi_ranks_is_a_launch_input_error(): + from tensorrt_llm.executor import GenerationExecutor + + with pytest.raises(ValueError, match="Cannot instantiate an executor requiring 4 ranks") as e: + GenerationExecutor.create(model_world_size=4, world_size=2) + + assert type(e.value) is errors.ConfigInputError + info = se.classify_startup_error(e.value) + assert info.category == "config_input" + assert info.summary.startswith("ConfigInputError: Cannot instantiate an executor") + assert "srun -n / mpirun -np" in info.summary + + +@pytest.mark.parametrize( + "module, category", + [ + ("tensorrt_llm.commands.serve", "config_input"), + ("tensorrt_llm.commands.eval", "config_input"), + ("tensorrt_llm.bench.benchmark.utils.general", "config_input"), + ("tensorrt_llm.llmapi.disagg_utils", "config_input"), + # bench tuning opens the checkpoint's weight shards. + ("tensorrt_llm.bench.tuning.dataclasses", "other"), + ("tensorrt_llm.llmapi.llm_utils", "other"), + ("tensorrt_llm._torch.models.checkpoints.hf.weight_loader", "model_load"), + ], +) +def test_missing_file_is_config_input_only_where_cli_code_opens_it(module, category): + missing = FileNotFoundError(errno.ENOENT, "No such file or directory", "/x/config.yaml") + info = se.classify_startup_error(_raise_from_module(module, missing)) + assert info.category == category + if category == "config_input": + assert info.hint == errors.ERROR_HINTS["config"] + + +def test_unknown_quant_algo_is_unsupported_but_an_invalid_config_value_is_config_input(): + from tensorrt_llm.models.modeling_utils import QuantConfig + from tensorrt_llm.quantization.mode import QuantAlgo + + # E.g. the quant_algo of a checkpoint quantized with a newer format. + with pytest.raises(ValueError, match="^'FP8_NEW' is not a valid QuantAlgo$") as e: + QuantAlgo("FP8_NEW") + assert type(e.value) is errors.UnsupportedError + assert se.classify_startup_error(e.value).category == "unsupported" + assert "FP8_NEW" not in QuantAlgo + assert QuantAlgo("FP8") is QuantAlgo.FP8 + # pydantic reports any error of _missing_ as its own enum validation error. + with pytest.raises(pydantic.ValidationError) as e: + QuantConfig(quant_algo="FP8_NEW") + assert se.classify_startup_error(e.value).category == "config_input" + + +def test_python_operand_type_error_is_not_unsupported(): + exc = _raise_from_module( + "tensorrt_llm._torch.pyexecutor.fake", TypeError("unsupported operand type(s) for +") + ) + info = se.classify_startup_error(exc) + assert (info.category, info.reason) == ("other", "internal") + + +@pytest.mark.parametrize( + "make_exc", + [ + lambda: _raise_from_module("tensorrt_llm._torch.models.fake", AssertionError("tp")), + lambda: _fake_exception("AcceleratorError", "torch", error_code=700), + ], +) +def test_internal_looking_errors_are_other_with_the_internal_hint(make_exc): + info = se.classify_startup_error(make_exc()) + assert (info.category, info.reason) == ("other", "internal") + assert info.hint == errors.ERROR_HINTS["internal"] + + +@pytest.mark.parametrize( + "make_exc, reason", + [ + (lambda: torch.OutOfMemoryError("CUDA out of memory"), "gpu_oom"), + (lambda: OSError(errno.EADDRINUSE, "Address already in use"), "port"), + (lambda: RuntimeError("NCCL error in: comm.cpp:42, unhandled system error"), "distributed"), + (lambda: errors.ConfigInputError("bad"), "config"), + (lambda: errors.ResourceError("no gpu"), ""), + (lambda: errors.RuntimeInitError("x", reason="worker_died"), "worker_died"), + (lambda: RuntimeError("[unsupported] x"), "unsupported"), + (lambda: RuntimeError("something odd"), ""), + ], +) +def test_reason_is_the_standard_hint_key(make_exc, reason): + info = se.classify_startup_error(make_exc()) + assert info.reason == reason + assert info.hint == errors.ERROR_HINTS.get(reason, "") + + +def test_runtime_init_default_hint_is_not_the_internal_hint(): + for exc in ( + errors.RuntimeInitError("x"), + RuntimeError("[runtime_init] x"), + ): + info = se.classify_startup_error(exc) + assert (info.category, info.reason) == ("runtime_init", "runtime_init") + assert info.hint == errors.ERROR_HINTS["runtime_init"] != errors.ERROR_HINTS["internal"] + + +def test_config_hint_uses_python_api_terms(): + assert "--" not in errors.ERROR_HINTS["config"] + assert "LLM arguments" in errors.ERROR_HINTS["config"] + + +def test_deepest_typed_cause_wins_and_is_summarized(): + gated = _fake_exception("GatedRepoError", "huggingface_hub.errors") + exc = _chained(ValueError("Could not find model locally or on the Hub"), gated) + + info = se.classify_startup_error(exc) + + assert info.category == "config_input" + assert "HF_TOKEN" in info.hint + assert info.summary == "GatedRepoError: GatedRepoError message" + + +@pytest.mark.parametrize( + "name, module", + [ + ("GatedRepoError", "huggingface_hub.errors"), + ("RepositoryNotFoundError", "huggingface_hub.errors"), + ("HfHubHTTPError", "huggingface_hub.errors"), + ("OfflineModeIsEnabled", "huggingface_hub.errors"), + ("HTTPStatusError", "httpx"), + ], +) +def test_hub_errors_share_one_hint(name, module): + exc = _chained(ValueError("Could not find model"), _fake_exception(name, module)) + info = se.classify_startup_error(exc) + assert (info.category, info.hint) == ("config_input", errors.ERROR_HINTS["hf"]) + + +def test_missing_file_in_found_repo_is_model_load_but_offline_miss_is_not(): + entry_cls = type("EntryNotFoundError", (Exception,), {"__module__": "huggingface_hub.errors"}) + local = _fake_exception("LocalEntryNotFoundError", "huggingface_hub.errors", base=entry_cls) + assert se.classify_startup_error(entry_cls("x")).category == "model_load" + assert se.classify_startup_error(local).category == "config_input" + + +def test_summary_drops_text_repeated_by_the_hint(): + exc = errors.ResourceError( + "No GPU is visible.\nPossible causes: a, b.", hint="Possible causes: a, b." + ) + assert se.classify_startup_error(exc).summary == "ResourceError: No GPU is visible." + + +def test_implicit_context_is_followed(): + # serve.py raises "Failed to bind socket" while handling the OSError. + exc = _context( + RuntimeError("Failed to bind socket"), OSError(errno.EADDRINUSE, "Address in use") + ) + info = se.classify_startup_error(exc) + assert info.category == "resource" + assert "--port" in info.hint + + +@pytest.mark.parametrize( + "cls, base", + [ + (errors.ConfigInputError, ValueError), + (errors.UnsupportedError, ValueError), + (errors.ResourceError, RuntimeError), + (errors.ModelLoadError, RuntimeError), + (errors.RuntimeInitError, RuntimeError), + ], +) +def test_typed_error_declares_category_and_keeps_builtin_base(cls, base): + exc = cls("went wrong") + assert isinstance(exc, base) + # The only exception base is the builtin; the shared mixin is not an exception type. + assert [b for b in cls.__mro__ if issubclass(b, BaseException)][1] is base + assert not issubclass(errors._CategorizedError, BaseException) + assert str(exc) == "went wrong" + info = se.classify_startup_error(exc) + assert info.category == cls.category + assert info.summary == f"{cls.__name__}: went wrong" + + +def test_any_exception_class_declares_a_category_with_an_attribute(): + class OptedIn(KeyError): + category = "resource" + + class InvalidCategory(RuntimeError): + category = "gpu" + + info = se.classify_startup_error(OptedIn("x")) + assert (info.category, info.reason, info.hint) == ("resource", "", "") + assert se.classify_startup_error(InvalidCategory("x")).category == "other" + + +def test_declared_category_beats_message_phrases(): + exc = errors.ConfigInputError( + "The following arguments are not supported by the PyTorch backend: ['x']" + ) + info = se.classify_startup_error(exc) + assert info.category == "config_input" + assert info.hint == errors.ERROR_HINTS["config"] + + +def test_declared_category_beats_typed_rules(): + # A raise site that wraps an error decides over the wrapped error's type. + inner = _chained(errors.ModelLoadError("shard unreadable"), OSError(errno.ENOSPC, "No space")) + exc = _chained(RuntimeError("outer"), inner) + assert se.classify_startup_error(exc).category == "model_load" + + +def test_explicit_hint_is_kept(): + exc = errors.ResourceError("no gpu", hint="Mount the GPUs.") + assert se.classify_startup_error(exc).hint == "Mount the GPUs." + + +def test_marker_declares_category_of_builtin_error(): + exc = RuntimeError("[model_load] Checkpoint has no quant_algo.") + info = se.classify_startup_error(exc) + assert info.category == "model_load" + assert info.hint == errors.ERROR_HINTS["model_load"] + assert info.summary == "RuntimeError: Checkpoint has no quant_algo." + + +def test_marker_beats_message_phrases(): + exc = RuntimeError("[config_input] ValueError: Option foo is not supported.") + assert se.classify_startup_error(exc).category == "config_input" + + +def test_marker_keeps_the_specific_hint_of_a_matching_phrase(): + # What the proxy sees from trtllm-llmapi-launch: only the forwarded text. + exc = RuntimeError( + "Remote MPI worker died: RuntimeError: Executor worker returned error on rank 1: " + "[resource] RuntimeError: CUDA Runtime API error: cudaErrorMemoryAllocation" + ) + info = se.classify_startup_error(exc) + assert (info.category, info.hint) == ("resource", errors.ERROR_HINTS["gpu_oom"]) + + +@pytest.mark.parametrize("message", ["[other] x", "[gpu] x", "[resource]x"]) +def test_unknown_or_malformed_marker_is_ignored(message): + assert se.classify_startup_error(RuntimeError(message)).category == "other" + + +def test_summary_is_one_bounded_line(): + exc = ValueError("line one\n line two\t" + "x" * 1000) + summary = se.classify_startup_error(exc).summary + assert "\n" not in summary + assert summary.startswith("ValueError: line one line two x") + assert summary.endswith("...") + assert len(summary) <= len("ValueError: ") + 300 + + +def test_classification_never_raises(): + class Unprintable(Exception): + def __str__(self): + raise RuntimeError("broken __str__") + + info = se.classify_startup_error(Unprintable()) + assert info.category == "other" + assert info.summary == "Unprintable" + + +def test_rank_is_recorded(): + assert se.classify_startup_error(RuntimeError("x"), rank=3).rank == 3 + + +def test_startup_timer_marks_innermost_phase(): + exc = RuntimeError("weights") + with pytest.raises(RuntimeError): + with _StartupTimer("executor_creation") as timer: + with timer.phase("model"): + with timer.phase("inner"): + raise exc + assert se.classify_startup_error(exc).stage == "executor_creation/inner" + + +def test_startup_timer_marks_timer_name_outside_phases(): + exc = RuntimeError("config") + with pytest.raises(RuntimeError): + with _StartupTimer("executor_creation"): + raise exc + assert se.classify_startup_error(exc).stage == "executor_creation" + + +def test_warmup_phase_beats_enclosing_startup_phase(): + exc = torch.OutOfMemoryError("CUDA out of memory") + with pytest.raises(torch.OutOfMemoryError): + with _StartupTimer("executor_creation") as timer: + with timer.phase("final_executor_creation"): + with _WarmupTimer(rank=0) as warmup: + with warmup.phase("cuda_graph_capture"): + raise exc + assert se.classify_startup_error(exc).stage == "warmup/cuda_graph_capture" + + +def test_metadata_survives_pickling(): + exc = errors.UnsupportedError("bad", hint="Use X.") + se.mark_startup_stage(exc, "executor_creation/model") + + restored = pickle.loads(pickle.dumps(exc)) + info = se.classify_startup_error(restored) + + assert type(restored) is errors.UnsupportedError + assert str(restored) == "bad" + assert (info.category, info.reason, info.hint, info.stage) == ( + "unsupported", + "unsupported", + "Use X.", + "executor_creation/model", + ) + worker_died = pickle.loads(pickle.dumps(errors.RuntimeInitError("x", reason="worker_died"))) + assert (worker_died.reason, worker_died.hint) == ( + "worker_died", + errors.ERROR_HINTS["worker_died"], + ) + assert pickle.loads(pickle.dumps(info)) == info + + +def test_worker_error_wrapper_carries_worker_classification(): + # What the proxy holds: the unpickled worker exception (cause chain and + # traceback gone) plus the classification the worker computed. + worker_exc = _raise_from_module( + "tensorrt_llm._torch.models.checkpoints.hf.weight_loader", KeyError("lm_head.weight") + ) + se.mark_startup_stage(worker_exc, "executor_creation/model") + info = se.classify_startup_error(worker_exc, rank=1) + unpickled = pickle.loads(pickle.dumps(worker_exc)) + + wrapper = _chained( + se.wrap_worker_startup_error("Executor worker returned error", info), unpickled + ) + + assert str(wrapper) == ( + "Executor worker returned error on rank 1 during executor_creation/model: " + "[model_load] KeyError: 'lm_head.weight'" + ) + assert wrapper.__cause__ is unpickled + proxy_info = se.classify_startup_error(wrapper) + assert proxy_info == info + assert proxy_info.category == "model_load" + # Forwarded as text only (trtllm-llmapi-launch), the marker keeps the category. + assert se.classify_startup_error(RuntimeError(str(wrapper))).category == "model_load" + + +@pytest.mark.parametrize( + "cls", + [ + errors.ConfigInputError, + errors.UnsupportedError, + errors.ResourceError, + errors.ModelLoadError, + errors.RuntimeInitError, + ], +) +def test_wrapper_marks_typed_worker_error_for_text_only_channels(cls): + # A typed error's message has no marker. The worker classifies it in-process and + # the wrapper writes the marker, so trtllm-llmapi-launch's forwarded text keeps it. + info = se.classify_startup_error(cls("went wrong"), rank=1) + wrapper = se.wrap_worker_startup_error("Executor worker returned error", info) + + assert str(wrapper) == ( + f"Executor worker returned error on rank 1: [{cls.category}] {cls.__name__}: went wrong" + ) + forwarded = RuntimeError(f"Remote MPI worker died: RuntimeError: {wrapper}") + assert se.classify_startup_error(forwarded).category == cls.category + + +def test_wrapper_without_worker_info_keeps_plain_message(): + assert str(se.wrap_worker_startup_error("Executor worker returned error", None)) == ( + "Executor worker returned error" + ) + + +def test_wrapper_of_unclassified_error_has_no_marker(): + info = se.StartupErrorInfo("other", "ZeroDivisionError: division by zero", rank=0) + assert str(se.wrap_worker_startup_error("Executor worker returned error", info)) == ( + "Executor worker returned error on rank 0: ZeroDivisionError: division by zero" + ) + + +def test_log_startup_error_logs_once_per_chain(): + exc = _chained(RuntimeError("outer"), torch.OutOfMemoryError("CUDA out of memory")) + with mock.patch.object(se.logger, "error") as log_error: + info = se.log_startup_error(exc, "LLM initialization failed") + assert se.log_startup_error(exc, "Command failed") is None + + assert info is not None and info.category == "resource" + log_error.assert_called_once() + line = log_error.call_args.args[0] + assert line.startswith("LLM initialization failed [resource]: OutOfMemoryError: ") + assert "Hint: GPU memory ran out." in line + + +def test_format_includes_rank_and_stage(): + info = se.StartupErrorInfo("resource", "OutOfMemoryError: oom", "Lower it.", "warmup/x", 2) + assert se.format_startup_error(info, "Command failed") == ( + "Command failed [resource] on rank 2 during warmup/x: OutOfMemoryError: oom Hint: Lower it." + ) + + +def test_log_startup_error_never_raises(): + with mock.patch.object(se.logger, "error", side_effect=RuntimeError("logger broken")): + assert se.log_startup_error(ValueError("x"), "Command failed") is None + + +@pytest.fixture +def fresh_process(monkeypatch): + """No LLM has initialized in this process yet.""" + monkeypatch.setattr(se, "_llm_initialized", False) + + +def _cli_error_lines(exc: BaseException) -> list: + """Run a failing command through the CLI boundary and return the ERROR lines it logged.""" + from tensorrt_llm.commands import _telemetry + + def fail(): + raise exc + + with ( + mock.patch.object(_telemetry.usage, "report_exit"), + mock.patch.object(_telemetry.usage, "get_observed_signal", return_value=0), + mock.patch.object(se.logger, "error") as log_error, + pytest.raises(type(exc)), + ): + _telemetry.run_with_terminal_reporting(fail) + return [call.args[0] for call in log_error.call_args_list] + + +def test_cli_boundary_classifies_startup_failures(fresh_process): + lines = _cli_error_lines(OSError(errno.EADDRINUSE, "Address already in use")) + assert len(lines) == 1 + assert lines[0].startswith("Command failed [resource]: OSError: ") + + +def test_cli_boundary_skips_failures_after_an_llm_initialized(fresh_process): + se.mark_llm_initialized() + assert se.llm_initialized() + + assert _cli_error_lines(torch.OutOfMemoryError("CUDA out of memory")) == [] + # Another LLM that fails to initialize is still classified at its own boundary. + with mock.patch.object(se.logger, "error") as log_error: + info = se.log_startup_error(ValueError("x"), "LLM initialization failed") + assert info is not None + log_error.assert_called_once() diff --git a/tests/unittest/usage/conftest.py b/tests/unittest/usage/conftest.py index e1ecad5231ec..a5a270b17470 100644 --- a/tests/unittest/usage/conftest.py +++ b/tests/unittest/usage/conftest.py @@ -91,3 +91,11 @@ def enable_telemetry(monkeypatch): "tensorrt_llm.usage.usage_lib._OPT_OUT_FILE", Path("/nonexistent/path/do_not_track"), ) + + +@pytest.fixture(autouse=True) +def isolated_startup_state_dir(monkeypatch, tmp_path): + """Keep the local startup failure count of every test out of the real home.""" + state_dir = tmp_path / "usage-state" + monkeypatch.setenv("TRTLLM_USAGE_STATE_DIR", str(state_dir)) + return state_dir diff --git a/tests/unittest/usage/test_cli_telemetry.py b/tests/unittest/usage/test_cli_telemetry.py index 4297327afd72..d34008501929 100644 --- a/tests/unittest/usage/test_cli_telemetry.py +++ b/tests/unittest/usage/test_cli_telemetry.py @@ -25,7 +25,9 @@ import click import pytest +from tensorrt_llm._errors import ERROR_HINTS, ConfigInputError from tensorrt_llm.commands import _telemetry +from tensorrt_llm.logger import logger from tensorrt_llm.usage import usage_lib from tensorrt_llm.usage.config import UsageContext @@ -164,7 +166,7 @@ def _captured_terminal_parameters(payloads): """Return the parameters from one fully serialized terminal event.""" assert len(payloads) == 1 payload = payloads[0] - assert payload["eventSchemaVer"] == "0.7" + assert payload["eventSchemaVer"] == "0.8" assert len(payload["events"]) == 1 event = payload["events"][0] assert event["name"] == "trtllm_exit_report" @@ -577,3 +579,175 @@ def fail_after_child_observation(): assert params["signalNumber"] == 9 assert params["component"] == "engine_worker" assert params["reportingSource"] == "supervisor" + + +class TestStartupErrorCategory: + """The CLI boundary records the category of startup failures for the exit report.""" + + @pytest.fixture(autouse=True) + def _no_llm_initialized(self, monkeypatch): + # Other tests in the same process may have initialized an LLM. + monkeypatch.setattr(_telemetry, "llm_initialized", lambda: False) + + @staticmethod + def _run(callback=None, args=("run",)): + cli = _make_cli(callback) + with pytest.raises((SystemExit, Exception)): + cli.main(args=list(args), prog_name="test-cli") + + @staticmethod + def _restart_process(): + usage_lib._HEARTBEAT_STOP.set() + usage_lib._SESSION = None + usage_lib._SESSION_DISABLED = False + usage_lib._HEARTBEAT_STOP = threading.Event() + + def test_click_usage_error_is_config_input(self, captured_exit_payloads): + self._run(args=["--invalid-option"]) + params = _captured_terminal_parameters(captured_exit_payloads) + assert params["lifecyclePhase"] == "cli_parsing" + assert params["errorCategory"] == "config_input" + assert (params["priorStartupFailures"], params["startupStateStatus"]) == (0, "new") + + @pytest.mark.parametrize( + "args", + [ + # serve() rejects --grpc-protocol without --grpc before the model is inspected. + ["dummy/model", "--enable_visual_gen", "--grpc-protocol", "openengine"], + # Click rejects the port before serve() runs. + ["dummy/model", "--enable_visual_gen", "--port", "invalid"], + ["serve", "dummy/model", "--visual_gen_args=vg.yaml", "--port", "invalid"], + ], + ) + def test_explicit_visual_gen_usage_error_reports_no_startup_fields( + self, captured_exit_payloads, isolated_startup_state_dir, args + ): + from tensorrt_llm.commands.serve import main as serve_main + + with pytest.raises(SystemExit) as exc: + serve_main(args=args) + assert exc.value.code == 2 + params = _captured_terminal_parameters(captured_exit_payloads) + assert params["errorCategory"] == "" + assert (params["priorStartupFailures"], params["startupStateStatus"]) == (0, "") + assert not isolated_startup_state_dir.exists() + + @pytest.mark.parametrize( + ("error", "category"), + [ + ( + lambda: ConfigInputError("Configuration file root must be a mapping."), + "config_input", + ), + (lambda: RuntimeError("unrecognized"), "other"), + ], + ) + def test_uncaught_startup_exception(self, captured_exit_payloads, error, category): + def fail(): + raise error() + + self._run(fail) + assert _captured_terminal_parameters(captured_exit_payloads)["errorCategory"] == category + + @pytest.mark.parametrize("command", ["serve", "disaggregated", "eval", "bench"]) + def test_missing_config_file_is_config_input( + self, tmp_path, monkeypatch, captured_exit_payloads, command + ): + """A config file that does not exist is invalid input and gets the config hint.""" + from tensorrt_llm.bench.benchmark.utils.general import get_settings + from tensorrt_llm.commands.eval import main as eval_main + from tensorrt_llm.commands.serve import main as serve_main + + missing = str(tmp_path / "missing.yaml") + run = { + "serve": lambda: serve_main(args=["dummy/model", "--config", missing]), + "disaggregated": lambda: serve_main(args=["disaggregated", "-c", missing]), + "eval": lambda: eval_main( + args=["--model", "dummy/model", "--config", missing, "gsm8k"] + ), + # trtllm-bench reads the file in get_settings, once the dataset is loaded. + "bench": lambda: _make_cli( + lambda: get_settings({"extra_llm_api_options": missing}, None, "dummy/model", None) + ).main(args=["run"]), + }[command] + # The disaggregated command creates its Prometheus directory before reading the file. + monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", str(tmp_path)) + with ( + patch("tensorrt_llm.commands.serve.get_is_diffusion_only_model", return_value=False), + patch.object(logger, "error") as log_error, + pytest.raises(FileNotFoundError), + ): + run() + + (line,) = [ + str(call.args[0]) + for call in log_error.call_args_list + if str(call.args[0]).startswith("Command failed") + ] + assert line.startswith( + "Command failed [config_input]: FileNotFoundError: [Errno 2] No such file or directory: " + ) + assert missing in line + assert line.endswith(f"Hint: {ERROR_HINTS['config']}") + params = _captured_terminal_parameters(captured_exit_payloads) + assert params["errorCategory"] == "config_input" + + def test_llm_category_takes_precedence(self, captured_exit_payloads): + def llm_failed_then_wrapped(): + usage_lib.record_llm_initialization_attempt(default_usage_context="cli_serve") + usage_lib.record_llm_initialization_failure("resource") + raise RuntimeError("serve failed") + + self._run(llm_failed_then_wrapped) + (payload,) = captured_exit_payloads + assert payload["events"][-1]["parameters"]["errorCategory"] == "resource" + + def test_failure_after_llm_initialized_resets_the_count( + self, captured_exit_payloads, isolated_startup_state_dir + ): + def fail_after_success(): + usage_lib.record_llm_initialization_attempt(default_usage_context="cli_serve") + usage_lib.record_llm_initialized() + raise RuntimeError("benchmark failed") + + self._run(fail_after_success) + params = _captured_terminal_parameters(captured_exit_payloads) + assert params["errorCategory"] == "" + (state_file,) = isolated_startup_state_dir.iterdir() + assert '"streak": 0' in state_file.read_text() + + @pytest.mark.parametrize("suppress_context", [False, True]) + def test_failing_system_exit_uses_its_cause(self, captured_exit_payloads, suppress_context): + def exit_after_error(): + try: + raise ConfigInputError("bad option") + except ConfigInputError: + if suppress_context: + raise SystemExit(1) from None + sys.exit(1) + + self._run(exit_after_error) + params = _captured_terminal_parameters(captured_exit_payloads) + assert params["terminationKind"] == "exception" + assert params["errorCategory"] == ("other" if suppress_context else "config_input") + + def test_signal_reports_no_category_and_keeps_the_count(self, captured_exit_payloads): + def stopped(): + raise _telemetry.SignalExit(signal.SIGTERM) + + self._run(stopped) + params = _captured_terminal_parameters(captured_exit_payloads) + assert params["terminationKind"] == "signal" + assert params["errorCategory"] == "" + + def test_restart_loop_counts_failed_starts(self, captured_exit_payloads): + def fail(): + raise ConfigInputError("bad option") + + priors = [] + for _ in range(4): + self._run(fail) + priors.append(captured_exit_payloads.pop()["events"][-1]["parameters"]) + self._restart_process() + assert [p["priorStartupFailures"] for p in priors] == [0, 1, 2, 3] + assert [p["startupStateStatus"] for p in priors] == ["new"] + ["persisted"] * 3 diff --git a/tests/unittest/usage/test_e2e_capture.py b/tests/unittest/usage/test_e2e_capture.py index 7053533df19f..721512523798 100644 --- a/tests/unittest/usage/test_e2e_capture.py +++ b/tests/unittest/usage/test_e2e_capture.py @@ -256,6 +256,13 @@ def fail_loading(self): ] for event in payload["events"]: _assert_event_matches_sms_schema(event) + terminal = payload["events"][1]["parameters"] + assert terminal["errorCategory"] == ( + "config_input" if failure == "config_validation" else "other" + ) + for event in payload["events"]: + assert event["parameters"]["priorStartupFailures"] == 0 + assert event["parameters"]["startupStateStatus"] == "new" params = payload["events"][0]["parameters"] meta = json.loads(params["llmApiConfigMetaJson"]) assert meta["report_context"] == "pre_initialization_exit" @@ -392,7 +399,7 @@ def test_initial_report_captured(self, capture_server, monkeypatch): assert set(features.keys()) == expected_keys # Schema version - assert payload["eventSchemaVer"] == "0.7" + assert payload["eventSchemaVer"] == "0.8" # Disagg fields present (may be empty strings) assert "disaggRole" in params @@ -405,7 +412,7 @@ def test_initial_report_captured(self, capture_server, monkeypatch): # heartbeat must contain the current counter snapshot for the same # process-scoped telemetry session. assert heartbeat_payload["sessionId"] == payload["sessionId"] - assert heartbeat_payload["eventSchemaVer"] == "0.7" + assert heartbeat_payload["eventSchemaVer"] == "0.8" heartbeat = heartbeat_payload["events"][0] assert heartbeat["name"] == "trtllm_heartbeat" _assert_lifecycle_snapshot( @@ -444,7 +451,7 @@ def test_cli_serve_context_e2e(self, capture_server, monkeypatch): payload = CaptureHandler.captured_payloads[0] event = payload["events"][0] params = event["parameters"] - assert payload["eventSchemaVer"] == "0.7" + assert payload["eventSchemaVer"] == "0.8" _assert_lifecycle_snapshot( params, active_instances=1, @@ -504,7 +511,7 @@ def test_direct_llm_process_exit_e2e(self, capture_server, monkeypatch): initial_payload = _wait_for_event("trtllm_initial_report", timeout=5) exit_payload = _wait_for_event("trtllm_exit_report", timeout=5) assert exit_payload["sessionId"] == initial_payload["sessionId"] - assert exit_payload["eventSchemaVer"] == "0.7" + assert exit_payload["eventSchemaVer"] == "0.8" terminal_events = [ payload @@ -520,6 +527,11 @@ def test_direct_llm_process_exit_e2e(self, capture_server, monkeypatch): assert params["signalNumber"] == 0 assert params["terminationKind"] == "unknown" assert params["lifecyclePhase"] == "serving" + assert params["errorCategory"] == "" + initial_params = initial_payload["events"][0]["parameters"] + for reported in (initial_params, params): + assert reported["priorStartupFailures"] == 0 + assert reported["startupStateStatus"] == "new" assert params["component"] == "llm" assert params["reportingSource"] == "self" _assert_lifecycle_snapshot( diff --git a/tests/unittest/usage/test_reporter.py b/tests/unittest/usage/test_reporter.py index 896dc511868b..253e18d67f6a 100644 --- a/tests/unittest/usage/test_reporter.py +++ b/tests/unittest/usage/test_reporter.py @@ -1442,7 +1442,8 @@ def blocking_send(payload): def track_thread(*args, **kwargs): thread = real_thread(*args, **kwargs) - terminal_threads.append(thread) + if thread.name == "trtllm-usage-terminal": + terminal_threads.append(thread) return thread started = time.monotonic() @@ -1634,3 +1635,426 @@ def call_report_exit(): assert results.count(True) == 1 assert len(sent) == 1 + + +# --------------------------------------------------------------------------- +# Startup failure category and local consecutive failure count +# --------------------------------------------------------------------------- + + +def _restart_process() -> None: + """Drop the session as a process exit would; the launch key stays the same.""" + usage_lib._HEARTBEAT_STOP.set() + session = usage_lib._SESSION + if session is not None and isinstance(session.terminal_thread, threading.Thread): + session.terminal_thread.join(timeout=2) + usage_lib._SESSION = None + usage_lib._SESSION_DISABLED = False + usage_lib._REPORTER_STARTED = False + usage_lib._REPORTER_ACTIVE = False + usage_lib._HEARTBEAT_STOP = threading.Event() + + +class TestStartupFailureReporting: + """errorCategory, priorStartupFailures and startupStateStatus on the wire.""" + + @pytest.fixture + def consults(self, monkeypatch): + """Record every state-file action while keeping the real behavior.""" + actions = [] + real_consult = usage_lib.startup_state.consult + + def spy(key, action, **kwargs): + actions.append(action) + return real_consult(key, action, **kwargs) + + monkeypatch.setattr(usage_lib.startup_state, "consult", spy) + return actions + + @staticmethod + def _report_exit(outcome, lifecycle_phase=None): + sent = [] + with patch.object(usage_lib, "_send_to_gxt", side_effect=sent.append): + usage_lib.report_exit(outcome, lifecycle_phase=lifecycle_phase) + return sent + + @staticmethod + def _failed_start(category="resource", kind="exception"): + """One process whose LLM constructor failed; returns the exit parameters.""" + assert usage_lib.record_llm_initialization_attempt(default_usage_context="cli_serve") + usage_lib.record_llm_initialization_failure(category) + sent = TestStartupFailureReporting._report_exit( + usage_lib.TerminalOutcome(kind, exit_code_known=True, exit_code=1) + ) + _restart_process() + return sent[0]["events"][-1]["parameters"] + + @pytest.mark.parametrize( + "kind,phase,source,failures,created,action,category", + [ + ("exception", "model_initialization", "self", 1, 0, "increment", "resource"), + ("exception", "config_validation", "self", 0, 0, "increment", "other"), + ("exception", "cli_parsing", "self", 0, 0, "increment", "other"), + ("worker_failure", "config_validation", "supervisor", 0, 0, "increment", ""), + ("timeout", "model_initialization", "self", 0, 0, "read", ""), + ("unknown", "model_initialization", "self", 1, 0, "increment", "resource"), + ("unknown", "model_initialization", "self", 0, 0, "read", ""), + ("signal", "model_initialization", "self", 1, 0, "read", ""), + ("clean", "model_initialization", "self", 1, 0, "read", ""), + ("exception", "unknown", "self", 0, 0, "read", ""), + ("exception", "serving", "self", 0, 0, "reset", ""), + ("signal", "serving", "self", 0, 1, "reset", ""), + ("exception", "model_initialization", "self", 1, 1, "reset", ""), + ], + ) + def test_exit_rules( + self, enable_telemetry, consults, kind, phase, source, failures, created, action, category + ): + assert usage_lib.apply_usage_session_config(default_usage_context="cli_serve") + if failures or created: + assert usage_lib.record_llm_initialization_attempt() + if failures: + usage_lib.record_llm_initialization_failure("resource") + if created: + assert usage_lib.record_llm_initialization_attempt() + assert usage_lib.record_llm_initialized() + outcome = usage_lib.TerminalOutcome(kind, reporting_source=source) + + sent = self._report_exit(outcome, lifecycle_phase=phase) + + params = sent[0]["events"][-1]["parameters"] + assert consults == [action] + assert params["errorCategory"] == category + assert params["priorStartupFailures"] == 0 + assert params["startupStateStatus"] == "new" + + def test_restart_loop_reports_buckets_until_success(self, enable_telemetry): + reports = [self._failed_start() for _ in range(5)] + assert [p["priorStartupFailures"] for p in reports] == [0, 1, 2, 3, 3] + assert [p["startupStateStatus"] for p in reports] == ["new"] + ["persisted"] * 4 + assert {p["errorCategory"] for p in reports} == {"resource"} + + assert usage_lib.record_llm_initialization_attempt() + assert usage_lib.record_llm_initialized() + sent = self._report_exit(usage_lib.TerminalOutcome("signal", signal_number=2)) + params = sent[0]["events"][-1]["parameters"] + assert (params["priorStartupFailures"], params["startupStateStatus"]) == (3, "persisted") + assert params["errorCategory"] == "" + _restart_process() + + assert self._failed_start()["priorStartupFailures"] == 0 + + def test_background_reporter_resets_once_per_process( + self, enable_telemetry, consults, isolated_startup_state_dir + ): + for _ in range(2): + self._failed_start() + assert usage_lib.record_llm_initialization_attempt() + assert usage_lib.record_llm_initialized() + + sent = [] + stop_event = threading.Event() + stop_event.set() + with ( + patch.object(usage_lib, "_send_to_gxt", side_effect=sent.append), + patch.object(usage_lib, "_HEARTBEAT_STOP", stop_event), + ): + usage_lib._background_reporter(None, None, "cli_serve") + initial = sent[0]["events"][0]["parameters"] + assert (initial["priorStartupFailures"], initial["startupStateStatus"]) == (2, "persisted") + (state_file,) = isolated_startup_state_dir.iterdir() + assert json.loads(state_file.read_text())["streak"] == 0 + + sent = self._report_exit(usage_lib.TerminalOutcome("exception"), "serving") + params = sent[0]["events"][-1]["parameters"] + assert (params["priorStartupFailures"], params["startupStateStatus"]) == (2, "persisted") + assert consults == ["increment", "increment", "reset"] + + def test_partial_initial_report_carries_the_same_state(self, enable_telemetry): + self._failed_start() + usage_lib._mark_llm_startup() + assert usage_lib.record_llm_initialization_attempt() + usage_lib.record_llm_initialization_failure("model_load") + + sent = self._report_exit(usage_lib.TerminalOutcome("exception", exit_code_known=True)) + initial, terminal = (event["parameters"] for event in sent[0]["events"]) + assert terminal["errorCategory"] == "model_load" + for params in (initial, terminal): + assert (params["priorStartupFailures"], params["startupStateStatus"]) == ( + 1, + "persisted", + ) + assert "errorCategory" not in initial + + def test_category_recording_rules(self, enable_telemetry): + assert usage_lib.record_llm_initialization_attempt() + usage_lib.record_llm_initialization_failure("not-a-category") + assert usage_lib._SESSION.snapshot()["errorCategory"] == "other" + usage_lib.record_llm_initialization_failure("resource") + usage_lib.record_startup_error_category("config_input") + assert usage_lib._SESSION.snapshot()["errorCategory"] == "resource" + + assert usage_lib.record_llm_initialization_attempt() + assert usage_lib.record_llm_initialized() + assert usage_lib._SESSION.snapshot()["errorCategory"] == "" + usage_lib.record_startup_error_category("config_input") + assert usage_lib._SESSION.snapshot()["errorCategory"] == "" + + def test_boundary_category_fills_an_empty_slot(self, enable_telemetry): + assert usage_lib.apply_usage_session_config(lifecycle_phase="config_validation") + usage_lib.record_startup_error_category("config_input") + sent = self._report_exit(usage_lib.TerminalOutcome("exception", exit_code_known=True)) + assert sent[0]["events"][-1]["parameters"]["errorCategory"] == "config_input" + + def test_visual_gen_session_reports_sentinels_without_file_access( + self, enable_telemetry, consults, isolated_startup_state_dir + ): + assert usage_lib.apply_usage_session_config(lifecycle_phase="config_validation") + usage_lib._mark_visual_gen() + usage_lib.record_startup_error_category("resource") + + sent = self._report_exit(usage_lib.TerminalOutcome("exception", exit_code_known=True)) + + params = sent[0]["events"][-1]["parameters"] + assert params["errorCategory"] == "" + assert (params["priorStartupFailures"], params["startupStateStatus"]) == (0, "") + assert consults == [] + assert not isolated_startup_state_dir.exists() + + def test_help_probe_and_non_reporting_rank_touch_no_file( + self, enable_telemetry, consults, isolated_startup_state_dir + ): + assert usage_lib.apply_usage_session_config(lifecycle_phase="cli_parsing") + assert self._report_exit(usage_lib.TerminalOutcome("clean", exit_code_known=True)) == [] + _restart_process() + + assert usage_lib.apply_usage_session_config(lifecycle_phase="config_validation") + with patch.object(usage_lib, "_is_reporting_rank", return_value=False): + assert self._report_exit(usage_lib.TerminalOutcome("exception")) == [] + + assert consults == [] + assert not isolated_startup_state_dir.exists() + + @pytest.mark.parametrize("opt_out", ["env", "config", "ci"]) + def test_opt_out_touches_no_file( + self, monkeypatch, enable_telemetry, consults, isolated_startup_state_dir, opt_out + ): + assert usage_lib.record_llm_initialization_attempt() + usage_lib.record_llm_initialization_failure("resource") + telemetry_config = None + if opt_out == "env": + monkeypatch.setenv("TRTLLM_NO_USAGE_STATS", "1") + elif opt_out == "config": + telemetry_config = SimpleNamespace(disabled=True, usage_context="cli_serve") + else: + monkeypatch.delenv("TRTLLM_USAGE_FORCE_ENABLED") + monkeypatch.setenv("CI", "true") + + sent = [] + with patch.object(usage_lib, "_send_to_gxt", side_effect=sent.append): + assert not usage_lib.report_exit( + usage_lib.TerminalOutcome("exception"), telemetry_config=telemetry_config + ) + usage_lib.report_usage(telemetry_config=telemetry_config) + usage_lib._report_process_exit() + + assert sent == [] + assert consults == [] + assert not isolated_startup_state_dir.exists() + + @pytest.mark.parametrize("hook", ["_collect_system_info", "_event_snapshot"]) + def test_late_opt_out_in_reporter_touches_no_file( + self, monkeypatch, enable_telemetry, consults, isolated_startup_state_dir, hook + ): + """Opt-out before ("read") or after ("reset") the reporter's snapshot.""" + assert usage_lib.record_llm_initialization_attempt(default_usage_context="cli_serve") + assert usage_lib.record_llm_initialized() + original = getattr(usage_lib, hook) + + def opt_out_after(*args, **kwargs): + result = original(*args, **kwargs) + usage_lib._deactivate_usage_session() + return result + + monkeypatch.setattr(usage_lib, hook, opt_out_after) + with patch.object(usage_lib, "_send_to_gxt") as send: + usage_lib._background_reporter(None, None, "cli_serve") + + send.assert_not_called() + assert consults == [] + assert not isolated_startup_state_dir.exists() + + def test_late_opt_out_on_exit_path_touches_no_file( + self, monkeypatch, enable_telemetry, consults, isolated_startup_state_dir + ): + assert usage_lib.record_llm_initialization_attempt(default_usage_context="cli_serve") + usage_lib.record_llm_initialization_failure("resource") + session = usage_lib._SESSION + original_claim = session.claim_terminal + + def claim_then_opt_out(*args, **kwargs): + result = original_claim(*args, **kwargs) + usage_lib._deactivate_usage_session() + return result + + monkeypatch.setattr(session, "claim_terminal", claim_then_opt_out) + assert self._report_exit(usage_lib.TerminalOutcome("exception", exit_code=1)) == [] + assert consults == [] + assert not isolated_startup_state_dir.exists() + + @staticmethod + def _assert_unavailable_exit(sent, elapsed): + """The failed start's report went out in time, with the state unavailable.""" + assert elapsed < usage_lib._TERMINAL_FLUSH_TIMEOUT + initial, terminal = (event["parameters"] for event in sent[0]["events"]) + assert terminal["errorCategory"] == "resource" + for params in (initial, terminal): + assert (params["priorStartupFailures"], params["startupStateStatus"]) == ( + 0, + "unavailable", + ) + + @staticmethod + def _join_state_threads(): + """Wait for the exit path's helper threads once the stalled consult is released.""" + for thread in threading.enumerate(): + if thread.name == "trtllm-usage-state": + thread.join(timeout=5) + assert not thread.is_alive() + + def test_exit_does_not_wait_for_stalled_state_io(self, monkeypatch, enable_telemetry): + monkeypatch.setattr(usage_lib, "bounded_gpu_fields", lambda deadline: {}) + release = threading.Event() + actions = [] + + def stalled_consult(key, action, **kwargs): + actions.append(action) + release.wait(timeout=5) + return 1, "persisted" + + monkeypatch.setattr(usage_lib.startup_state, "consult", stalled_consult) + assert usage_lib.record_llm_initialization_attempt(default_usage_context="cli_serve") + usage_lib.record_llm_initialization_failure("resource") + try: + started = time.monotonic() + sent = self._report_exit(usage_lib.TerminalOutcome("exception", exit_code=1)) + elapsed = time.monotonic() - started + finally: + release.set() + self._join_state_threads() + + self._assert_unavailable_exit(sent, elapsed) + assert actions == ["increment"] + + def test_exit_does_not_wait_for_reporter_holding_state_lock( + self, monkeypatch, enable_telemetry + ): + sent = [] + monkeypatch.setattr(usage_lib, "_send_to_gxt", sent.append) + entered = threading.Event() + release = threading.Event() + actions = [] + + def stalled_consult(key, action, **kwargs): + actions.append(action) + entered.set() + release.wait(timeout=5) + return 1, "persisted" + + monkeypatch.setattr(usage_lib.startup_state, "consult", stalled_consult) + assert usage_lib.record_llm_initialization_attempt(default_usage_context="cli_serve") + assert usage_lib.record_llm_initialized() + reporter = threading.Thread( + target=usage_lib._background_reporter, args=(None, None, "cli_serve") + ) + reporter.start() + try: + assert entered.wait(timeout=5) + started = time.monotonic() + assert usage_lib.report_exit( + usage_lib.TerminalOutcome("signal", signal_number=2), lifecycle_phase="serving" + ) + elapsed = time.monotonic() - started + finally: + release.set() + reporter.join(timeout=5) + self._join_state_threads() + + assert not reporter.is_alive() + assert actions == ["reset"] + assert elapsed < usage_lib._TERMINAL_FLUSH_TIMEOUT + # The reporter stops waiting too, so its initial report may also have gone out. + events = [event for payload in sent for event in payload["events"]] + assert "trtllm_exit_report" in [event["name"] for event in events] + for event in events: + params = event["parameters"] + assert (params["priorStartupFailures"], params["startupStateStatus"]) == ( + 0, + "unavailable", + ) + + @pytest.mark.parametrize("stall", ["file_io", "state_lock"]) + def test_initial_report_does_not_wait_for_stalled_state( + self, monkeypatch, enable_telemetry, stall + ): + release = threading.Event() + actions = [] + + def stalled_consult(key, action, **kwargs): + actions.append(action) + release.wait(timeout=5) + return 1, "persisted" + + monkeypatch.setattr(usage_lib.startup_state, "consult", stalled_consult) + assert usage_lib.record_llm_initialization_attempt(default_usage_context="cli_serve") + assert usage_lib.record_llm_initialized() + state_lock = usage_lib._SESSION.startup_state_lock + if stall == "state_lock": + state_lock.acquire() + sent = [] + stop_event = threading.Event() + stop_event.set() + try: + with ( + patch.object(usage_lib, "_send_to_gxt", side_effect=sent.append), + patch.object(usage_lib, "_HEARTBEAT_STOP", stop_event), + ): + usage_lib._background_reporter(None, None, "cli_serve") + finally: + if stall == "state_lock": + state_lock.release() + release.set() + self._join_state_threads() + + (payload,) = sent + initial = payload["events"][0] + assert initial["name"] == "trtllm_initial_report" + params = initial["parameters"] + assert (params["priorStartupFailures"], params["startupStateStatus"]) == (0, "unavailable") + # The late consult still records the successful start. + assert actions == ["reset"] + + def test_exit_without_a_state_thread_reports_unavailable( + self, monkeypatch, enable_telemetry, consults + ): + monkeypatch.setattr(usage_lib, "bounded_gpu_fields", lambda deadline: {}) + real_thread = threading.Thread + + def thread(*args, **kwargs): + if kwargs.get("name") == "trtllm-usage-state": + raise RuntimeError("can't create new thread at interpreter shutdown") + return real_thread(*args, **kwargs) + + assert usage_lib.record_llm_initialization_attempt(default_usage_context="cli_serve") + usage_lib.record_llm_initialization_failure("resource") + with patch.object(usage_lib.threading, "Thread", side_effect=thread): + started = time.monotonic() + sent = self._report_exit(usage_lib.TerminalOutcome("exception", exit_code=1)) + elapsed = time.monotonic() - started + + self._assert_unavailable_exit(sent, elapsed) + assert consults == [] + + def test_heartbeat_has_no_startup_fields(self): + fields = {f.alias for f in usage_lib.schema.TrtllmHeartbeat.model_fields.values()} + assert not fields & {"errorCategory", "priorStartupFailures", "startupStateStatus"} diff --git a/tests/unittest/usage/test_schema.py b/tests/unittest/usage/test_schema.py index c911ad02b75c..0aa34ec55436 100644 --- a/tests/unittest/usage/test_schema.py +++ b/tests/unittest/usage/test_schema.py @@ -15,11 +15,13 @@ """Tests for Pydantic schema models, GXT payload format, and SMS schema compliance.""" import json +from typing import get_args import pytest from pydantic import ValidationError -from tensorrt_llm.usage import schema, schemas +from tensorrt_llm import _errors +from tensorrt_llm.usage import schema, schemas, startup_state pytestmark = pytest.mark.cpu_only @@ -251,7 +253,7 @@ class TestSchemaConstants: [ ("CLIENT_ID", "616561816355034"), ("EVENT_PROTOCOL", "1.6"), - ("EVENT_SCHEMA_VER", "0.7"), + ("EVENT_SCHEMA_VER", "0.8"), ("EVENT_SYS_VER", "trtllm-telemetry/1.0"), ], ) @@ -484,6 +486,8 @@ def test_initial_report_has_all_expected_fields(self): "activeLlmInstances", "maxConcurrentLlmInstances", "llmInitializationFailures", + "priorStartupFailures", + "startupStateStatus", } missing = expected_fields - props @@ -654,6 +658,8 @@ def test_initial_report_validates_against_json_schema(self): llmApiConfigMetaJson='{"capture_succeeded":true}', disaggRole="", deploymentId="", + priorStartupFailures=3, + startupStateStatus="new", ) payload = report.model_dump(by_alias=True) initial_schema = sms_schema["definitions"]["events"]["trtllm_initial_report"].copy() @@ -685,6 +691,9 @@ def test_exit_report_validates_against_json_schema(self): component="server", reportingSource="self", ingressPoint="cli_serve", + errorCategory="resource", + priorStartupFailures=30, + startupStateStatus="persisted", ) payload = report.model_dump(by_alias=True) exit_schema = sms_schema["definitions"]["events"]["trtllm_exit_report"].copy() @@ -759,3 +768,57 @@ def test_lifecycle_counters_reject_out_of_range_values(self, field_name, invalid """Lifecycle counters enforce the unsigned 32-bit wire bounds.""" with pytest.raises(ValidationError): schema.TrtllmInitialReport(**{field_name: invalid_value}) + + +class TestStartupFailureFields: + """The startup failure fields are closed and match their sources.""" + + def _exit_report(self, **fields): + return schema.TrtllmExitReport( + exitCodeKnown=True, + exitCode=1, + signalNumber=0, + terminationKind="exception", + lifecyclePhase="model_initialization", + component="llm", + reportingSource="self", + **fields, + ) + + def test_error_categories_match_the_classifier(self): + assert get_args(schema.ErrorCategory) == ("",) + _errors.ERROR_CATEGORIES + + def test_prior_failure_buckets_match_the_state_module(self): + assert get_args(schema.PriorStartupFailures) == startup_state.PRIOR_STARTUP_FAILURE_BUCKETS + + def test_json_enums_match_the_models(self): + events = json.loads(schemas.SMS_SCHEMA_PATH.read_text())["definitions"]["events"] + exit_props = events["trtllm_exit_report"]["properties"] + assert tuple(exit_props["errorCategory"]["enum"]) == get_args(schema.ErrorCategory) + for event_name in ("trtllm_initial_report", "trtllm_exit_report"): + props = events[event_name]["properties"] + assert tuple(props["startupStateStatus"]["enum"]) == get_args(schema.StartupStateStatus) + assert props["priorStartupFailures"] == {"$ref": "#/definitions/types/PositiveInt"} + + def test_defaults_are_sentinels(self): + params = self._exit_report().model_dump(by_alias=True) + assert params["errorCategory"] == "" + assert params["priorStartupFailures"] == 0 + assert params["startupStateStatus"] == "" + + @pytest.mark.parametrize( + "fields", + [ + {"errorCategory": "internal"}, + {"errorCategory": "ValueError: bad"}, + {"priorStartupFailures": 4}, + {"priorStartupFailures": -1}, + {"startupStateStatus": "missing"}, + ], + ) + def test_rejects_values_outside_the_allowlist(self, fields): + with pytest.raises(ValidationError): + self._exit_report(**fields) + if "errorCategory" not in fields: + with pytest.raises(ValidationError): + schema.TrtllmInitialReport(**fields) diff --git a/tests/unittest/usage/test_startup_state.py b/tests/unittest/usage/test_startup_state.py new file mode 100644 index 000000000000..55f88255e08f --- /dev/null +++ b/tests/unittest/usage/test_startup_state.py @@ -0,0 +1,239 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the local consecutive startup failure count.""" + +import errno +import json +import os +import stat +import time +from pathlib import Path + +import pytest + +from tensorrt_llm.usage import startup_state + +pytestmark = pytest.mark.cpu_only + +_DAY = 24 * 3600 +_KEY = "0123456789abcdef" + + +def _state_file(state_dir: Path) -> Path: + return state_dir / f"startup-{_KEY}.json" + + +def _load(state_dir: Path) -> dict: + return json.loads(_state_file(state_dir).read_text()) + + +@pytest.mark.parametrize( + "streak,expected", + [ + (-1, 0), + (0, 0), + (1, 1), + (2, 2), + (3, 3), + (4, 3), + (9, 3), + (10, 10), + (29, 10), + (30, 30), + (99, 30), + (100, 100), + (299, 100), + (300, 300), + (999, 300), + (1000, 1000), + (10**6, 1000), + ], +) +def test_bucket_lower_bounds(streak, expected): + assert startup_state.bucket(streak) == expected + + +class TestLaunchKey: + _ENV = {"CUDA_VISIBLE_DEVICES": "0,1"} + + def _key(self, argv=("trtllm-serve", "model"), environ=None, version="1.4.0"): + return startup_state.launch_key( + version, argv=list(argv), environ=self._ENV if environ is None else environ + ) + + def test_is_deterministic_and_reveals_no_value(self): + key = self._key() + assert key == self._key() + assert len(key) == 16 and int(key, 16) >= 0 + for value in ("trtllm-serve", "model", "0,1", "1.4.0"): + assert value not in key + + @pytest.mark.parametrize( + "name", + [ + "SLURM_PROCID", + "OMPI_COMM_WORLD_RANK", + "PMI_RANK", + "PMIX_RANK", + "CUDA_VISIBLE_DEVICES", + "NVIDIA_VISIBLE_DEVICES", + "TLLM_DISAGG_INSTANCE_IDX", + "TRTLLM_DISAGG_WORKER_PROCESS_ID", + "TLLM_EXECUTOR_FRONTEND_ID", + ], + ) + def test_separates_launch_variables(self, name): + assert self._key(environ={**self._ENV, name: "7"}) != self._key() + + def test_separates_versions_and_command_lines(self): + assert self._key(version="1.4.1") != self._key() + assert self._key(argv=("trtllm-serve", "model", "--port", "8001")) != self._key() + + def test_ignores_per_start_and_unrelated_variables(self): + environ = { + **self._ENV, + "SLURM_JOB_ID": "123", + "TRTLLM_DISAGG_DEPLOYMENT_ID": "random", + "HOME": "/x", + } + assert self._key(environ=environ) == self._key() + + +class TestStateDir: + def test_precedence(self, monkeypatch, tmp_path): + environ = {"TRTLLM_USAGE_STATE_DIR": str(tmp_path / "a"), "XDG_CACHE_HOME": "/cache"} + assert startup_state.state_dir(environ) == tmp_path / "a" + assert startup_state.state_dir({"XDG_CACHE_HOME": "/cache"}) == Path( + "/cache/tensorrt_llm/usage" + ) + monkeypatch.setattr(Path, "home", lambda: Path("/home/u")) + assert startup_state.state_dir({"XDG_CACHE_HOME": "relative"}) == Path( + "/home/u/.cache/tensorrt_llm/usage" + ) + + def test_missing_home_is_unavailable(self, monkeypatch): + def no_home(): + raise RuntimeError("no home") + + monkeypatch.delenv("TRTLLM_USAGE_STATE_DIR") + monkeypatch.delenv("XDG_CACHE_HOME", raising=False) + monkeypatch.setattr(Path, "home", no_home) + assert startup_state.state_dir() is None + assert startup_state.consult(_KEY, "increment") == (0, "unavailable") + + +class TestConsult: + def test_increment_creates_private_state(self, isolated_startup_state_dir): + state_dir = isolated_startup_state_dir + assert startup_state.consult(_KEY, "increment", now=1000.0) == (0, "new") + assert _load(state_dir) == {"v": 1, "streak": 1, "last_failure": 1000} + assert stat.S_IMODE(os.stat(state_dir).st_mode) == 0o700 + assert stat.S_IMODE(os.stat(_state_file(state_dir)).st_mode) == 0o600 + + assert startup_state.consult(_KEY, "increment", now=1010.0) == (1, "persisted") + assert startup_state.consult(_KEY, "increment", now=1020.0) == (2, "persisted") + assert _load(state_dir)["streak"] == 3 + assert sorted(os.listdir(state_dir)) == [_state_file(state_dir).name] + + def test_reset_returns_prior_and_writes_zero(self, isolated_startup_state_dir): + state_dir = isolated_startup_state_dir + for _ in range(4): + startup_state.consult(_KEY, "increment", now=1000.0) + assert startup_state.consult(_KEY, "reset", now=1001.0) == (4, "persisted") + assert _load(state_dir) == {"v": 1, "streak": 0, "last_failure": 0} + assert startup_state.consult(_KEY, "increment", now=1002.0) == (0, "persisted") + + def test_reset_of_a_new_launch_writes_the_file(self, isolated_startup_state_dir): + assert startup_state.consult(_KEY, "reset") == (0, "new") + assert _load(isolated_startup_state_dir)["streak"] == 0 + assert startup_state.consult(_KEY, "read") == (0, "persisted") + + def test_read_never_writes(self, isolated_startup_state_dir): + state_dir = isolated_startup_state_dir + assert startup_state.consult(_KEY, "read") == (0, "new") + assert not state_dir.exists() + + startup_state.consult(_KEY, "increment", now=1000.0) + before = _state_file(state_dir).stat() + assert startup_state.consult(_KEY, "read", now=1001.0) == (1, "persisted") + after = _state_file(state_dir).stat() + assert (after.st_mtime_ns, after.st_ino) == (before.st_mtime_ns, before.st_ino) + + def test_old_streak_expires(self, isolated_startup_state_dir): + now = time.time() + startup_state.consult(_KEY, "increment", now=now - 2 * _DAY) + assert startup_state.consult(_KEY, "read", now=now) == (0, "persisted") + assert startup_state.consult(_KEY, "increment", now=now) == (0, "persisted") + assert _load(isolated_startup_state_dir)["streak"] == 1 + + @pytest.mark.parametrize( + "content", + [ + b"not json", + b"[]", + b'{"v": 2, "streak": 3, "last_failure": 0}', + b'{"v": 1, "streak": -1, "last_failure": 0}', + b'{"v": 1, "streak": "3", "last_failure": 0}', + b'{"v": 1, "streak": true, "last_failure": 0}', + b'{"v": 1, "streak": 3}', + b'{"v": 1, "streak": 3, "last_failure": 0, "pad": "' + b"x" * 5000 + b'"}', + ], + ) + def test_corrupt_state_restarts_from_zero(self, isolated_startup_state_dir, content): + state_dir = isolated_startup_state_dir + state_dir.mkdir() + _state_file(state_dir).write_bytes(content) + assert startup_state.consult(_KEY, "increment") == (0, "new") + assert _load(state_dir)["streak"] == 1 + + def test_unusable_directory_is_unavailable(self, monkeypatch, tmp_path): + # A directory below a regular file fails with ENOTDIR, also for root. + blocker = tmp_path / "file" + blocker.write_text("") + monkeypatch.setenv("TRTLLM_USAGE_STATE_DIR", str(blocker / "state")) + for action in ("read", "increment", "reset"): + assert startup_state.consult(_KEY, action) == (0, "unavailable") + + def test_failed_write_keeps_prior_and_leaves_no_temporary_file( + self, monkeypatch, isolated_startup_state_dir + ): + state_dir = isolated_startup_state_dir + startup_state.consult(_KEY, "increment", now=1000.0) + + def read_only(*args, **kwargs): + raise OSError(errno.EROFS, "Read-only file system") + + monkeypatch.setattr(startup_state.os, "replace", read_only) + assert startup_state.consult(_KEY, "increment", now=1001.0) == (1, "unavailable") + assert sorted(os.listdir(state_dir)) == [_state_file(state_dir).name] + assert _load(state_dir)["streak"] == 1 + + def test_unexpected_error_is_swallowed(self, monkeypatch): + def broken(*args, **kwargs): + raise ValueError("unexpected") + + monkeypatch.setattr(startup_state, "_read", broken) + assert startup_state.consult(_KEY, "increment") == (0, "unavailable") + + def test_empty_key_is_unavailable(self, isolated_startup_state_dir): + assert startup_state.consult("", "increment") == (0, "unavailable") + assert not isolated_startup_state_dir.exists() + + def test_new_file_prunes_only_old_state_files(self, isolated_startup_state_dir): + state_dir = isolated_startup_state_dir + state_dir.mkdir() + old = time.time() - 31 * _DAY + names = { + "startup-aaaaaaaaaaaaaaaa.json": old, + ".startup-left.tmp": old, + "startup-bbbbbbbbbbbbbbbb.json": time.time(), + "user-notes.json": old, + } + for name, mtime in names.items(): + (state_dir / name).write_text("{}") + os.utime(state_dir / name, (mtime, mtime)) + + startup_state.consult(_KEY, "increment") + assert sorted(os.listdir(state_dir)) == sorted( + [_state_file(state_dir).name, "startup-bbbbbbbbbbbbbbbb.json", "user-notes.json"] + ) diff --git a/tests/unittest/usage/test_transport.py b/tests/unittest/usage/test_transport.py index 4df02b0f6bbb..f3d6b989fa2e 100644 --- a/tests/unittest/usage/test_transport.py +++ b/tests/unittest/usage/test_transport.py @@ -292,3 +292,81 @@ def test_ingress_point_in_accepted_payload(self): status = _post_to_staging(payload) assert status == 200, f"Staging rejected ingressPoint={context_value!r}: HTTP {status}" + + @staticmethod + def _startup_exit_report(category, prior, state_status): + return schema.TrtllmExitReport( + exitCodeKnown=True, + exitCode=1, + signalNumber=0, + terminationKind="exception", + lifecyclePhase="model_initialization", + component="server", + reportingSource="self", + ingressPoint="cli_serve", + llmInitializationAttempts=1, + llmInitializationFailures=1, + errorCategory=category, + priorStartupFailures=prior, + startupStateStatus=state_status, + ) + + def test_exit_report_startup_fields_accepted(self): + """Staging accepts every errorCategory and startupStateStatus value.""" + import uuid + from typing import get_args + + categories = get_args(schema.ErrorCategory) + statuses = get_args(schema.StartupStateStatus) + priors = (0, 3, 1000) + for i, category in enumerate(categories): + report = self._startup_exit_report( + category, priors[i % len(priors)], statuses[i % len(statuses)] + ) + payload = schema.build_gxt_payload( + event=report, + session_id=uuid.uuid4().hex, + trtllm_version="0.0.0-test", + ) + + status = _post_to_staging(payload) + assert status == 200, f"Staging rejected {report!r}: HTTP {status}" + + def test_partial_initial_report_with_exit_report_accepted(self): + """Staging accepts a failed start's partial initial report sent with its exit report.""" + import json + import platform as plat + import uuid + + payload = schema.build_gxt_payload( + event=self._startup_exit_report("resource", 3, "persisted"), + session_id=uuid.uuid4().hex, + trtllm_version="0.0.0-test", + ) + terminal = payload["events"][0] + initial = schema.TrtllmInitialReport( + trtllmVersion="0.0.0-test", + pythonVersion=plat.python_version(), + cpuArchitecture=plat.machine(), + backend="pytorch", + tensorParallelSize=2, + pipelineParallelSize=0, + contextParallelSize=0, + llmApiConfigMetaJson=json.dumps( + {"report_context": "pre_initialization_exit", "source": "unavailable"} + ), + ingressPoint="cli_serve", + llmInitializationAttempts=1, + llmInitializationFailures=1, + priorStartupFailures=3, + startupStateStatus="persisted", + ) + context_event = schema.GxtEvent( + ts=terminal["ts"], + name="trtllm_initial_report", + parameters=initial.model_dump(by_alias=True), + ).model_dump(by_alias=True) + payload["events"] = [context_event, terminal] + + status = _post_to_staging(payload) + assert status == 200, f"Expected HTTP 200, got {status}" From 6630b1040dc5e16d6e9c7e891815806e059cc89e Mon Sep 17 00:00:00 2001 From: Zhenhua Wang Date: Sun, 8 Nov 2026 22:04:00 -0800 Subject: [PATCH 2/2] [None][fix] Allow FP8 block scales on Ada and share the architecture lookup Address review feedback on the startup checks: - FP8 block scales and FP8 per-channel per-token now require SM89 instead of SM90. Their GEMMs dispatch on Ada, so dense checkpoints such as Qwen3-8B-FP8 on L40S were rejected before loading. - Which module provides an architecture is now resolved in one place, _model_arch_provider() in modeling_utils.py: external declarations first, then the static index. _ensure_model_registered() imports what it returns, and the new is_model_arch_known() checks the registry and then that same function without importing. The startup architecture check and trtllm-serve's has_registered_llm_architecture() both call is_model_arch_known(), so a new provider source only has to be added to _model_arch_provider() and every caller follows. Signed-off-by: Zhenhua Wang --- tensorrt_llm/_torch/models/modeling_utils.py | 34 +++++++++++++++++--- tensorrt_llm/commands/utils.py | 14 +++----- tensorrt_llm/llmapi/startup_checks.py | 19 ++++++----- tests/unittest/llmapi/test_startup_checks.py | 20 ++++++++++-- 4 files changed, 59 insertions(+), 28 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_utils.py b/tensorrt_llm/_torch/models/modeling_utils.py index f28059fa7196..1d2a4d267c2c 100755 --- a/tensorrt_llm/_torch/models/modeling_utils.py +++ b/tensorrt_llm/_torch/models/modeling_utils.py @@ -1206,6 +1206,21 @@ def register_external_model_modules(arch_to_module: Dict[str, str]) -> None: _rebuild_arch_registries(declared) +def _model_arch_provider(model_arch: str) -> Optional[str]: + """The module to import for ``model_arch``, or ``None`` if no source lists it. + + External declarations take precedence over the static index of built-in + architectures. Imports nothing; ``_ensure_model_registered`` and + ``is_model_arch_known`` both resolve providers through this function. + """ + full_name = _EXTERNAL_ARCH_TO_MODULE.get(model_arch) + if full_name is None: + module_name = MODEL_ARCH_TO_MODULE.get(model_arch) + if module_name is not None: + full_name = f"tensorrt_llm._torch.models.{module_name}" + return full_name + + def _ensure_model_registered(model_arch: str) -> None: """Import the module that provides ``model_arch``, if it isn't loaded yet. @@ -1226,12 +1241,9 @@ def _ensure_model_registered(model_arch: str) -> None: registration decorator; the import itself is idempotent via ``sys.modules``. """ - full_name = _EXTERNAL_ARCH_TO_MODULE.get(model_arch) + full_name = _model_arch_provider(model_arch) if full_name is None: - module_name = MODEL_ARCH_TO_MODULE.get(model_arch) - if module_name is None: - return - full_name = f"tensorrt_llm._torch.models.{module_name}" + return try: importlib.import_module(full_name) except ModuleNotFoundError as e: @@ -1258,6 +1270,18 @@ def get_registered_model_class(model_arch: str) -> Optional[Type[nn.Module]]: return MODEL_CLASS_MAPPING.get(model_arch) +def is_model_arch_known(model_arch: str) -> bool: + """Whether ``get_registered_model_class`` has a source for ``model_arch``. + + Checks the registry and then ``_model_arch_provider``, the same steps as + the lookup, but imports nothing, so it is cheap enough for checks that run + before the model is built. A provider whose module later fails to import + still counts as known; that error surfaces where the model is built. + """ + return (model_arch in MODEL_CLASS_MAPPING + or _model_arch_provider(model_arch) is not None) + + def _is_builtin_draft_model_builder(builder) -> bool: return is_builtin_zoo_module(getattr(builder, "__module__", "")) diff --git a/tensorrt_llm/commands/utils.py b/tensorrt_llm/commands/utils.py index 011f772d9145..a0c2fa97a9d8 100644 --- a/tensorrt_llm/commands/utils.py +++ b/tensorrt_llm/commands/utils.py @@ -91,16 +91,10 @@ def has_registered_llm_architecture(model_path: str) -> bool: with open(config_path) as f: architectures = json.load(f).get("architectures") or [] - # The model zoo is imported lazily, so MODEL_CLASS_MAPPING only holds what - # has been resolved so far; the static index knows every built-in - # architecture without importing anything. Imported here (not at module - # load) to sidestep circular imports via commands.serve. - from tensorrt_llm._torch.models._arch_index import MODEL_ARCH_TO_MODULE - from tensorrt_llm._torch.models.modeling_utils import MODEL_CLASS_MAPPING - - return any( - arch in MODEL_CLASS_MAPPING or arch in MODEL_ARCH_TO_MODULE for arch in architectures - ) + # Imported here (not at module load) to sidestep circular imports via commands.serve. + from tensorrt_llm._torch.models.modeling_utils import is_model_arch_known + + return any(is_model_arch_known(arch) for arch in architectures) def get_is_diffusion_only_model(model_path: str): diff --git a/tensorrt_llm/llmapi/startup_checks.py b/tensorrt_llm/llmapi/startup_checks.py index 867f3c100add..34e0d4cd147e 100644 --- a/tensorrt_llm/llmapi/startup_checks.py +++ b/tensorrt_llm/llmapi/startup_checks.py @@ -22,13 +22,15 @@ from tensorrt_llm.quantization.mode import QuantAlgo # Lowest architecture marked as supported in the "Hardware Support Matrix" of -# docs/source/features/quantization.md. Formats missing from the matrix are not -# checked, and neither are KV cache formats: attention kernels dequantize the KV -# cache before the MMA, so its support depends on the kernel, not on a lower bound. +# docs/source/features/quantization.md, except FP8 block scales and FP8 per-channel +# per-token: their GEMMs also dispatch on SM89 (Ada), which the matrix does not list. +# Formats missing from the matrix are not checked, and neither are KV cache formats: +# attention kernels dequantize the KV cache before the MMA, so its support depends on +# the kernel, not on a lower bound. _MIN_SM_BY_QUANT_ALGO = { QuantAlgo.FP8: 89, - QuantAlgo.FP8_BLOCK_SCALES: 90, - QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN: 90, + QuantAlgo.FP8_BLOCK_SCALES: 89, + QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN: 89, QuantAlgo.W4A8_AWQ: 89, QuantAlgo.NVFP4: 100, QuantAlgo.W4A8_MXFP4_MXFP8: 100, @@ -173,12 +175,9 @@ def check_architecture_registered( ) if not architectures: return - # The static index lists every built-in architecture without importing the model - # zoo; MODEL_CLASS_MAPPING adds custom models registered in this process. - from tensorrt_llm._torch.models._arch_index import MODEL_ARCH_TO_MODULE - from tensorrt_llm._torch.models.modeling_utils import MODEL_CLASS_MAPPING + from tensorrt_llm._torch.models.modeling_utils import is_model_arch_known - if any(arch in MODEL_CLASS_MAPPING or arch in MODEL_ARCH_TO_MODULE for arch in architectures): + if any(is_model_arch_known(arch) for arch in architectures): return raise UnsupportedError( f"Unknown model architecture: {architectures[0]}. TensorRT-LLM has no " diff --git a/tests/unittest/llmapi/test_startup_checks.py b/tests/unittest/llmapi/test_startup_checks.py index 02f61ad38948..657d2f4351b4 100644 --- a/tests/unittest/llmapi/test_startup_checks.py +++ b/tests/unittest/llmapi/test_startup_checks.py @@ -132,8 +132,8 @@ def _quant_args(quant_algo=None, kv_cache_quant_algo=None, nvfp4_backends=None): "sm, quant_algo, kv_cache_quant_algo, backends", [ (80, QuantAlgo.FP8, None, None), - (89, QuantAlgo.FP8_BLOCK_SCALES, None, None), - (89, QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN, None, None), + (80, QuantAlgo.FP8_BLOCK_SCALES, None, None), + (80, QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN, None, None), (90, QuantAlgo.NVFP4, None, None), (80, QuantAlgo.NVFP4, None, ["marlin"]), (90, QuantAlgo.W4A8_MXFP4_MXFP8, None, None), @@ -159,7 +159,9 @@ def test_quantization_below_minimum_is_unsupported( "sm, quant_algo, kv_cache_quant_algo, backends", [ (89, QuantAlgo.FP8, None, None), - (90, QuantAlgo.FP8_BLOCK_SCALES, None, None), + # FP8 block-scale and rowwise GEMMs dispatch on Ada (SM89). + (89, QuantAlgo.FP8_BLOCK_SCALES, None, None), + (89, QuantAlgo.FP8_PER_CHANNEL_PER_TOKEN, None, None), (100, QuantAlgo.NVFP4, None, None), (90, QuantAlgo.NVFP4, None, ["cutlass", "marlin"]), (120, QuantAlgo.NVFP4, QuantAlgo.FP8, None), @@ -247,3 +249,15 @@ def test_architecture_check_passes_or_is_skipped(args, architectures): def test_architecture_check_skipped_without_config(): startup_checks.check_architecture_registered(_arch_args(), None) + + +def test_externally_declared_architecture_passes(monkeypatch): + # A declared external provider resolves on lookup even before its module is imported. + from tensorrt_llm._torch.models import modeling_utils + + monkeypatch.setitem( + modeling_utils._EXTERNAL_ARCH_TO_MODULE, "FooBarForCausalLM", "custom_models.foo_bar" + ) + startup_checks.check_architecture_registered( + _arch_args(), SimpleNamespace(architectures=["FooBarForCausalLM"]) + )