[TRTLLM-17272][feat] Add public LLM health queries and a private-access lint for the serving frontends

tensorrt_llm/executor/executor.py

@@ -341,6 +341,16 @@ def is_shutdown(self) -> bool:
         """Return True if the executor is shutting down or fatally errored."""
         return self.doing_shutdown or self._fatal_error is not None
 
+    @property
+    def fatal_error(self) -> Optional[BaseException]:
+        """The unrecoverable engine error recorded by ``_set_fatal_error``, or None."""
+        return self._fatal_error
+
+    @property
+    def engine_dead(self) -> bool:
+        """Whether the engine is known dead (an unrecoverable error was recorded)."""
+        return self._fatal_error is not None
+
     def check_health(self) -> bool:
         """Check whether the executor is healthy and able to process requests.
 

tensorrt_llm/executor/proxy.py

@@ -485,6 +485,15 @@ def check_health(self) -> bool:
 
         return True
 
+    @property
+    def engine_dead(self) -> bool:
+        """Whether the engine is known dead (a worker died or a fatal error was recorded).
+
+        ``_mark_engine_dead`` can set the sticky flag before the fatal error
+        is recorded, so check both.
+        """
+        return self._engine_dead or self._fatal_error is not None
+
     def _set_fatal_error(self, error: BaseException) -> None:
         """Record the fatal error, then unblock every pending request.
 

tensorrt_llm/grpc/openengine/control.py

@@ -140,23 +140,19 @@ async def _get_kv_cache_load(self) -> dict:
     def _engine_is_healthy(self) -> bool:
         """Readiness via the predicate the engine's own HTTP /health uses.
 
-        `_check_health` is stronger than the `is_shutdown()` guard generate_async
+        `check_health()` is stronger than the `is_shutdown()` guard generate_async
         raises on: it drains the executor's error queue, so a fatal error queued
         but not yet promoted is caught, and the proxy polls MPI worker liveness.
-        It also covers encode-only engines, where `_executor` is None.
+        It also covers encode-only engines, where there is no generation executor.
         """
-        check = getattr(self._llm, "_check_health", None)
-        if callable(check):
-            try:
-                return bool(check())
-            except (RuntimeError, ValueError, OSError) as error:
-                logger.warning(f"OpenEngine health check reported a failure: {error}")
-                return False
-            except (TypeError, AttributeError) as error:
-                logger.error(f"OpenEngine health check contract failed: {error}")
-                return False
-        executor = getattr(self._llm, "_executor", None)
-        return executor is not None and not executor.is_shutdown()
+        try:
+            return bool(self._llm.check_health())
+        except (RuntimeError, ValueError, OSError) as error:
+            logger.warning(f"OpenEngine health check reported a failure: {error}")
+            return False
+        except (TypeError, AttributeError) as error:
+            logger.error(f"OpenEngine health check contract failed: {error}")
+            return False
 
     # -- Runtime metadata ---------------------------------------------------
 

tensorrt_llm/grpc/openengine/server.py

@@ -225,11 +225,7 @@ async def _refresh_inference_health(self) -> None:
                 except (RuntimeError, ValueError, OSError, CoordinationError) as error:
                     logger.warning(f"OpenEngine inference health check failed: {error}")
                     self._engine_healthy = False
-                executor = getattr(self._llm, "_executor", None)
-                if executor is not None and (
-                    getattr(executor, "_fatal_error", None) is not None
-                    or getattr(executor, "_engine_dead", False)
-                ):
+                if self._llm.engine_dead:
                     self._fatal_engine_failure = True
             if self._stopping:
                 return
@@ -422,11 +418,7 @@ def healthy() -> bool:
             )
 
         def terminal_engine_failure() -> bool:
-            executor = getattr(llm, "_executor", None)
-            return executor is not None and (
-                getattr(executor, "_fatal_error", None) is not None
-                or getattr(executor, "_engine_dead", False)
-            )
+            return llm.engine_dead
 
         async def supervise() -> None:
             while not stop_event.is_set():
@@ -438,7 +430,7 @@ async def supervise() -> None:
                     return
                 if server is None:
                     engine_ready = await asyncio.wait_for(
-                        asyncio.to_thread(llm._check_health),
+                        asyncio.to_thread(llm.check_health),
                         timeout=_HEALTH_CHECK_TIMEOUT_SECONDS,
                     )
                     if server is None and not engine_ready:

tensorrt_llm/grpc/smg/request_manager.py

@@ -168,20 +168,25 @@ async def health_check(self) -> Tuple[bool, str]:
             if self.llm is None:
                 return False, "LLM not initialized"
 
-            # Check executor health (includes error queue, MPI worker
-            # liveness, and fatal error state — not just doing_shutdown).
-            if hasattr(self.llm, "_executor"):
-                if self.llm._executor is None:
-                    return False, "Executor is not available"
-                if not self.llm._executor.check_health():
-                    error_msg = "Executor is unhealthy"
-                    if self.llm._executor._fatal_error is not None:
-                        exc = self.llm._executor._fatal_error
-                        lines = str(exc).splitlines()
-                        short = (lines[0] if lines else type(exc).__name__)[:200]
-                        error_msg = f"{type(exc).__name__}: {short}"
-                        logger.error(f"Health check fatal error: {repr(exc)}")
-                    return False, error_msg
+            # An encode-only LLM has no generation executor; SMG serves
+            # generate requests only.
+            if self.llm.args.encode_only is True:
+                return False, "Executor is not available"
+            # check_health() drains queued errors and checks MPI worker
+            # liveness and the fatal error state, not just shutdown.
+            if not self.llm.check_health():
+                error_msg = "Executor is unhealthy"
+                exc = self.llm.fatal_error
+                if exc is not None:
+                    lines = str(exc).splitlines()
+                    short = (lines[0] if lines else type(exc).__name__)[:200]
+                    error_msg = f"{type(exc).__name__}: {short}"
+                    logger.error(f"Health check fatal error: {repr(exc)}")
+                elif self.llm.shutdown_started and not self.llm.engine_dead:
+                    # No fatal error and no dead engine: shutdown() released
+                    # (or is releasing) the generation executor.
+                    error_msg = "Executor is not available"
+                return False, error_msg
 
             return True, "OK"
         except Exception as e:

tensorrt_llm/llmapi/llm.py

@@ -347,6 +347,9 @@ def _multimodal_params_have_encoder_features(
         llm_id (str): The unique ID of the LLM instance.
         disaggregated_params (dict): The disaggregated parameters of the LLM instance.
         startup_metrics (dict): The startup metrics reported by worker rank 0.
+        fatal_error (BaseException, optional): The unrecoverable engine error the executor recorded, or None.
+        shutdown_started (bool): Whether shutdown of the generation executor has begun.
+        engine_dead (bool): Whether the engine is known dead (a worker died or a fatal error was recorded).
 """
 
 
@@ -1807,11 +1810,20 @@ def _record_usage_shutdown(self) -> None:
         except Exception as exc:
             logger.debug(f"Usage telemetry shutdown tracking failed: {exc}")
 
-    def _check_health(self) -> bool:
-        """Check if the LLM is healthy.
+    @set_api_status("prototype")
+    def check_health(self) -> bool:
+        """Check whether the LLM can serve requests.
+
+        For ``encode_only=True`` the LLM is healthy while its encoder executor
+        exists. Otherwise this delegates to the generation executor's
+        ``check_health()``, which drains queued background errors (a fatal one
+        is recorded, see ``fatal_error``, and starts executor shutdown) and,
+        for the multi-process proxy, polls worker liveness. Without a
+        generation executor (never built, or after ``shutdown()``) the LLM is
+        unhealthy.
 
         Returns:
-            bool: True if the executor is running and healthy, False otherwise.
+            bool: True if the LLM is healthy, False otherwise.
         """
         if self._encode_only:
             return (hasattr(self, "_encoder_executor")
@@ -1821,6 +1833,46 @@ def _check_health(self) -> bool:
 
         return False
 
+    # Kept for one release for out-of-tree callers that look this name up
+    # (ai-dynamo components/src/dynamo/trtllm/engine.py); use check_health().
+    _check_health = check_health
+
+    @property
+    @set_api_status("prototype")
+    def fatal_error(self) -> Optional[BaseException]:
+        """The unrecoverable engine error the executor recorded, if any.
+
+        None when no fatal error was recorded or the LLM has no generation
+        executor (``encode_only=True``, or after ``shutdown()``).
+        """
+        executor = getattr(self, "_executor", None)
+        return None if executor is None else executor.fatal_error
+
+    @property
+    @set_api_status("prototype")
+    def shutdown_started(self) -> bool:
+        """Whether shutdown of the generation executor has begun.
+
+        True once ``shutdown()`` was called or the executor started shutting
+        down after a fatal error. False for ``encode_only=True``, which has no
+        generation executor.
+        """
+        executor = getattr(self, "_executor", None)
+        if executor is None:
+            return not getattr(self, "_encode_only", False)
+        return executor.doing_shutdown
+
+    @property
+    @set_api_status("prototype")
+    def engine_dead(self) -> bool:
+        """Whether the engine is known dead (a worker died or a fatal error was recorded).
+
+        False when the LLM has no generation executor (``encode_only=True``,
+        or after ``shutdown()``).
+        """
+        executor = getattr(self, "_executor", None)
+        return False if executor is None else executor.engine_dead
+
     @staticmethod
     def _shutdown_wrapper(self_ref):
         # Retrieve the instance if it still exists

tensorrt_llm/serve/openai_server.py

@@ -1230,9 +1230,11 @@ def _check_health(self) -> bool:
         batcher = self.embedding_batcher
         if batcher is not None and not batcher.is_alive():
             return False
-        if hasattr(self.generator, '_check_health'):
-            return self.generator._check_health()
-        return True
+        if self._is_visual_gen:
+            # VisualGen has no check_health(), so its /health always
+            # answers 200.
+            return True
+        return self.generator.check_health()
 
     def register_routes(self):
         self.app.add_api_route("/health", self.health, methods=["GET"])
@@ -1729,15 +1731,14 @@ async def health(self) -> Response:
             # its graceful shutdown sequence.  The doing_shutdown guard
             # ensures we only send SIGINT once — repeated health probes
             # won't stack signals during an in-progress teardown.
-            executor = getattr(self.generator, '_executor', None)
-            if executor is not None and getattr(executor, '_fatal_error',
-                                                None) is not None:
-                if not getattr(executor, 'doing_shutdown', True):
-                    logger.error(
-                        "Health check detected fatal engine error, initiating "
-                        f"server shutdown: {executor._fatal_error}")
-                    _record_generator_termination(self.generator)
-                    signal.raise_signal(signal.SIGINT)
+            fatal_error = (None if self._is_visual_gen else
+                           self.generator.fatal_error)
+            if fatal_error is not None and not self.generator.shutdown_started:
+                logger.error(
+                    "Health check detected fatal engine error, initiating "
+                    f"server shutdown: {fatal_error}")
+                _record_generator_termination(self.generator)
+                signal.raise_signal(signal.SIGINT)
             return Response(
                 status_code=503,
                 content=

tests/integration/test_lists/test-db/l0_cpu.yml

@@ -309,6 +309,8 @@ l0_cpu:
   - unittest/grpc/openengine/test_openengine.py
   - unittest/grpc/openengine/test_metrics.py
   - unittest/grpc/openengine/test_servicer.py
+  - unittest/grpc/openengine/test_private_access_lint_openengine.py
+  - unittest/grpc/smg/test_smg_health.py
   - unittest/grpc/smg/test_smg.py::test_streaming_deltas_preserve_sequences_logprobs_and_final_tokens
   - unittest/grpc/smg/test_smg.py::test_streaming_empty_token_history
   - unittest/grpc/test_grpc_optional.py::test_protobuf_warning_in_fresh_process
@@ -379,6 +381,7 @@ l0_cpu:
   - unittest/llmapi/test_llm_args.py
   - unittest/llmapi/test_llm.py::test_startup_metrics_by_server_info
   - unittest/llmapi/test_gms_args.py
+  - unittest/llmapi/test_llm_public_runtime_api.py
   - unittest/llmapi/test_mx_args.py
   - unittest/llmapi/test_custom_tokenizer_aliases.py
   # Reads the Muse Spark Open tokenizer files from LLM_MODELS_ROOT.

tests/unittest/api_stability/references/llm.yaml

@@ -544,6 +544,10 @@ methods:
         default: 2
     return_annotation: tensorrt_llm.executor.result.IterationResult
     status: beta
+  check_health:
+    parameters: {}
+    return_annotation: bool
+    status: prototype
   get_data_transceiver_state:
     parameters: {}
     return_annotation: bytes
@@ -600,3 +604,12 @@ properties:
   startup_metrics:
     annotation: dict
     default: None
+  fatal_error:
+    annotation: Optional[BaseException]
+    default: inspect._empty
+  shutdown_started:
+    annotation: bool
+    default: inspect._empty
+  engine_dead:
+    annotation: bool
+    default: inspect._empty