[None][fix] serve: answer a request that fails validation with a readable error

docs/source/release-notes.md

@@ -36,6 +36,8 @@ All published functionality in the Release Notes has been fully tested and verif
   in `get_dummy_mm_data()` results used for startup profiling. See the
   [multimodal developer guide](developer-guide/multimodal.md).
 
+- A request body that fails validation is now answered with OpenAI's error object by `trtllm-serve` and the disaggregated server: HTTP 400 with `{"error": {"message", "type": "invalid_request_error", "param", "code": null}}`. The `message` names each rejected parameter by its place in the body (for example `Unknown parameter: 'stores'.`) and never quotes the rejected value, and `param` holds the place of the first one. The `error` field was a string before, pydantic's rendering of the failure, which repeated every rejected value; clients that read it as a string should read `error.message` instead. `/v1/messages` keeps the Anthropic error envelope, now with the same message, and visual generation keeps its HTTP 422 envelope.
+
 ### Fixed Issues
 
 ### Known Issues

tensorrt_llm/serve/openai_disagg_server.py

@@ -62,7 +62,8 @@
 from tensorrt_llm.serve.openai_protocol import (
     ChatCompletionRequest, ChatCompletionResponse, CompletionRequest,
     ConversationParams, ErrorResponse, ResponsesRequest, UCompletionRequest,
-    UCompletionResponse, ensure_request_chat_template_allowed)
+    UCompletionResponse, ensure_request_chat_template_allowed,
+    summarize_validation_errors, validation_error_body)
 from tensorrt_llm.serve.perf_metrics import (DisaggPerfMetricsCollector,
                                              PerfMetricsJsonlWriter,
                                              PerfMetricsMiddleware,
@@ -298,36 +299,43 @@ async def lifespan(app) -> None:
                 writer=self._perf_metrics_writer)
         self.app.add_middleware(ServerArrivalTimeMiddleware)
 
-        # Log request-body validation failures so a client/server schema mismatch
-        # shows up server-side. Throttled (first, then every 1000th) to avoid
-        # flooding the event loop when every request fails identically.
         self._val_err_n = 0
-        @self.app.exception_handler(RequestValidationError)
-        async def validation_exception_handler(request: Request, exc):
-            self._perf_metrics_collector.validation_exceptions.inc()
-            # Anthropic clients parse the error envelope, so every /v1/messages
-            # route must fail in that shape rather than the generic one below.
-            if request.url.path.startswith("/v1/messages"):
-                return anthropic_error_response(str(exc),
-                                                "invalid_request_error", 400)
-            self._val_err_n += 1
-            if self._val_err_n == 1 or self._val_err_n % 1000 == 0:
-                try:
-                    errs = exc.errors()
-                    # Compact: [{loc, type, msg}] -- drops the (large) echoed input.
-                    brief = [{"loc": e.get("loc"), "type": e.get("type"),
-                              "msg": e.get("msg")} for e in errs][:8]
-                except Exception:  # noqa: BLE001
-                    brief = str(exc)[:500]
-                method = request.method.translate(_LOG_CONTROL_CHARACTERS)
-                path = request.url.path.translate(_LOG_CONTROL_CHARACTERS)
-                logger.warning(
-                    f"[validation] {method} {path} 400 "
-                    f"(n={self._val_err_n}): {brief}")
-            return JSONResponse(status_code=400, content={"error": str(exc)})
+        self.app.add_exception_handler(RequestValidationError,
+                                       self._request_validation_error)
 
         self.register_routes()
 
+    async def _request_validation_error(
+            self, request: Request, exc: RequestValidationError) -> Response:
+        """Answer a request whose body failed validation with a 400.
+
+        /v1/messages answers in the Anthropic envelope, which Anthropic clients
+        parse, and every other route in OpenAI's ``{"error": {...}}``. The
+        message comes from ``exc.errors()``, never ``str(exc)``, which quotes
+        every rejected value and names the endpoint's source file and line.
+
+        Failures are also logged so a client/server schema mismatch shows up
+        server-side, throttled (first, then every 1000th) so a client whose
+        every request fails the same way cannot flood the event loop.
+        """
+        self._perf_metrics_collector.validation_exceptions.inc()
+        if request.url.path.startswith("/v1/messages"):
+            message, _ = summarize_validation_errors(exc.errors())
+            return anthropic_error_response(message, "invalid_request_error",
+                                            400)
+        self._val_err_n += 1
+        if self._val_err_n == 1 or self._val_err_n % 1000 == 0:
+            # Compact: [{loc, type, msg}] -- drops the (large) echoed input.
+            brief = [{"loc": e.get("loc"), "type": e.get("type"),
+                      "msg": e.get("msg")} for e in exc.errors()][:8]
+            method = request.method.translate(_LOG_CONTROL_CHARACTERS)
+            path = request.url.path.translate(_LOG_CONTROL_CHARACTERS)
+            logger.warning(
+                f"[validation] {method} {path} 400 "
+                f"(n={self._val_err_n}): {brief}")
+        return JSONResponse(status_code=400,
+                            content=validation_error_body(exc.errors()))
+
     def _create_client(self, router: Router, role: ServerRole, max_retries: int = 1) -> OpenAIClient:
         async def disagg_id_generator():
             return await self._coordinator.get_disagg_request_id()

tensorrt_llm/serve/openai_protocol.py

@@ -19,7 +19,8 @@
 import math
 import time
 import uuid
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import (Any, Dict, List, Literal, Mapping, Optional, Sequence,
+                    Tuple, Union)
 
 import torch
 import xgrammar
@@ -383,6 +384,171 @@ class ErrorResponse(OpenAIBaseModel):
     code: Union[int, str]
 
 
+# A longer location segment, such as an unknown field name, is cut short so
+# the error cannot echo an arbitrarily long name back.
+_MAX_LOCATION_SEGMENT_CHARS = 64
+
+# Schema names pydantic tags a scalar union member with.
+_SCALAR_MEMBER_TAGS = frozenset(
+    {"str", "int", "float", "bool", "bytes", "none", "list", "dict"})
+
+
+def _is_union_member_tag(segment: Any) -> bool:
+    """Whether a location segment names a union member rather than a field.
+
+    Pydantic tags each member's errors with the member: a model or TypedDict
+    by its class name, any other type by its schema name ('str',
+    'list[...]', 'function-after[...]'). Request field names are lowercase,
+    but a key the client sent need not be; _union_member_tag_positions keeps
+    such a key.
+    """
+    return isinstance(segment,
+                      str) and ("[" in segment or segment[:1].isupper()
+                                or segment in _SCALAR_MEMBER_TAGS)
+
+
+# Error types whose last location segment is a key the client sent or a field
+# the schema requires, never a union member.
+_KEY_NAMING_ERROR_TYPES = frozenset({"extra_forbidden", "missing"})
+
+
+def _union_member_tag_positions(error: Mapping[str, Any]) -> List[int]:
+    """Where an error's location names a union member rather than a field.
+
+    The last segment of an unknown or missing parameter is that parameter's
+    own name, whatever it looks like ('Temperature', 'list'), so it is never
+    taken for a member.
+    """
+    loc = tuple(error.get("loc", ()))
+    end = len(loc) - 1 if error.get("type") in _KEY_NAMING_ERROR_TYPES else len(
+        loc)
+    return [i for i in range(end) if _is_union_member_tag(loc[i])]
+
+
+def _matched_union_member_errors(
+        errors: Sequence[Mapping[str, Any]]) -> List[Mapping[str, Any]]:
+    """The errors left once each union keeps only the members its value chose.
+
+    A union reports every member's errors. A member whose literal field (a
+    message's "role", a content part's "type") rejected the value was not
+    the one the value was written for, so when other members accepted that
+    literal, the rejected members' errors are dropped.
+    """
+    members_by_union: Dict[tuple, Dict[str, List[int]]] = {}
+    for index, error in enumerate(errors):
+        loc = tuple(error.get("loc", ()))
+        for depth in _union_member_tag_positions(error):
+            members = members_by_union.setdefault(loc[:depth], {})
+            members.setdefault(loc[depth], []).append(index)
+    dropped = set()
+    for prefix, members in members_by_union.items():
+        rejected = {
+            tag
+            for tag, indices in members.items()
+            if any(errors[i].get("type") == "literal_error"
+                   and len(errors[i].get("loc", ())) == len(prefix) + 2
+                   for i in indices)
+        }
+        if rejected and len(rejected) < len(members):
+            for tag in rejected:
+                dropped.update(members[tag])
+    return [error for i, error in enumerate(errors) if i not in dropped]
+
+
+def _error_location(error: Mapping[str, Any]) -> str:
+    """The dotted place of a validation error within the request body."""
+    loc = tuple(error.get("loc", ()))
+    member_tags = set(_union_member_tag_positions(error))
+    segments = []
+    for index, segment in enumerate(loc):
+        # FastAPI prefixes body errors with "body"; any later "body" is a key.
+        if (index == 0 and segment == "body") or index in member_tags:
+            continue
+        text = str(segment)
+        if len(text) > _MAX_LOCATION_SEGMENT_CHARS:
+            text = text[:_MAX_LOCATION_SEGMENT_CHARS] + "..."
+        segments.append(text)
+    return ".".join(segments)
+
+
+def summarize_validation_errors(errors: Sequence[Mapping[str, Any]],
+                                limit: int = 20) -> Tuple[str, Optional[str]]:
+    """Describe why a request body failed validation, without quoting it.
+
+    Each error is named by its place in the body, worded the way OpenAI words
+    the same rejection: "Unknown parameter: 'store'.", "Missing required
+    parameter: 'messages'.", "Invalid value for 'messages.0.content': ...".
+    The offending value is never quoted. The client already has it, and a
+    rejected inline image would make the message megabytes long.
+
+    Args:
+        errors: ``RequestValidationError.errors()`` of the rejected request.
+        limit: How many errors to describe; the rest are only counted.
+
+    Returns:
+        The message, and the dotted location of the first error, or None when
+        that error has no location in the body (the body is not valid JSON).
+    """
+    parts: List[str] = []
+    param: Optional[str] = None
+    errors = _matched_union_member_errors(errors)
+    # A field that failed in several union members is described once.
+    seen_locations = set()
+    described = []
+    for error in errors:
+        location = _error_location(error)
+        if error.get("type") != "json_invalid":
+            if location in seen_locations:
+                continue
+            seen_locations.add(location)
+        described.append((error, location))
+    for index, (error, location) in enumerate(described):
+        error_type = error.get("type", "")
+        if error_type == "json_invalid":
+            # The location is a character offset, not a parameter.
+            reason = (error.get("ctx") or {}).get("error") or error.get(
+                "msg", "")
+            description = f"The request body is not valid JSON: {reason}."
+            location = ""
+        elif error_type == "extra_forbidden":
+            description = f"Unknown parameter: '{location}'."
+        elif error_type == "missing":
+            description = (f"Missing required parameter: '{location}'."
+                           if location else "The request body is missing.")
+        elif location:
+            # A validator's own message is passed through as written, so a
+            # validator in this module must not put a request value into it.
+            description = f"Invalid value for '{location}': {error.get('msg', '')}."
+        else:
+            description = f"{error.get('msg', '')}."
+        if index == 0:
+            param = location or None
+        if index < limit:
+            parts.append(description)
+    if len(described) > limit:
+        parts.append(f"({len(described) - limit} more errors not shown.)")
+    return " ".join(parts) or "The request body is invalid.", param
+
+
+def validation_error_body(
+        errors: Sequence[Mapping[str, Any]]) -> Dict[str, Any]:
+    """The OpenAI error envelope for a request body that failed validation.
+
+    The shape OpenAI answers with, ``{"error": {"message", "type", "param",
+    "code"}}``, so OpenAI SDKs surface the message, and clients that only
+    check for an ``error`` key still find one.
+    """
+    message, param = summarize_validation_errors(errors)
+    return {
+        "error": {
+            "message": message,
+            "type": "invalid_request_error",
+            "param": param,
+            "code": None,
+        }
+    }
+
+
 class CompletionLogProbs(OpenAIBaseModel):
     text_offset: List[int] = Field(default_factory=list)
     token_logprobs: List[Optional[float]] = Field(default_factory=list)
@@ -1031,8 +1197,10 @@ def check_enum_value_cap(self):
         if self.parameters is not None:
             num_enum_values = _count_schema_enum_values(self.parameters)
             if num_enum_values > TOOL_PARAM_MAX_ENUM_VALUES:
+                # The name is left out: it is client text of any length, and
+                # the error's location already identifies the tool.
                 raise ValueError(
-                    f"tool function {self.name!r} declares {num_enum_values} "
+                    f"tool function declares {num_enum_values} "
                     f"enum values across its parameters schema; the maximum "
                     f"is {TOOL_PARAM_MAX_ENUM_VALUES}.")
         return self

tensorrt_llm/serve/openai_server.py

@@ -110,8 +110,9 @@
     ResponseFormat, ResponsesRequest, ResponsesResponse,
     RuntimeMemoryUpdateRequest, StartProfileRequest, TokenizeRequest,
     TokenizeResponse, UpdateWeightsRequest, UsageInfo,
-    ensure_request_chat_template_allowed, to_llm_conversation_params,
-    to_llm_disaggregated_params)
+    ensure_request_chat_template_allowed, summarize_validation_errors,
+    to_llm_conversation_params, to_llm_disaggregated_params,
+    validation_error_body)
 from tensorrt_llm.serve.openai_video_routes import _VideoRoutesMixin
 from tensorrt_llm.serve.perf_metrics import (PerfMetricsJsonlWriter,
                                              PerfMetricsMiddleware,
@@ -885,30 +886,8 @@ async def lifespan(app: FastAPI):
         self.app = FastAPI(lifespan=lifespan)
         self.app.router.route_class = _MsgspecRoute
 
-        @self.app.exception_handler(RequestValidationError)
-        async def validation_exception_handler(request, exc):
-            if request.url.path.startswith("/v1/messages"):
-                # Anthropic clients expect a 400 with the Anthropic error
-                # envelope, not FastAPI's 422 shape.
-                if self.metrics_collector:
-                    self.metrics_collector.log_request_error(http_code=400)
-                return anthropic_error_response(str(exc),
-                                                "invalid_request_error", 400)
-            if self.server_role is ServerRole.VISUAL_GEN:
-                return self._create_visual_gen_validation_error_response(exc)
-            # Non-visual-gen roles keep the shared 400 + ``{"error": ...}``
-            # response shape that integration tests (e.g.
-            # ``test_malformed_json_request``) and existing clients
-            # expect.
-            if self.metrics_collector:
-                self.metrics_collector.log_request_error(http_code=400)
-            # pydantic echoes each failing input verbatim, which can be
-            # megabytes of base64; the error locations are what matters.
-            detail = str(exc)
-            if len(detail) > 4096:
-                detail = detail[:4096] + " ... [truncated %d bytes]" % (
-                    len(detail) - 4096)
-            return JSONResponse(status_code=400, content={"error": detail})
+        self.app.add_exception_handler(RequestValidationError,
+                                       self._request_validation_error)
 
         if self.server_role is ServerRole.VISUAL_GEN:
             assert self._is_visual_gen, \
@@ -1190,6 +1169,30 @@ def _create_not_supported_error(self, message: str) -> Response:
             status_code=HTTPStatus.NOT_IMPLEMENTED,
         )
 
+    async def _request_validation_error(
+            self, request: Request, exc: RequestValidationError) -> Response:
+        """Answer a request whose body failed validation with a 400.
+
+        /v1/messages answers in the Anthropic envelope, visual generation in
+        its own, and every other route in OpenAI's ``{"error": {...}}``,
+        whose top-level ``error`` key existing clients look for. The message
+        comes from ``exc.errors()``, never ``str(exc)``: that quotes every
+        rejected value (an inline image is megabytes of base64) and names the
+        endpoint's source file and line.
+        """
+        if request.url.path.startswith("/v1/messages"):
+            if self.metrics_collector:
+                self.metrics_collector.log_request_error(http_code=400)
+            message, _ = summarize_validation_errors(exc.errors())
+            return anthropic_error_response(message, "invalid_request_error",
+                                            400)
+        if self.server_role is ServerRole.VISUAL_GEN:
+            return self._create_visual_gen_validation_error_response(exc)
+        if self.metrics_collector:
+            self.metrics_collector.log_request_error(http_code=400)
+        return JSONResponse(status_code=400,
+                            content=validation_error_body(exc.errors()))
+
     def _create_visual_gen_validation_error_response(
             self, exc: RequestValidationError) -> Response:
         """Render a ``RequestValidationError`` as the visual-gen 422 envelope.

tests/integration/test_lists/test-db/l0_cpu.yml

@@ -259,6 +259,7 @@ l0_cpu:
   - unittest/llmapi/apps/test_request_metrics.py::test_header_derived_record_keeps_kv_transfer_without_kv_cache_size
   - unittest/llmapi/apps/test_request_metrics.py::test_unpopulated_kv_transfer_timestamps_stay_absent
   - unittest/llmapi/apps/test_request_metrics.py::TestJsonlWriterSurvival
+  - unittest/llmapi/apps/test_request_validation_errors.py
   - unittest/llmapi/apps/test_responses_custom_tools.py
   - unittest/llmapi/apps/test_responses_input_preprocess.py
   - unittest/llmapi/apps/test_responses_store_replay.py