[#19994][feat] Add short-clip Whisper audio transcriptions

docs/source/commands/trtllm-serve/audio-transcriptions.md

@@ -0,0 +1,79 @@
+<!-- SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
+<!-- SPDX-License-Identifier: Apache-2.0 -->
+
+# Whisper audio transcriptions
+
+When serving a supported Whisper checkpoint, `trtllm-serve` accepts short audio
+uploads at `POST /v1/audio/transcriptions`.
+
+Use the encoder-decoder runtime settings from the
+[Whisper guide](../../models/encoder-decoder.md#transcribe-audio-with-whisper).
+For example, save the following as `whisper.yaml`:
+
+```yaml
+attn_backend: TRTLLM
+dtype: float16
+max_batch_size: 4
+max_input_len: 1500
+max_seq_len: 2048
+max_num_tokens: 3000
+enable_chunked_prefill: false
+kv_cache_config:
+  enable_block_reuse: false
+  free_gpu_memory_fraction: 0.8
+  cross_kv_cache_fraction: 0.5
+scheduler_config:
+  use_python_scheduler: true
+```
+
+`max_input_len` must cover Whisper's 1500 encoder positions even for a short
+recording. This endpoint does not change engine or cross-KV cache sizing.
+
+```bash
+trtllm-serve openai/whisper-large-v3 --backend pytorch --config whisper.yaml --port 8000
+```
+
+Submit a multipart request using the model identifier reported by `/v1/models`:
+
+```bash
+curl http://localhost:8000/v1/audio/transcriptions \
+  -F model=openai/whisper-large-v3 \
+  -F file=@short.wav \
+  -F language=en \
+  -F response_format=json
+```
+
+The JSON response is `{"text":"..."}`. Set `response_format=text` for a plain
+text response. OpenAI-compatible clients can use the same endpoint:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8000/v1", api_key="unused")
+with open("short.wav", "rb") as audio:
+    result = client.audio.transcriptions.create(
+        model="openai/whisper-large-v3",
+        file=audio,
+        language="en",
+    )
+print(result.text)
+```
+
+Uploads are limited to 25 MiB and to the checkpoint's input window (normally
+30 seconds). Audio is decoded with SoundFile, downmixed to mono, and resampled
+to the checkpoint's sampling rate. WAV and FLAC are supported; other formats
+depend on the installed libsndfile codecs. Malformed, empty, nonfinite, or
+overlong audio is rejected. Split longer recordings on the client side.
+
+`language` selects the language prefix on multilingual checkpoints. Omitting it
+uses the checkpoint's current English-transcription default; automatic language
+detection is not implemented. English-only checkpoints accept `language=en`.
+`temperature` accepts values from 0 to 1 and defaults to greedy decoding (0).
+There is no temperature fallback/retry schedule.
+
+This endpoint currently supports JSON/text output, one clip per request, and
+non-streaming transcription. Nonempty `prompt`, `stream=true`, timestamp
+granularities, other response formats, and unknown form fields are rejected
+explicitly. Translation, timestamps, long-form windowing, and hotword biasing
+are not implemented. The endpoint requires a Whisper model and does not route
+through the disaggregated-serving proxy.

docs/source/commands/trtllm-serve/index.rst

@@ -8,3 +8,4 @@ trtllm-serve
    trtllm-serve
    render-endpoints
    run-benchmark-with-trtllm-serve
+   audio-transcriptions

tensorrt_llm/serve/audio_transcriptions.py

@@ -0,0 +1,171 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Multipart parsing and audio decoding for short Whisper transcriptions."""
+
+from __future__ import annotations
+
+import asyncio
+import io
+import math
+from collections.abc import Awaitable, Callable
+from typing import TYPE_CHECKING, Literal, Optional
+
+import numpy as np
+from fastapi import HTTPException, Request
+from fastapi.responses import JSONResponse, PlainTextResponse, Response
+from numpy.typing import NDArray
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+from starlette.datastructures import UploadFile
+
+if TYPE_CHECKING:
+    from transformers import WhisperConfig, WhisperProcessor
+
+    from tensorrt_llm.inputs.data import TextPrompt
+
+MAX_AUDIO_UPLOAD_BYTES = 25 * 1024 * 1024
+
+
+class TranscriptionOptions(BaseModel):
+    """Supported form fields for the short-clip transcription endpoint."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    model: str = Field(min_length=1, description="Served Whisper model identifier.")
+    language: Optional[str] = Field(
+        default=None, description="Spoken language; omitted uses the checkpoint's English default."
+    )
+    temperature: float = Field(
+        default=0.0, ge=0.0, le=1.0, allow_inf_nan=False, description="Sampling temperature."
+    )
+    response_format: Literal["json", "text"] = Field(
+        default="json", description="Transcription response format."
+    )
+    prompt: str = Field(default="", description="Only an empty prompt is currently supported.")
+    stream: bool = Field(default=False, description="Only non-streaming output is supported.")
+
+    @field_validator("prompt")
+    @classmethod
+    def _reject_prompt(cls, value: str) -> str:
+        if value:
+            raise ValueError("Prompt conditioning is not supported for audio transcriptions yet.")
+        return value
+
+    @field_validator("stream")
+    @classmethod
+    def _reject_streaming(cls, value: bool) -> bool:
+        if value:
+            raise ValueError("Streaming audio transcriptions are not supported yet.")
+        return value
+
+
+def _decode_audio(data: bytes, sampling_rate: int, max_samples: int) -> NDArray[np.float32]:
+    # Importing audio codecs/scipy is unnecessary for servers that never transcribe.
+    import soundfile as sf
+    from scipy.signal import resample_poly
+
+    try:
+        with sf.SoundFile(io.BytesIO(data)) as audio_file:
+            source_rate = audio_file.samplerate
+            if audio_file.frames == 0:
+                raise ValueError("The audio file contains no samples.")
+            if audio_file.frames * sampling_rate > max_samples * source_rate:
+                raise ValueError(
+                    f"Audio duration exceeds the model's {max_samples / sampling_rate:g}s window. "
+                    "Chunk the input on the client side."
+                )
+            waveform = audio_file.read(dtype="float32", always_2d=True).mean(axis=1)
+    except (sf.LibsndfileError, sf.SoundFileRuntimeError) as exc:
+        raise ValueError("Cannot decode audio: upload a supported, nonempty audio file.") from exc
+    if not np.isfinite(waveform).all():
+        raise ValueError("The audio file contains nonfinite samples.")
+    if source_rate != sampling_rate:
+        divisor = math.gcd(source_rate, sampling_rate)
+        waveform = resample_poly(waveform, sampling_rate // divisor, source_rate // divisor)
+    return np.ascontiguousarray(waveform, dtype=np.float32)
+
+
+async def parse_transcription_request(
+    request: Request, *, served_model: str, sampling_rate: int, max_samples: int
+) -> tuple[TranscriptionOptions, NDArray[np.float32]]:
+    """Validate one multipart upload and decode mono audio at the model's rate.
+
+    Decoding runs off the event loop. The model window is checked from file
+    metadata before allocating the decoded waveform.
+    """
+    if request.headers.get("content-type", "").split(";", 1)[0] != "multipart/form-data":
+        raise ValueError("Audio transcriptions require multipart/form-data with a file upload.")
+    async with request.form(max_files=1, max_fields=16) as form:
+        fields: dict[str, str] = {}
+        upload = None
+        for name, value in form.multi_items():
+            if name == "file":
+                if not isinstance(value, UploadFile):
+                    raise ValueError("'file' must be an uploaded audio file.")
+                upload = value
+            else:
+                if name in fields or not isinstance(value, str):
+                    raise ValueError(f"Expected one text value for {name!r}.")
+                fields[name] = value
+        options = TranscriptionOptions.model_validate(fields)
+        if options.model != served_model:
+            raise HTTPException(status_code=404, detail=f"Model {options.model!r} is not served.")
+        if upload is None:
+            raise ValueError("A 'file' audio upload is required.")
+        data = await upload.read(MAX_AUDIO_UPLOAD_BYTES + 1)
+        if len(data) > MAX_AUDIO_UPLOAD_BYTES:
+            raise HTTPException(status_code=413, detail="Audio upload exceeds the 25 MiB limit.")
+    waveform = await asyncio.to_thread(_decode_audio, data, sampling_rate, max_samples)
+    return options, waveform
+
+
+def transcription_response(text: str, response_format: Literal["json", "text"]) -> Response:
+    """Render decoded text in the requested OpenAI-compatible response format."""
+    if response_format == "text":
+        return PlainTextResponse(text)
+    return JSONResponse({"text": text})
+
+
+async def create_transcription_response(
+    request: Request,
+    *,
+    served_model: str,
+    model_config: Optional[WhisperConfig],
+    processor: Optional[WhisperProcessor],
+    generate: Callable[[TextPrompt, float, Request], Awaitable[str]],
+) -> Response:
+    """Convert one short audio request into Whisper inputs and render the result.
+
+    ``generate`` submits the inputs through the serving engine. Keeping this
+    conversion independent of the engine permits CPU request-contract tests.
+    """
+    if model_config is None or model_config.model_type != "whisper":
+        raise ValueError("Audio transcriptions require a Whisper model.")
+    if processor is None:
+        raise RuntimeError("The Whisper processor is unavailable.")
+    extractor = processor.feature_extractor
+    options, waveform = await parse_transcription_request(
+        request,
+        served_model=served_model,
+        sampling_rate=int(extractor.sampling_rate),
+        max_samples=int(extractor.n_samples),
+    )
+    prompt_text = ""
+    if options.language is not None:
+        try:
+            forced = processor.get_decoder_prompt_ids(
+                language=options.language, task="transcribe", no_timestamps=True
+            )
+        except ValueError:
+            # English-only checkpoints have no language/task tokens.
+            if options.language != "en":
+                raise
+            forced = processor.get_decoder_prompt_ids(no_timestamps=True)
+        prefix = [int(model_config.decoder_start_token_id)]
+        prefix.extend(int(token) for _, token in sorted(forced))
+        prompt_text = processor.tokenizer.decode(prefix, skip_special_tokens=False)
+    inputs: TextPrompt = {
+        "prompt": prompt_text,
+        "multi_modal_data": {"audio": [(waveform, int(extractor.sampling_rate))]},
+    }
+    text = await generate(inputs, options.temperature, request)
+    return transcription_response(text, options.response_format)

tensorrt_llm/serve/openai_server.py

@@ -47,7 +47,7 @@
 from tensorrt_llm.executor.utils import (CONTEXT_LENGTH_EXCEEDED_CODE,
                                          is_context_length_exceeded_message)
 from tensorrt_llm.inputs import prompt_inputs
-from tensorrt_llm.inputs.data import TokensPrompt
+from tensorrt_llm.inputs.data import TextPrompt, TokensPrompt
 from tensorrt_llm.inputs.media_io import (BaseMediaIO,
                                           log_allowed_local_media_path)
 from tensorrt_llm.inputs.multimodal import MultimodalServerConfig
@@ -1291,6 +1291,9 @@ def register_routes(self):
         self.app.add_api_route("/v1/completions",
                                self.openai_completion,
                                methods=["POST"])
+        self.app.add_api_route("/v1/audio/transcriptions",
+                               self.openai_transcription,
+                               methods=["POST"])
         self.app.add_api_route(
             "/v1/chat/completions",
             self.openai_chat if not self.use_harmony else self.chat_harmony,
@@ -2056,6 +2059,60 @@ def _apply_spec_decode_stats_opt_in(self,
         postproc_args.spec_decode_num_spec_tokens = getattr(
             self, "_spec_decode_num_spec_tokens", None)
 
+    async def openai_transcription(self, raw_request: Request) -> Response:
+        """Transcribe one short audio upload with the served Whisper model."""
+        from .audio_transcriptions import create_transcription_response
+
+        try:
+            return await create_transcription_response(
+                raw_request,
+                served_model=self.model,
+                model_config=self.model_config,
+                processor=self.processor,
+                generate=self._generate_audio_transcription)
+        except HTTPException as exc:
+            return self.create_error_response(str(exc.detail),
+                                              status_code=HTTPStatus(
+                                                  exc.status_code))
+        except ValueError as exc:
+            return self.create_error_response(str(exc))
+        except Exception:
+            logger.error("Audio transcription failed: %s",
+                         traceback.format_exc())
+            return self.create_error_response(
+                "Audio transcription failed.",
+                err_type="InternalServerError",
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+
+    async def _generate_audio_transcription(self, inputs: TextPrompt,
+                                            temperature: float,
+                                            raw_request: Request) -> str:
+        sampling_params = SamplingParams(
+            max_tokens=int(self.model_config.max_target_positions) - 1,
+            temperature=temperature,
+            top_k=1 if temperature == 0 else 0)
+        preprocess_fn = getattr(self.generator, "preprocess", None)
+        if preprocess_fn is not None:
+            inputs = await asyncio.get_running_loop().run_in_executor(
+                self._input_proc_executor,
+                functools.partial(preprocess_fn, inputs, sampling_params))
+        promise = self.generator.generate_async(
+            inputs=inputs,
+            sampling_params=sampling_params,
+            streaming=False,
+            trace_headers=tracing.extract_trace_headers(raw_request.headers))
+        disconnect_task = asyncio.create_task(
+            self.await_disconnected(raw_request, promise))
+        try:
+            await promise.aresult()
+            return promise.outputs[0].text
+        except asyncio.CancelledError:
+            promise.abort()
+            raise
+        finally:
+            disconnect_task.cancel()
+            await asyncio.gather(disconnect_task, return_exceptions=True)
+
     def _record_iteration_stat(self, llm_stat: dict) -> None:
         self.metrics_collector.log_iteration_stats(llm_stat)
         # Tee into the /metrics snapshot buffer so the HTTP handler can serve