diff --git a/docs/source/commands/trtllm-serve/audio-transcriptions.md b/docs/source/commands/trtllm-serve/audio-transcriptions.md new file mode 100644 index 000000000000..e557adbdb41e --- /dev/null +++ b/docs/source/commands/trtllm-serve/audio-transcriptions.md @@ -0,0 +1,79 @@ + + + +# Whisper audio transcriptions + +When serving a supported Whisper checkpoint, `trtllm-serve` accepts short audio +uploads at `POST /v1/audio/transcriptions`. + +Use the encoder-decoder runtime settings from the +[Whisper guide](../../models/encoder-decoder.md#transcribe-audio-with-whisper). +For example, save the following as `whisper.yaml`: + +```yaml +attn_backend: TRTLLM +dtype: float16 +max_batch_size: 4 +max_input_len: 1500 +max_seq_len: 2048 +max_num_tokens: 3000 +enable_chunked_prefill: false +kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + cross_kv_cache_fraction: 0.5 +scheduler_config: + use_python_scheduler: true +``` + +`max_input_len` must cover Whisper's 1500 encoder positions even for a short +recording. This endpoint does not change engine or cross-KV cache sizing. + +```bash +trtllm-serve openai/whisper-large-v3 --backend pytorch --config whisper.yaml --port 8000 +``` + +Submit a multipart request using the model identifier reported by `/v1/models`: + +```bash +curl http://localhost:8000/v1/audio/transcriptions \ + -F model=openai/whisper-large-v3 \ + -F file=@short.wav \ + -F language=en \ + -F response_format=json +``` + +The JSON response is `{"text":"..."}`. Set `response_format=text` for a plain +text response. OpenAI-compatible clients can use the same endpoint: + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:8000/v1", api_key="unused") +with open("short.wav", "rb") as audio: + result = client.audio.transcriptions.create( + model="openai/whisper-large-v3", + file=audio, + language="en", + ) +print(result.text) +``` + +Uploads are limited to 25 MiB and to the checkpoint's input window (normally +30 seconds). Audio is decoded with SoundFile, downmixed to mono, and resampled +to the checkpoint's sampling rate. WAV and FLAC are supported; other formats +depend on the installed libsndfile codecs. Malformed, empty, nonfinite, or +overlong audio is rejected. Split longer recordings on the client side. + +`language` selects the language prefix on multilingual checkpoints. Omitting it +uses the checkpoint's current English-transcription default; automatic language +detection is not implemented. English-only checkpoints accept `language=en`. +`temperature` accepts values from 0 to 1 and defaults to greedy decoding (0). +There is no temperature fallback/retry schedule. + +This endpoint currently supports JSON/text output, one clip per request, and +non-streaming transcription. Nonempty `prompt`, `stream=true`, timestamp +granularities, other response formats, and unknown form fields are rejected +explicitly. Translation, timestamps, long-form windowing, and hotword biasing +are not implemented. The endpoint requires a Whisper model and does not route +through the disaggregated-serving proxy. diff --git a/docs/source/commands/trtllm-serve/index.rst b/docs/source/commands/trtllm-serve/index.rst index bec1594f33f6..33cff39fadc0 100644 --- a/docs/source/commands/trtllm-serve/index.rst +++ b/docs/source/commands/trtllm-serve/index.rst @@ -8,3 +8,4 @@ trtllm-serve trtllm-serve render-endpoints run-benchmark-with-trtllm-serve + audio-transcriptions diff --git a/tensorrt_llm/serve/audio_transcriptions.py b/tensorrt_llm/serve/audio_transcriptions.py new file mode 100644 index 000000000000..50445ec840df --- /dev/null +++ b/tensorrt_llm/serve/audio_transcriptions.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Multipart parsing and audio decoding for short Whisper transcriptions.""" + +from __future__ import annotations + +import asyncio +import io +import math +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Literal, Optional + +import numpy as np +from fastapi import HTTPException, Request +from fastapi.responses import JSONResponse, PlainTextResponse, Response +from numpy.typing import NDArray +from pydantic import BaseModel, ConfigDict, Field, field_validator +from starlette.datastructures import UploadFile + +if TYPE_CHECKING: + from transformers import WhisperConfig, WhisperProcessor + + from tensorrt_llm.inputs.data import TextPrompt + +MAX_AUDIO_UPLOAD_BYTES = 25 * 1024 * 1024 + + +class TranscriptionOptions(BaseModel): + """Supported form fields for the short-clip transcription endpoint.""" + + model_config = ConfigDict(extra="forbid") + + model: str = Field(min_length=1, description="Served Whisper model identifier.") + language: Optional[str] = Field( + default=None, description="Spoken language; omitted uses the checkpoint's English default." + ) + temperature: float = Field( + default=0.0, ge=0.0, le=1.0, allow_inf_nan=False, description="Sampling temperature." + ) + response_format: Literal["json", "text"] = Field( + default="json", description="Transcription response format." + ) + prompt: str = Field(default="", description="Only an empty prompt is currently supported.") + stream: bool = Field(default=False, description="Only non-streaming output is supported.") + + @field_validator("prompt") + @classmethod + def _reject_prompt(cls, value: str) -> str: + if value: + raise ValueError("Prompt conditioning is not supported for audio transcriptions yet.") + return value + + @field_validator("stream") + @classmethod + def _reject_streaming(cls, value: bool) -> bool: + if value: + raise ValueError("Streaming audio transcriptions are not supported yet.") + return value + + +def _decode_audio(data: bytes, sampling_rate: int, max_samples: int) -> NDArray[np.float32]: + # Importing audio codecs/scipy is unnecessary for servers that never transcribe. + import soundfile as sf + from scipy.signal import resample_poly + + try: + with sf.SoundFile(io.BytesIO(data)) as audio_file: + source_rate = audio_file.samplerate + if audio_file.frames == 0: + raise ValueError("The audio file contains no samples.") + if audio_file.frames * sampling_rate > max_samples * source_rate: + raise ValueError( + f"Audio duration exceeds the model's {max_samples / sampling_rate:g}s window. " + "Chunk the input on the client side." + ) + waveform = audio_file.read(dtype="float32", always_2d=True).mean(axis=1) + except (sf.LibsndfileError, sf.SoundFileRuntimeError) as exc: + raise ValueError("Cannot decode audio: upload a supported, nonempty audio file.") from exc + if not np.isfinite(waveform).all(): + raise ValueError("The audio file contains nonfinite samples.") + if source_rate != sampling_rate: + divisor = math.gcd(source_rate, sampling_rate) + waveform = resample_poly(waveform, sampling_rate // divisor, source_rate // divisor) + return np.ascontiguousarray(waveform, dtype=np.float32) + + +async def parse_transcription_request( + request: Request, *, served_model: str, sampling_rate: int, max_samples: int +) -> tuple[TranscriptionOptions, NDArray[np.float32]]: + """Validate one multipart upload and decode mono audio at the model's rate. + + Decoding runs off the event loop. The model window is checked from file + metadata before allocating the decoded waveform. + """ + if request.headers.get("content-type", "").split(";", 1)[0] != "multipart/form-data": + raise ValueError("Audio transcriptions require multipart/form-data with a file upload.") + async with request.form(max_files=1, max_fields=16) as form: + fields: dict[str, str] = {} + upload = None + for name, value in form.multi_items(): + if name == "file": + if not isinstance(value, UploadFile): + raise ValueError("'file' must be an uploaded audio file.") + upload = value + else: + if name in fields or not isinstance(value, str): + raise ValueError(f"Expected one text value for {name!r}.") + fields[name] = value + options = TranscriptionOptions.model_validate(fields) + if options.model != served_model: + raise HTTPException(status_code=404, detail=f"Model {options.model!r} is not served.") + if upload is None: + raise ValueError("A 'file' audio upload is required.") + data = await upload.read(MAX_AUDIO_UPLOAD_BYTES + 1) + if len(data) > MAX_AUDIO_UPLOAD_BYTES: + raise HTTPException(status_code=413, detail="Audio upload exceeds the 25 MiB limit.") + waveform = await asyncio.to_thread(_decode_audio, data, sampling_rate, max_samples) + return options, waveform + + +def transcription_response(text: str, response_format: Literal["json", "text"]) -> Response: + """Render decoded text in the requested OpenAI-compatible response format.""" + if response_format == "text": + return PlainTextResponse(text) + return JSONResponse({"text": text}) + + +async def create_transcription_response( + request: Request, + *, + served_model: str, + model_config: Optional[WhisperConfig], + processor: Optional[WhisperProcessor], + generate: Callable[[TextPrompt, float, Request], Awaitable[str]], +) -> Response: + """Convert one short audio request into Whisper inputs and render the result. + + ``generate`` submits the inputs through the serving engine. Keeping this + conversion independent of the engine permits CPU request-contract tests. + """ + if model_config is None or model_config.model_type != "whisper": + raise ValueError("Audio transcriptions require a Whisper model.") + if processor is None: + raise RuntimeError("The Whisper processor is unavailable.") + extractor = processor.feature_extractor + options, waveform = await parse_transcription_request( + request, + served_model=served_model, + sampling_rate=int(extractor.sampling_rate), + max_samples=int(extractor.n_samples), + ) + prompt_text = "" + if options.language is not None: + try: + forced = processor.get_decoder_prompt_ids( + language=options.language, task="transcribe", no_timestamps=True + ) + except ValueError: + # English-only checkpoints have no language/task tokens. + if options.language != "en": + raise + forced = processor.get_decoder_prompt_ids(no_timestamps=True) + prefix = [int(model_config.decoder_start_token_id)] + prefix.extend(int(token) for _, token in sorted(forced)) + prompt_text = processor.tokenizer.decode(prefix, skip_special_tokens=False) + inputs: TextPrompt = { + "prompt": prompt_text, + "multi_modal_data": {"audio": [(waveform, int(extractor.sampling_rate))]}, + } + text = await generate(inputs, options.temperature, request) + return transcription_response(text, options.response_format) diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 7e7afa6256fc..d3dd588b1e5c 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -47,7 +47,7 @@ from tensorrt_llm.executor.utils import (CONTEXT_LENGTH_EXCEEDED_CODE, is_context_length_exceeded_message) from tensorrt_llm.inputs import prompt_inputs -from tensorrt_llm.inputs.data import TokensPrompt +from tensorrt_llm.inputs.data import TextPrompt, TokensPrompt from tensorrt_llm.inputs.media_io import (BaseMediaIO, log_allowed_local_media_path) from tensorrt_llm.inputs.multimodal import MultimodalServerConfig @@ -1291,6 +1291,9 @@ def register_routes(self): self.app.add_api_route("/v1/completions", self.openai_completion, methods=["POST"]) + self.app.add_api_route("/v1/audio/transcriptions", + self.openai_transcription, + methods=["POST"]) self.app.add_api_route( "/v1/chat/completions", self.openai_chat if not self.use_harmony else self.chat_harmony, @@ -2056,6 +2059,60 @@ def _apply_spec_decode_stats_opt_in(self, postproc_args.spec_decode_num_spec_tokens = getattr( self, "_spec_decode_num_spec_tokens", None) + async def openai_transcription(self, raw_request: Request) -> Response: + """Transcribe one short audio upload with the served Whisper model.""" + from .audio_transcriptions import create_transcription_response + + try: + return await create_transcription_response( + raw_request, + served_model=self.model, + model_config=self.model_config, + processor=self.processor, + generate=self._generate_audio_transcription) + except HTTPException as exc: + return self.create_error_response(str(exc.detail), + status_code=HTTPStatus( + exc.status_code)) + except ValueError as exc: + return self.create_error_response(str(exc)) + except Exception: + logger.error("Audio transcription failed: %s", + traceback.format_exc()) + return self.create_error_response( + "Audio transcription failed.", + err_type="InternalServerError", + status_code=HTTPStatus.INTERNAL_SERVER_ERROR) + + async def _generate_audio_transcription(self, inputs: TextPrompt, + temperature: float, + raw_request: Request) -> str: + sampling_params = SamplingParams( + max_tokens=int(self.model_config.max_target_positions) - 1, + temperature=temperature, + top_k=1 if temperature == 0 else 0) + preprocess_fn = getattr(self.generator, "preprocess", None) + if preprocess_fn is not None: + inputs = await asyncio.get_running_loop().run_in_executor( + self._input_proc_executor, + functools.partial(preprocess_fn, inputs, sampling_params)) + promise = self.generator.generate_async( + inputs=inputs, + sampling_params=sampling_params, + streaming=False, + trace_headers=tracing.extract_trace_headers(raw_request.headers)) + disconnect_task = asyncio.create_task( + self.await_disconnected(raw_request, promise)) + try: + await promise.aresult() + return promise.outputs[0].text + except asyncio.CancelledError: + promise.abort() + raise + finally: + disconnect_task.cancel() + await asyncio.gather(disconnect_task, return_exceptions=True) + def _record_iteration_stat(self, llm_stat: dict) -> None: self.metrics_collector.log_iteration_stats(llm_stat) # Tee into the /metrics snapshot buffer so the HTTP handler can serve diff --git a/tests/unittest/serve/test_audio_transcriptions.py b/tests/unittest/serve/test_audio_transcriptions.py new file mode 100644 index 000000000000..d09458b7ed99 --- /dev/null +++ b/tests/unittest/serve/test_audio_transcriptions.py @@ -0,0 +1,290 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU contract tests for multipart Whisper transcription requests.""" + +import importlib.util +import io +import sys +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +import soundfile as sf +from fastapi import FastAPI, HTTPException, Request +from fastapi.testclient import TestClient + +# Load this runtime-independent module without initializing TensorRT/CUDA. +_SOURCE = Path(__file__).resolve().parents[3] / "tensorrt_llm/serve/audio_transcriptions.py" +_SPEC = importlib.util.spec_from_file_location("audio_transcriptions_cpu_test", _SOURCE) +audio = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = audio +_SPEC.loader.exec_module(audio) + +pytestmark = pytest.mark.cpu_only + + +def _wav(samples: np.ndarray, rate: int = 16000) -> bytes: + buffer = io.BytesIO() + sf.write(buffer, samples, rate, format="WAV", subtype="FLOAT") + return buffer.getvalue() + + +@pytest.fixture +def client() -> TestClient: + app = FastAPI() + + @app.post("/v1/audio/transcriptions") + async def transcribe(request: Request): + try: + options, waveform = await audio.parse_transcription_request( + request, served_model="whisper-test", sampling_rate=16000, max_samples=16000 + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return { + "options": options.model_dump(), + "samples": len(waveform), + "dtype": str(waveform.dtype), + "peak": float(np.max(np.abs(waveform))), + } + + return TestClient(app) + + +def _post(client: TestClient, *, data=None, content=None): + return client.post( + "/v1/audio/transcriptions", + data={"model": "whisper-test", **(data or {})}, + files={ + "file": ( + "clip.wav", + content if content is not None else _wav(np.zeros(1600, dtype=np.float32)), + "audio/wav", + ) + }, + ) + + +def test_valid_upload_and_defaults(client: TestClient) -> None: + response = _post(client) + assert response.status_code == 200 + body = response.json() + assert body["samples"] == 1600 + assert body["dtype"] == "float32" + assert body["options"]["response_format"] == "json" + assert body["options"]["temperature"] == 0 + assert body["options"]["language"] is None + + +def test_explicit_options(client: TestClient) -> None: + response = _post( + client, + data={"language": "de", "temperature": "0.4", "response_format": "text", "stream": "false"}, + ) + assert response.status_code == 200 + assert response.json()["options"]["language"] == "de" + assert response.json()["options"]["temperature"] == 0.4 + + +@pytest.mark.parametrize( + "fields", + [ + {"temperature": "nan"}, + {"temperature": "inf"}, + {"temperature": "-0.1"}, + {"temperature": "1.1"}, + {"response_format": "verbose_json"}, + {"stream": "true"}, + {"prompt": "previous transcript"}, + {"timestamp_granularities[]": "word"}, + {"typo": "value"}, + ], +) +def test_unsupported_or_invalid_options(client: TestClient, fields: dict) -> None: + assert _post(client, data=fields).status_code == 400 + + +def test_unknown_model(client: TestClient) -> None: + assert _post(client, data={"model": "wrong"}).status_code == 404 + + +@pytest.mark.parametrize("content", [b"", b"not an audio file"]) +def test_invalid_audio(client: TestClient, content: bytes) -> None: + assert _post(client, content=content).status_code == 400 + + +def test_empty_waveform(client: TestClient) -> None: + assert _post(client, content=_wav(np.zeros(0))).status_code == 400 + + +def test_nonfinite_waveform(client: TestClient) -> None: + assert _post(client, content=_wav(np.array([np.nan]))).status_code == 400 + + +def test_resamples_and_downmixes(client: TestClient) -> None: + stereo = np.column_stack((np.ones(800), -np.ones(800))) + response = _post(client, content=_wav(stereo, rate=8000)) + assert response.status_code == 200 + assert response.json()["samples"] == 1600 + assert response.json()["peak"] == 0 + + +def test_duration_limit_and_exact_boundary(client: TestClient) -> None: + assert _post(client, content=_wav(np.zeros(16000))).status_code == 200 + response = _post(client, content=_wav(np.zeros(16001))) + assert response.status_code == 400 + assert "duration" in response.json()["detail"].lower() + + +def test_missing_file(client: TestClient) -> None: + response = client.post( + "/v1/audio/transcriptions", + data={"model": "whisper-test"}, + files={"other": ("clip.wav", b"audio")}, + ) + assert response.status_code == 400 + + +def test_missing_model(client: TestClient) -> None: + response = client.post( + "/v1/audio/transcriptions", files={"file": ("clip.wav", _wav(np.zeros(10)))} + ) + assert response.status_code == 400 + + +def test_duplicate_fields(client: TestClient) -> None: + response = client.post( + "/v1/audio/transcriptions", + files=[ + ("model", (None, "whisper-test")), + ("model", (None, "wrong")), + ("file", ("clip.wav", _wav(np.zeros(10)), "audio/wav")), + ], + ) + assert response.status_code == 400 + + +def test_file_limit(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(audio, "MAX_AUDIO_UPLOAD_BYTES", 20) + assert _post(client).status_code == 413 + + +def test_text_and_json_responses() -> None: + assert audio.transcription_response("hello", "json").body == b'{"text":"hello"}' + response = audio.transcription_response("hello", "text") + assert response.body == b"hello" + assert response.media_type == "text/plain" + + +class _Processor: + def __init__(self) -> None: + self.feature_extractor = SimpleNamespace(sampling_rate=16000, n_samples=16000) + self.tokenizer = SimpleNamespace(decode=self.decode) + self.english_only = False + self.prefix_calls = [] + + def get_decoder_prompt_ids(self, **kwargs) -> list: + self.prefix_calls.append(kwargs) + if self.english_only: + if "language" in kwargs: + raise ValueError("English-only tokenizer does not accept a language.") + return [] + if kwargs["language"] not in ("de", "en"): + raise ValueError("Unsupported language.") + return [(3, 13), (1, 11), (2, 12)] + + @staticmethod + def decode(tokens: list[int], *, skip_special_tokens: bool) -> str: + assert not skip_special_tokens + return " ".join(map(str, tokens)) + + +@pytest.fixture +def transcription_client() -> TestClient: + app = FastAPI() + app.state.processor = _Processor() + app.state.model_config = SimpleNamespace(model_type="whisper", decoder_start_token_id=10) + app.state.calls = [] + app.state.fail_generation = False + + async def generate(inputs: dict, temperature: float, request: Request) -> str: + app.state.calls.append((inputs, temperature, request)) + if app.state.fail_generation: + raise RuntimeError("engine failed") + return "Hello world." + + @app.post("/v1/audio/transcriptions") + async def transcribe(request: Request): + try: + return await audio.create_transcription_response( + request, + served_model="whisper-test", + model_config=app.state.model_config, + processor=app.state.processor, + generate=generate, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + return TestClient(app, raise_server_exceptions=False) + + +def test_transcription_handoff(transcription_client: TestClient) -> None: + response = _post(transcription_client, data={"temperature": "0.4"}) + assert response.status_code == 200 + assert response.json() == {"text": "Hello world."} + inputs, temperature, _ = transcription_client.app.state.calls[0] + assert inputs["prompt"] == "" + waveform, sampling_rate = inputs["multi_modal_data"]["audio"][0] + assert waveform.shape == (1600,) + assert waveform.dtype == np.float32 + assert sampling_rate == 16000 + assert temperature == 0.4 + assert transcription_client.app.state.processor.prefix_calls == [] + + +def test_transcription_text_output(transcription_client: TestClient) -> None: + response = _post(transcription_client, data={"response_format": "text"}) + assert response.status_code == 200 + assert response.text == "Hello world." + assert response.headers["content-type"].startswith("text/plain") + + +def test_language_prefix(transcription_client: TestClient) -> None: + assert _post(transcription_client, data={"language": "de"}).status_code == 200 + inputs, _, _ = transcription_client.app.state.calls[0] + assert inputs["prompt"] == "10 11 12 13" + assert transcription_client.app.state.processor.prefix_calls == [ + {"language": "de", "task": "transcribe", "no_timestamps": True} + ] + + +def test_unsupported_language_does_not_generate(transcription_client: TestClient) -> None: + assert _post(transcription_client, data={"language": "xx"}).status_code == 400 + assert transcription_client.app.state.calls == [] + + +def test_english_only_checkpoint(transcription_client: TestClient) -> None: + transcription_client.app.state.processor.english_only = True + assert _post(transcription_client, data={"language": "en"}).status_code == 200 + assert transcription_client.app.state.calls[0][0]["prompt"] == "10" + assert _post(transcription_client, data={"language": "de"}).status_code == 400 + assert len(transcription_client.app.state.calls) == 1 + + +def test_wrong_model_type_does_not_generate(transcription_client: TestClient) -> None: + transcription_client.app.state.model_config.model_type = "llama" + assert _post(transcription_client).status_code == 400 + assert transcription_client.app.state.calls == [] + + +def test_missing_processor_does_not_generate(transcription_client: TestClient) -> None: + transcription_client.app.state.processor = None + assert _post(transcription_client).status_code == 500 + assert transcription_client.app.state.calls == [] + + +def test_engine_failure_does_not_return_success(transcription_client: TestClient) -> None: + transcription_client.app.state.fail_generation = True + assert _post(transcription_client).status_code == 500 diff --git a/tests/unittest/serve/test_openai_audio_transcriptions.py b/tests/unittest/serve/test_openai_audio_transcriptions.py new file mode 100644 index 000000000000..e870bad22fef --- /dev/null +++ b/tests/unittest/serve/test_openai_audio_transcriptions.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Serving integration checks using a fake generator, without GPU inference.""" + +import asyncio +import io +from types import SimpleNamespace + +import numpy as np +import pytest +import soundfile as sf +from fastapi.testclient import TestClient +from test_common.serve.app_factory import AggConfig +from test_common.serve.routes import effective_routes + +pytestmark = pytest.mark.cpu_only + + +class _Result: + def __init__(self, *, fail: bool = False, cancel: bool = False) -> None: + self.outputs = [SimpleNamespace(text="A short transcript.")] + self.fail = fail + self.cancel = cancel + self.aborted = False + + async def aresult(self): + await asyncio.sleep(0) + if self.cancel: + raise asyncio.CancelledError + if self.fail: + raise RuntimeError("backend failed") + return self + + def abort(self) -> None: + self.aborted = True + + +@pytest.fixture +def server(serve_env): + instance = serve_env.build_openai_server(AggConfig(model_name="whisper-test")) + instance.model_config = SimpleNamespace( + model_type="whisper", decoder_start_token_id=1, max_target_positions=448 + ) + instance.processor = SimpleNamespace( + feature_extractor=SimpleNamespace(sampling_rate=16000, n_samples=16000) + ) + instance.result = _Result() + instance.calls = [] + instance.preprocessed = [] + instance.watch_stopped = False + + def preprocess(inputs, sampling_params): + instance.preprocessed.append((inputs, sampling_params)) + return inputs + + def generate(**kwargs): + instance.calls.append(kwargs) + return instance.result + + async def watch(request, result): + try: + await asyncio.Event().wait() + finally: + instance.watch_stopped = True + + instance.await_disconnected = watch + instance.generator.preprocess = preprocess + instance.generator.generate_async = generate + instance._transcription_test_client = serve_env.client(instance.app, lifespan=False) + yield instance + + +def _upload(client: TestClient, **fields): + buffer = io.BytesIO() + sf.write(buffer, np.zeros(1600), 16000, format="WAV") + return client.post( + "/v1/audio/transcriptions", + data={"model": "whisper-test", **fields}, + files={"file": ("short.wav", buffer.getvalue(), "audio/wav")}, + ) + + +def _client(server) -> TestClient: + return server._transcription_test_client + + +def test_generation_and_disconnect_cleanup(server) -> None: + response = _upload(_client(server)) + assert response.status_code == 200 + assert response.json() == {"text": "A short transcript."} + assert len(server.preprocessed) == 1 + params = server.calls[0]["sampling_params"] + assert params.max_tokens == 447 + assert params.temperature == 0 + assert params.top_k == 1 + assert server.calls[0]["streaming"] is False + assert server.watch_stopped + assert not server.result.aborted + + +def test_sampling_temperature_and_text_response(server) -> None: + response = _upload(_client(server), temperature="0.4", response_format="text") + assert response.status_code == 200 + assert response.text == "A short transcript." + assert server.calls[0]["sampling_params"].temperature == 0.4 + assert server.calls[0]["sampling_params"].top_k == 0 + + +def test_backend_error_is_500_and_cleans_up(server) -> None: + server.result.fail = True + response = _upload(_client(server)) + assert response.status_code == 500 + assert response.json()["type"] == "InternalServerError" + assert server.watch_stopped + + +def test_unknown_model_error_envelope(server) -> None: + response = _upload(_client(server), model="other") + assert response.status_code == 404 + assert response.json()["object"] == "error" + assert response.json()["code"] == 404 + assert server.calls == [] + + +def test_validation_error_envelope(server) -> None: + response = _upload(_client(server), prompt="context") + assert response.status_code == 400 + assert response.json()["type"] == "BadRequestError" + assert server.calls == [] + + +def test_cancelled_generation_aborts_backend(server) -> None: + server.result.cancel = True + + async def run() -> None: + with pytest.raises(asyncio.CancelledError): + await server._generate_audio_transcription( + {"prompt": "", "multi_modal_data": {}}, + 0.0, + SimpleNamespace(headers={}), + ) + + asyncio.run(run()) + assert server.result.aborted + assert server.watch_stopped + + +def test_route_is_registered(server) -> None: + routes = [ + route for path, route in effective_routes(server.app) if path == "/v1/audio/transcriptions" + ] + assert len(routes) == 1 + assert routes[0].methods == {"POST"}