From e7bdf9e11827d016cacf761b5765068575760b81 Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:26:38 +0000 Subject: [PATCH 01/11] [None][refactor] resolve unsupported prompt-tuning request state Audit prompt-adapter preparation through request translation, model inputs, and cache keys. BaseWorker prepended virtual tokens and forwarded the table, but the PyTorch model path only consumes normal embeddings or Python-owned multimodal embeddings. No production reader uses the native prompt table or vocabulary size, and LlmRequest::validate has no production caller. Reject high-level prompt adapters before loading or token mutation. Translate direct executor prompt configs into a rejection marker and fail them during active admission, preserving request-scoped errors and valid batch peers. Reject native executor-to-LlmRequest prompt-config conversion as well. Remove native prompt table/vocabulary storage, constructor arguments, accessors, bindings, dead translation, and obsolete storage/validation tests. Retain independently live extra IDs, unique cache tokens, original lengths, and child behavior. Document the unsupported public parameter behavior and correct the stale VILA model-input comment. Internal C++ request layouts/signatures and nanobind constructor/property surfaces change and require matching rebuilt libraries and bindings. Public PromptTuningConfig and executor Request serialization/pickle layouts remain unchanged; public API retirement is outside this change. Validation: - Rebuilt and installed editable package with run/run_build_wheel.sh, --job_count 68, RelWithDebInfo, SM80. - 394 CPU executor/request-utils/worker tests passed. - 57 native binding tests passed, including public Request/config round trips. - 120 of 129 native request/cache/scheduler/agent tests passed. Nine existing KV partial-copy cases require libtensorrt_llm_nixl_wrapper.so, absent from this build without NIXL_ROOT. - Admission regressions cover identical virtual IDs with different tables, optional extra IDs, request phases, cache-manager selections, children, native pickle transport, and unaffected valid peers. Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- .../tensorrt_llm/batch_manager/llmRequest.h | 61 +--------- cpp/tensorrt_llm/batch_manager/llmRequest.cpp | 12 +- .../nanobind/batch_manager/bindings.cpp | 21 ++-- .../batch_manager/agentTreeTest.cpp | 2 - .../batch_manager/kvCacheManagerTest.cpp | 94 +++++---------- .../batch_manager/llmRequestTest.cpp | 107 +++++++----------- .../batch_manager/microBatchSchedulerTest.cpp | 1 - tensorrt_llm/_torch/models/modeling_vila.py | 2 +- tensorrt_llm/_torch/pyexecutor/llm_request.py | 9 +- tensorrt_llm/_torch/pyexecutor/py_executor.py | 3 + tensorrt_llm/executor/base_worker.py | 32 +----- tensorrt_llm/llmapi/llm.py | 4 +- .../_torch/pyexecutor/test_py_executor.py | 49 ++++++++ tests/unittest/bindings/test_bindings_ut.py | 5 - tests/unittest/executor/test_base_worker.py | 12 ++ 15 files changed, 158 insertions(+), 256 deletions(-) diff --git a/cpp/include/tensorrt_llm/batch_manager/llmRequest.h b/cpp/include/tensorrt_llm/batch_manager/llmRequest.h index 788c2255c003..dc08360d395a 100644 --- a/cpp/include/tensorrt_llm/batch_manager/llmRequest.h +++ b/cpp/include/tensorrt_llm/batch_manager/llmRequest.h @@ -112,8 +112,6 @@ class GenericLlmRequest GenericLlmRequest(RequestIdType requestId, SizeType32 maxNewTokens, std::shared_ptr const& inputTokens, executor::SamplingConfig const& samplingConfig, bool isStreaming, - std::optional promptEmbeddingTable = std::nullopt, - std::optional promptVocabSize = std::nullopt, std::optional>>> multimodalHashes = std::nullopt, std::optional>> multimodalPositions = std::nullopt, std::optional>> multimodalLengths = std::nullopt, @@ -144,8 +142,6 @@ class GenericLlmRequest , mIsStreaming(isStreaming) , mOrigPromptLen(mPromptLen) , mMaxSentTokenLen(mPromptLen) - , mPromptEmbeddingTable(std::move(promptEmbeddingTable)) - , mPromptVocabSize(promptVocabSize) , mMultimodalHashes(std::move(multimodalHashes)) , mMultimodalPositions(std::move(multimodalPositions)) , mMultimodalLengths(std::move(multimodalLengths)) @@ -188,8 +184,6 @@ class GenericLlmRequest GenericLlmRequest(RequestIdType requestId, SizeType32 maxNewTokens, VecTokens const& inputTokens, executor::SamplingConfig const& samplingConfig, bool isStreaming, - std::optional promptEmbeddingTable = std::nullopt, - std::optional promptVocabSize = std::nullopt, std::optional loraTaskId = std::nullopt, std::optional loraWeights = std::nullopt, std::optional loraConfig = std::nullopt, std::optional draftTokens = std::nullopt, bool excludeInputFromOutput = false, std::optional encoderInputTokens = std::nullopt, @@ -203,8 +197,6 @@ class GenericLlmRequest , mIsStreaming(isStreaming) , mOrigPromptLen(mPromptLen) , mMaxSentTokenLen(mPromptLen) - , mPromptEmbeddingTable(std::move(promptEmbeddingTable)) - , mPromptVocabSize(promptVocabSize) , mLoraTaskId(loraTaskId) , mLoraWeights(std::move(loraWeights)) , mLoraConfig(std::move(loraConfig)) @@ -271,21 +263,8 @@ class GenericLlmRequest } } - auto pTuningConfig = req.getPromptTuningConfig(); - if (pTuningConfig) - { - mPromptEmbeddingTable = tensorrt_llm::runtime::ITensor::view( - executor::detail::toITensor(pTuningConfig.value().getEmbeddingTable())); - TLLM_CHECK(mPromptEmbeddingTable.value()->getShape().nbDims == 2); - mPromptVocabSize = mPromptEmbeddingTable.value()->getShape().d[0]; - mPromptEmbeddingTable.value()->unsqueeze(0); - - if (pTuningConfig->getInputTokenExtraIds()) - { - mInputTokenExtraIds - = std::make_shared(pTuningConfig->getInputTokenExtraIds().value()); - } - } + TLLM_CHECK_WITH_INFO( + !req.getPromptTuningConfig().has_value(), "Prompt tuning is not supported by the PyTorch executor."); auto multimodalInput = req.getMultimodalInput(); if (multimodalInput) { @@ -776,21 +755,6 @@ class GenericLlmRequest mMaxSentTokenLen = maxSentLength; } - [[nodiscard]] std::optional getPromptEmbeddingTable() const - { - return mPromptEmbeddingTable; - } - - [[nodiscard]] std::optional& getPromptEmbeddingTableMutable() - { - return mPromptEmbeddingTable; - } - - [[nodiscard]] std::optional getPromptVocabSize() const - { - return mPromptVocabSize; - } - [[nodiscard]] std::optional>>> getMultimodalHashes() const { return mMultimodalHashes; @@ -846,11 +810,6 @@ class GenericLlmRequest mLoraWeights = weights; } - void setPromptVocabSize(SizeType32 size) - { - mPromptVocabSize = size; - } - void clearLoraWeights() { mLoraWeights = std::nullopt; @@ -1534,8 +1493,6 @@ class GenericLlmRequest SizeType32 mMaxSentTokenLen; - std::optional mPromptEmbeddingTable{std::nullopt}; - std::optional mPromptVocabSize{std::nullopt}; std::optional>>> mMultimodalHashes{std::nullopt}; std::optional>> mMultimodalPositions{std::nullopt}; std::optional>> mMultimodalLengths{std::nullopt}; @@ -1686,16 +1643,6 @@ class GenericLlmRequest mEncoderUniqueTokens = encoderUniqueTokens; } - if ((mPromptEmbeddingTable.has_value() && !mPromptVocabSize.has_value()) - || (!mPromptEmbeddingTable.has_value() && mPromptVocabSize.has_value())) - { - std::string errStr - = "Prompt embedding table and prompt vocab size tensors must both be provided for requests with " - "prompt " - "tuning enabled."; - TLLM_THROW(errStr); - } - if (!isChild()) { // Initialize result states unless it is a child and a child request should share parent's one. @@ -1742,8 +1689,6 @@ class LlmRequest : public GenericLlmRequest LlmRequest(RequestIdType requestId, SizeType32 maxNewTokens, std::vector inputTokens, executor::SamplingConfig const& samplingConfig, bool isStreaming, - std::optional promptEmbeddingTable = std::nullopt, - std::optional promptVocabSize = std::nullopt, std::optional>> multimodalHashes = std::nullopt, std::optional> multimodalPositions = std::nullopt, std::optional> multimodalLengths = std::nullopt, @@ -1766,7 +1711,7 @@ class LlmRequest : public GenericLlmRequest std::optional> multimodalRunLengths = std::nullopt, std::optional cacheSalt = std::nullopt) : Base(requestId, maxNewTokens, std::make_shared>(std::move(inputTokens)), - samplingConfig, isStreaming, std::move(promptEmbeddingTable), promptVocabSize, + samplingConfig, isStreaming, multimodalHashes.has_value() ? std::make_shared>>(std::move(multimodalHashes.value())) : std::optional>>>(std::nullopt), diff --git a/cpp/tensorrt_llm/batch_manager/llmRequest.cpp b/cpp/tensorrt_llm/batch_manager/llmRequest.cpp index eda9da60d093..5f5a73e15547 100644 --- a/cpp/tensorrt_llm/batch_manager/llmRequest.cpp +++ b/cpp/tensorrt_llm/batch_manager/llmRequest.cpp @@ -198,7 +198,7 @@ bool LlmRequest::checkTokenIdRange(SizeType32 vocabSize) } void LlmRequest::validate( - SizeType32 maxInputLen, SizeType32 maxSequenceLen, SizeType32 maxDraftLen, bool enableKVCacheReuse) + SizeType32 maxInputLen, SizeType32 maxSequenceLen, SizeType32 maxDraftLen, bool /*enableKVCacheReuse*/) { if (mPromptLen > maxInputLen) { @@ -244,16 +244,6 @@ void LlmRequest::validate( mPromptLen, mMaxNewTokens, draftLenPerEngineStep, maxSequenceLen, maxNewTokens); mMaxNewTokens = maxNewTokens; } - - // validate extra ids when enabling kv cache reuse with prompt table - if (enableKVCacheReuse && mPromptEmbeddingTable.has_value() && mPromptVocabSize.has_value()) - { - TLLM_CHECK_WITH_INFO(mInputTokenExtraIds.has_value() && mInputTokenExtraIds.value(), - "Input token extra ids must be provided when enabling kv cache reuse with prompt table"); - TLLM_CHECK_WITH_INFO(mInputTokenExtraIds.value()->size() == static_cast(mOrigPromptLen), - "inputTokenExtraIds vector size (%lu) must be the same as input token vector size (%lu).", - mInputTokenExtraIds.value()->size(), static_cast(mOrigPromptLen)); - } } LlmRequest::LlmRequest(LlmRequest const& parent, RequestIdType requestId) diff --git a/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp b/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp index 8454026f5791..ccad76f837c5 100644 --- a/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp @@ -150,10 +150,8 @@ void initBindings(nb::module_& m) .def("set_generated_tokens", &GenLlmReq::setGeneratedTokens, nb::arg("generated_beam_tokens")) .def("pause", &GenLlmReq::pause, nb::arg("max_input_len")) .def_prop_rw("max_sent_token_len", &GenLlmReq::getMaxSentTokenLen, &GenLlmReq::setMaxSentTokenLen) - .def_prop_ro("prompt_embedding_table", &GenLlmReq::getPromptEmbeddingTable) .def_prop_rw("lora_config", &GenLlmReq::getLoraConfig, &GenLlmReq::setLoraConfig) .def_prop_rw("lora_weights", &GenLlmReq::getLoraWeights, &GenLlmReq::setLoraWeights) - .def_prop_ro("prompt_vocab_size", &GenLlmReq::getPromptVocabSize) .def_prop_ro("lora_task_id", &GenLlmReq::getLoraTaskId) .def_prop_rw("context_chunk_size", &GenLlmReq::getContextChunkSize, &GenLlmReq::setContextChunkSize) .def_prop_rw("decoding_iter", &GenLlmReq::getDecodingIter, &GenLlmReq::setDecodingIter) @@ -359,8 +357,6 @@ void initBindings(nb::module_& m) [](tb::LlmRequest* self, tb::LlmRequest::RequestIdType request_id, tb::LlmRequest::SizeType32 max_new_tokens, std::vector input_tokens, executor::SamplingConfig sampling_config, bool is_streaming, - std::optional prompt_embedding_table, - std::optional prompt_vocab_size, std::optional>> multimodal_hashes, std::optional> multimodal_positions, std::optional> multimodal_lengths, @@ -391,22 +387,19 @@ void initBindings(nb::module_& m) return tensorPtr; }; - auto prompt_embedding_table_tensor_ptr = makeOptionalTensor(prompt_embedding_table); auto lora_weights_tensor_ptr = makeOptionalTensor(lora_weights); auto lora_config_tensor_ptr = makeOptionalTensor(lora_config); new (self) tb::LlmRequest{request_id, max_new_tokens, input_tokens, sampling_config, is_streaming, - prompt_embedding_table_tensor_ptr, prompt_vocab_size, multimodal_hashes, multimodal_positions, - multimodal_lengths, multimodal_uuids, lora_task_id, lora_weights_tensor_ptr, lora_config_tensor_ptr, - kv_cache_retention_config, draft_tokens, exclude_input_from_output, encoder_input_tokens, priority, - encoder_input_length, encoder_output_length, llm_request_type, input_token_extra_ids, - return_perf_metrics, context_phase_params, arrival_time, std::move(agent_hierarchy), - multimodal_item_run_cu_offsets, multimodal_run_positions, multimodal_run_lengths, - std::move(cache_salt)}; + multimodal_hashes, multimodal_positions, multimodal_lengths, multimodal_uuids, lora_task_id, + lora_weights_tensor_ptr, lora_config_tensor_ptr, kv_cache_retention_config, draft_tokens, + exclude_input_from_output, encoder_input_tokens, priority, encoder_input_length, + encoder_output_length, llm_request_type, input_token_extra_ids, return_perf_metrics, + context_phase_params, arrival_time, std::move(agent_hierarchy), multimodal_item_run_cu_offsets, + multimodal_run_positions, multimodal_run_lengths, std::move(cache_salt)}; }, nb::arg("request_id"), nb::arg("max_new_tokens"), nb::arg("input_tokens"), nb::arg("sampling_config"), - nb::arg("is_streaming"), nb::arg("prompt_embedding_table") = std::nullopt, - nb::arg("prompt_vocab_size") = std::nullopt, nb::arg("multimodal_hashes") = std::nullopt, + nb::arg("is_streaming"), nb::arg("multimodal_hashes") = std::nullopt, nb::arg("multimodal_positions") = std::nullopt, nb::arg("multimodal_lengths") = std::nullopt, nb::arg("multimodal_uuids") = std::nullopt, nb::arg("lora_task_id") = std::nullopt, nb::arg("lora_weights") = std::nullopt, nb::arg("lora_config") = std::nullopt, diff --git a/cpp/tests/unit_tests/batch_manager/agentTreeTest.cpp b/cpp/tests/unit_tests/batch_manager/agentTreeTest.cpp index e39c0b1e137b..e38a50ea1eb8 100644 --- a/cpp/tests/unit_tests/batch_manager/agentTreeTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/agentTreeTest.cpp @@ -58,8 +58,6 @@ class AgentTreeTest : public ::testing::Test tensorrt_llm::executor::SamplingConfig samplingConfig(1); return std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, false, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, diff --git a/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp b/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp index 6501396a79ff..1b10ff8f0822 100644 --- a/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp @@ -1227,8 +1227,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdTest) auto const inputLength = static_cast(inputTokens->size()); LlmRequest::RequestIdType requestId{0}; auto llmRequest0 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, @@ -1269,8 +1268,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdTest) // new request with same tokens and then remove it requestId = 1; auto llmRequest1 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, @@ -1306,8 +1304,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdTest) // reuse blocks 0, 1 and get recycled block 3 GenerationRequest seq0_dup{10, inputLength, beamWidth, blockManager.getWindowSizesMetadata()}; llmRequest0 = std::make_shared(seq0_dup.getRequestId(), maxNewTokens, inputTokens, samplingConfig, - isStreaming, /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + isStreaming, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, @@ -1337,8 +1334,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdTest) inputTokenExtraIds1->push_back(0); GenerationRequest seq1_dup{11, inputLength, beamWidth, blockManager.getWindowSizesMetadata()}; llmRequest1 = std::make_shared(seq1_dup.getRequestId(), maxNewTokens, inputTokens1, samplingConfig, - isStreaming, /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + isStreaming, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, @@ -1376,8 +1372,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdTest) auto inputTokenExtraIds2 = std::make_shared(VecTokenExtraIds{4, 4, 5, 5, 6, 6, 0, 0, 0}); requestId = 2; auto llmRequest2 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, @@ -1409,8 +1404,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdTest) auto inputTokenExtraIds3 = std::make_shared(VecTokenExtraIds{1, 1, 2, 2, 4, 4, 0, 0, 0}); requestId = 3; auto llmRequest3 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, @@ -1490,8 +1484,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithMultimodalHashTest) auto const inputLength = static_cast(inputTokens->size()); LlmRequest::RequestIdType requestId{0}; auto llmRequest0 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, multimodalHashes, multimodalPositions, multimodalLengths, + multimodalHashes, multimodalPositions, multimodalLengths, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, @@ -1538,8 +1531,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithMultimodalHashTest) // new request with same tokens and same multimodal hash - should reuse requestId = 1; auto llmRequest1 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, multimodalHashes, multimodalPositions, multimodalLengths, + multimodalHashes, multimodalPositions, multimodalLengths, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, @@ -1581,8 +1573,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithMultimodalHashTest) = std::make_shared>(std::vector{2}); // Start at token 2 auto multimodalLengths2 = std::make_shared>(std::vector{4}); // Length 4 tokens auto llmRequest2 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, multimodalHashes2, multimodalPositions2, multimodalLengths2, + multimodalHashes2, multimodalPositions2, multimodalLengths2, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, @@ -1623,8 +1614,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithMultimodalHashTest) = std::make_shared>(std::vector{2, 2}); // Length 2 tokens auto llmRequest3 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, multimodalHashes3, multimodalPositions3, multimodalLengths3, + multimodalHashes3, multimodalPositions3, multimodalLengths3, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, @@ -1701,8 +1691,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithLoraTaskIdTest) auto const inputLength = static_cast(inputTokens->size()); LlmRequest::RequestIdType requestId{0}; auto llmRequest0 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId); GenerationRequest seq0{requestId, inputLength, beamWidth, blockManager.getWindowSizesMetadata()}; @@ -1739,8 +1728,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithLoraTaskIdTest) // inputTokens = (0, 1, 2, 3, 4, 5, 6, 7, 8) requestId = 1; auto llmRequest1 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId); GenerationRequest seq1{requestId, inputLength, beamWidth, blockManager.getWindowSizesMetadata()}; @@ -1772,8 +1760,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithLoraTaskIdTest) // reuse blocks 0, 1 and get recycled block 3 GenerationRequest seq0_dup{10, inputLength, beamWidth, blockManager.getWindowSizesMetadata()}; llmRequest0 = std::make_shared(seq0_dup.getRequestId(), maxNewTokens, inputTokens, samplingConfig, - isStreaming, /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + isStreaming, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId); promptLen0 = llmRequest0->getNumTokens(beamIdx); @@ -1799,8 +1786,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithLoraTaskIdTest) auto inputTokens1 = std::make_shared(llmRequest1->getTokens(0)); GenerationRequest seq1_dup{11, inputLength, beamWidth, blockManager.getWindowSizesMetadata()}; llmRequest1 = std::make_shared(seq1_dup.getRequestId(), maxNewTokens, inputTokens1, samplingConfig, - isStreaming, /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + isStreaming, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId); promptLen1 = llmRequest1->getNumTokens(beamIdx); @@ -1834,8 +1820,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithLoraTaskIdTest) loraTaskId = static_cast(1); requestId = 2; auto llmRequest2 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId); numTokens = llmRequest2->getNumTokens(beamIdx); @@ -1868,8 +1853,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithLoraTaskIdTest) auto inputTokens3 = std::make_shared(VecTokens{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11}); requestId = 3; auto llmRequest3 = std::make_shared(requestId, maxNewTokens, inputTokens3, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId); numTokens = llmRequest3->getNumTokens(beamIdx); @@ -1903,8 +1887,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithLoraTaskIdTest) auto inputTokens4 = std::make_shared(VecTokens{0, 1, 2, 3, 4}); requestId = 4; auto llmRequest4 = std::make_shared(requestId, maxNewTokens, inputTokens4, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId); numTokens = llmRequest4->getNumTokens(beamIdx); @@ -1933,9 +1916,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithLoraTaskIdTest) // add request with same tokens as request0 but without loraTaskId requestId = 5; - auto llmRequest5 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt); + auto llmRequest5 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming); numTokens = llmRequest5->getNumTokens(beamIdx); GenerationRequest seq5{requestId, numTokens, beamWidth, blockManager.getWindowSizesMetadata()}; @@ -2004,8 +1985,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdAndLoraTaskIdTest) LlmRequest::LoraTaskIdType loraTaskId1{1}; LlmRequest::RequestIdType requestId{0}; auto llmRequest0 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId1, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, /*encoderInputTokens=*/std::nullopt, @@ -2047,8 +2027,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdAndLoraTaskIdTest) requestId = 1; LlmRequest::LoraTaskIdType loraTaskId2 = static_cast(2); auto llmRequest1 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId2, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, /*encoderInputTokens=*/std::nullopt, @@ -2083,8 +2062,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdAndLoraTaskIdTest) // add both requests again and then remove them GenerationRequest seq0_dup{10, inputLength, beamWidth, blockManager.getWindowSizesMetadata()}; llmRequest0 = std::make_shared(seq0_dup.getRequestId(), maxNewTokens, inputTokens, samplingConfig, - isStreaming, /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + isStreaming, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId1, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, @@ -2113,8 +2091,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdAndLoraTaskIdTest) inputTokenExtraIds1->push_back(0); GenerationRequest seq1_dup{11, inputLength, beamWidth, blockManager.getWindowSizesMetadata()}; llmRequest1 = std::make_shared(seq1_dup.getRequestId(), maxNewTokens, inputTokens1, samplingConfig, - isStreaming, /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + isStreaming, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId2, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, @@ -2149,8 +2126,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdAndLoraTaskIdTest) auto inputTokenExtraIds2 = std::make_shared(VecTokenExtraIds{4, 4, 5, 5, 6, 6, 0, 0, 0}); requestId = 2; auto llmRequest2 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId1, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, /*encoderInputTokens=*/std::nullopt, @@ -2182,8 +2158,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdAndLoraTaskIdTest) auto inputTokenExtraIds3 = std::make_shared(VecTokenExtraIds{1, 1, 2, 2, 4, 4, 0, 0, 0}); requestId = 3; auto llmRequest3 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId1, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, /*encoderInputTokens=*/std::nullopt, @@ -2214,8 +2189,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdAndLoraTaskIdTest) // add request with partial different extra ids and loraTaskId 2 requestId = 4; auto llmRequest4 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, loraTaskId2, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, /*encoderInputTokens=*/std::nullopt, @@ -2295,8 +2269,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithCacheSaltTest) // Test Case 1: Request without cache_salt LlmRequest::RequestIdType requestId{0}; auto llmRequest0 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, @@ -2342,8 +2315,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithCacheSaltTest) requestId = 1; std::string const cacheSalt1{"tenant-A"}; auto llmRequest1 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, @@ -2385,8 +2357,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithCacheSaltTest) // Test Case 3: Request with same tokens and same cache_salt = "tenant-A" requestId = 2; auto llmRequest2 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, @@ -2429,8 +2400,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithCacheSaltTest) requestId = 3; std::string const cacheSalt2{"tenant-B"}; auto llmRequest3 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, @@ -2466,8 +2436,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithCacheSaltTest) // Test Case 5: Request without cache_salt again requestId = 4; auto llmRequest4 = std::make_shared(requestId, maxNewTokens, inputTokens, samplingConfig, isStreaming, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, @@ -8652,8 +8621,7 @@ TEST_F(KVCacheManagerTest, AddSequenceBatchLeavesOneFinalMultimodalContextTokenA auto const makeRequest = [&](LlmRequest::RequestIdType reqId, SizeType32 maxNewTokens) { return std::make_shared(reqId, maxNewTokens, inputTokens, samplingConfig, /*isStreaming=*/false, - /*promptEmbeddingTable=*/std::nullopt, - /*promptVocabSize=*/std::nullopt, multimodalHashes, multimodalPositions, multimodalLengths, + multimodalHashes, multimodalPositions, multimodalLengths, /*multimodalUuids=*/std::nullopt); }; diff --git a/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp b/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp index bb2e9f577e98..e7aa3808e7c4 100644 --- a/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp @@ -23,6 +23,8 @@ #include #include +#include + #include #include #include @@ -36,7 +38,6 @@ namespace tb = tensorrt_llm::batch_manager; using VecTokens = tb::LlmRequest::VecTokens; using SizeType32 = tb::LlmRequest::SizeType32; using VecTokenExtraIds = tb::LlmRequest::VecTokenExtraIds; -using VecUniqueTokens = tb::LlmRequest::VecUniqueTokens; class LlmRequestTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) { @@ -52,8 +53,8 @@ TEST_F(LlmRequestTest, encoderInputLengthValidation) auto const makeRequest = [&inputTokens](std::optional encoderInputLength) { return tb::LlmRequest(/*requestId=*/77, /*maxNewTokens=*/10, inputTokens, texec::SamplingConfig{}, - /*isStreaming=*/false, /*promptEmbeddingTable=*/std::nullopt, /*promptVocabSize=*/std::nullopt, - /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, /*multimodalLengths=*/std::nullopt, + /*isStreaming=*/false, /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, /*excludeInputFromOutput=*/false, /*encoderInputTokens=*/std::nullopt, texec::Request::kDefaultPriority, @@ -128,38 +129,47 @@ TEST_F(LlmRequestTest, fromExecutorRequest) EXPECT_FALSE(llmReq.mSeqSlot); // No speculative decoding config, draft tokens should be empty EXPECT_EQ(llmReq.getNumDraftTokens(), 0); - EXPECT_FALSE(llmReq.getPromptEmbeddingTable().has_value()); - EXPECT_FALSE(llmReq.getPromptVocabSize().has_value()); } - // Prompt tuning + // Unsupported tables must never reach model execution or cache allocation. + for (int const tableValue : {0, 1}) { texec::Request execReq(inputTokens, maxNewTokens); - SizeType32 vocabSize = 100; - SizeType32 hiddenSize = 64; - auto embeddingTable = texec::Tensor::cpu(texec::DataType::kFP32, {vocabSize, hiddenSize}); - VecTokenExtraIds extraIds{1, 1, 1, 0, 0}; - texec::PromptTuningConfig config(embeddingTable, extraIds); - execReq.setPromptTuningConfig(config); - tb::LlmRequest llmReq(requestId, execReq); - - EXPECT_TRUE(llmReq.getPromptEmbeddingTable().has_value()); - EXPECT_TRUE(llmReq.getPromptVocabSize().has_value()); - EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getShape().nbDims, 3); - EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getShape().d[0], 1); - EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getShape().d[1], vocabSize); - EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getShape().d[2], hiddenSize); - EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getDataType(), tensorrt_llm::DataType::kFLOAT); - EXPECT_EQ(llmReq.getPromptVocabSize().value(), vocabSize); - VecUniqueTokens uniqueTokens; - for (size_t i = 0; i < inputTokens.size(); ++i) - { - uniqueTokens.push_back({inputTokens[i], extraIds[i]}); - } - EXPECT_EQ(llmReq.getUniqueTokens(0), uniqueTokens); + auto embeddingTable = texec::Tensor::cpu(texec::DataType::kFP32, {4, 8}); + std::fill_n(static_cast(embeddingTable.getData()), 32, static_cast(tableValue)); + execReq.setPromptTuningConfig( + texec::PromptTuningConfig(embeddingTable, VecTokenExtraIds(inputTokens.size(), 1))); + EXPECT_THROW(tb::LlmRequest(requestId, execReq), tc::TllmException); } } +TEST_F(LlmRequestTest, extraIdsWithoutPromptTuning) +{ + auto const inputTokens = std::make_shared(VecTokens{1, 2, 3}); + auto const makeRequest = [&inputTokens](VecTokenExtraIds extraIds) + { + texec::SamplingConfig samplingConfig; + samplingConfig.setNumReturnSequences(2); + return tb::LlmRequest(77, 10, inputTokens, samplingConfig, /*isStreaming=*/false, + /*multimodalHashes=*/std::nullopt, /*multimodalPositions=*/std::nullopt, + /*multimodalLengths=*/std::nullopt, /*multimodalUuids=*/std::nullopt, + /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, /*loraConfig=*/std::nullopt, + /*kvCacheRetentionConfig=*/std::nullopt, /*draftTokens=*/std::nullopt, + /*excludeInputFromOutput=*/false, /*encoderInputTokens=*/std::nullopt, texec::Request::kDefaultPriority, + /*encoderInputLength=*/std::nullopt, + /*encoderOutputLength=*/std::nullopt, tb::LlmRequestType::LLMREQUEST_TYPE_CONTEXT_AND_GENERATION, + std::make_shared(std::move(extraIds))); + }; + + EXPECT_THROW(makeRequest(VecTokenExtraIds{1, 1}), tc::TllmException); + auto request = makeRequest(VecTokenExtraIds{1, 1, 0}); + auto const other = makeRequest(VecTokenExtraIds{2, 2, 0}); + EXPECT_NE(request.getUniqueTokens(0), other.getUniqueTokens(0)); + auto const child = request.createChildRequest(78); + EXPECT_EQ(child->getUniqueTokens(0), request.getUniqueTokens(0)); + EXPECT_EQ(child->getOrigPromptLen(), inputTokens->size()); +} + TEST_F(LlmRequestTest, invalidExecRequest) { VecTokens inputTokens{1, 2, 3, 4, 5}; @@ -204,35 +214,6 @@ TEST_F(LlmRequestTest, invalidExecRequest) lambdaErrMsgs.emplace_back(lambda, "Expected prompt embedding table to have shape"); } - // Invalid extra id vector's size - { - auto lambda = [&inputTokens, maxNewTokens, requestId]() - { - texec::Request execReq(inputTokens, maxNewTokens); - auto embeddingTable = texec::Tensor::cpu(texec::DataType::kFP32, {4, 8}); - VecTokenExtraIds extraIds(inputTokens.size() - 1, 0); - texec::PromptTuningConfig config(embeddingTable, extraIds); - execReq.setPromptTuningConfig(config); - tb::LlmRequest llmReq(requestId, execReq); - }; - lambdaErrMsgs.emplace_back(lambda, "must be the same as input token vector size"); - } - - // Extra ids not provided when enabling kv cache reuse with prompt table - { - auto lambda = [&inputTokens, maxNewTokens, requestId]() - { - texec::Request execReq(inputTokens, maxNewTokens); - auto embeddingTable = texec::Tensor::cpu(texec::DataType::kFP32, {4, 8}); - texec::PromptTuningConfig config(embeddingTable); - execReq.setPromptTuningConfig(config); - tb::LlmRequest llmReq(requestId, execReq); - - llmReq.validate(500, 1000, 1, true); - }; - lambdaErrMsgs.emplace_back(lambda, "Input token extra ids must be provided"); - } - for (auto& lambdaErrMsg : lambdaErrMsgs) { auto& lambda = lambdaErrMsg.first; @@ -266,18 +247,6 @@ TEST_F(LlmRequestTest, invalidExecRequest) llmReq.validate(10, 60, 2); EXPECT_EQ(llmReq.mMaxNewTokens, 60 - inputTokens.size() - 2); } - { - // Validate extra ids when enabling kv cache reuse with prompt table - texec::Request execReq(inputTokens, maxNewTokens); - auto embeddingTable = texec::Tensor::cpu(texec::DataType::kFP32, {6, 42}); - VecTokenExtraIds extraIds(inputTokens.size(), 1); - texec::PromptTuningConfig config(embeddingTable, extraIds); - execReq.setPromptTuningConfig(config); - tb::LlmRequest llmReq(requestId, execReq); - - EXPECT_EQ(static_cast(llmReq.getOrigPromptLen()), inputTokens.size()); - llmReq.validate(500, 1000, 1, true); - } { using AdditionalModelOutput = texec::AdditionalModelOutput; // Validate additional context and gen outputs diff --git a/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp b/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp index a27178520e1b..98f5e0ba109e 100644 --- a/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp @@ -62,7 +62,6 @@ class MicroBatchSchedulerTest : public ::testing::Test // NOLINT(cppcoreguidelin return std::make_shared(reqId, maxNewTokens, inputTokens, samplingConfig, /*isStreaming=*/false, - /*promptEmbeddingTable=*/std::nullopt, /*promptVocabSize=*/std::nullopt, /*multimodalHashes=*/std::nullopt, /*multimodalPos=*/std::nullopt, /*multimodalLength=*/std::nullopt, /*multimodalUuids=*/std::nullopt, /*loraTaskId=*/std::nullopt, /*loraWeights=*/std::nullopt, diff --git a/tensorrt_llm/_torch/models/modeling_vila.py b/tensorrt_llm/_torch/models/modeling_vila.py index 7bb6c6f1a95d..67ce5d7e0031 100644 --- a/tensorrt_llm/_torch/models/modeling_vila.py +++ b/tensorrt_llm/_torch/models/modeling_vila.py @@ -1164,7 +1164,7 @@ def call_with_text_prompt( (iii) later in LLM, we can slice-and-assign based on condition fused_embed[input_ids < vocab_size] = text_embed, fused_embed[inputI > vocab_size] = mm_embed. - (3) passed input_ids and mm_embed via LlmRequest's prompt_token_ids and prompt_embedding_table fields respectively. LlmRequests can be inflight batched, and the mm_embed is passed to LLM model as `multi_modal_data` which is List[torch.Tensor] for batched requests. + (3) The token IDs travel in the request tokens, and mm_embed travels in the Python multimodal data. Model input preparation passes the batched multimodal embeddings to the LLM. """ text_prompt, mm_data = inputs.get("prompt"), inputs.get( diff --git a/tensorrt_llm/_torch/pyexecutor/llm_request.py b/tensorrt_llm/_torch/pyexecutor/llm_request.py index a4e42209a2de..2a1304d15a3d 100644 --- a/tensorrt_llm/_torch/pyexecutor/llm_request.py +++ b/tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -719,6 +719,7 @@ def __init__( logprobs_simple_format: bool = False, return_routed_experts: bool = False, **kwargs): + self.py_prompt_tuning: bool = False self.py_logits_post_processors = kwargs.pop("py_logits_post_processors", None) # Owned entirely on the Python side: the TorchSampler is the only @@ -1666,10 +1667,6 @@ def executor_request_to_llm_request( embedding_bias=executor_request.embedding_bias, stop_words_list=stop_words_list, position_ids=position_ids, - prompt_embedding_table=None if executor_request.prompt_tuning_config - is None else executor_request.prompt_tuning_config.embedding_table, - prompt_vocab_size=None if executor_request.prompt_tuning_config is None - else executor_request.prompt_tuning_config.embedding_table.shape[0], multimodal_hashes=multimodal_hashes, multimodal_positions=multimodal_positions, multimodal_lengths=multimodal_lengths, @@ -1728,6 +1725,10 @@ def executor_request_to_llm_request( "py_return_routed_experts", False), ) + # Carry only the rejection marker through child construction. Admission + # reports the error per request before model or cache preparation. + llm_request.py_prompt_tuning = executor_request.prompt_tuning_config is not None + # Bad-words list for the TorchSampler path, kept in its native # list[list[int]] form (single- and multi-token words). This is the # TorchSampler's own input and is independent of any other sampler. diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index c70cda57a8be..3ec2063a05c9 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -7119,6 +7119,9 @@ def _configure_csa2_remote_tail(self, request: LlmRequest) -> None: request.py_csa2_remote_tail_split = split def _validate_request(self, request: LlmRequest): + if getattr(request, "py_prompt_tuning", False): + raise ValueError( + "Prompt tuning is not supported by the PyTorch executor.") self._configure_csa2_remote_tail(request) retention = request.kv_cache_retention_config if self._is_kv_manager_v2 and retention is not None: diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index 699fa7393ff2..d40c907f52b7 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -30,7 +30,6 @@ from tensorrt_llm.logger import logger from .._torch.peft.lora.manager import LoraManager -from .._torch.peft.prompt_adapter import PromptAdapterManager from .._torch.pyexecutor.kv_cache_stats import append_kv_cache_iteration_stats from .._torch.pyexecutor.llm_request import LlmResponse from .._utils import (global_mpi_rank, global_mpi_size, mpi_rank, @@ -44,13 +43,12 @@ from ..llmapi.tokenizer import TokenizerBase from ..llmapi.tracer import global_tracer from ..llmapi.utils import _SyncQueue, configure_cpu_affinity, logger_debug -from ..runtime import ModelConfig from ..sampling_params import SamplingParams from .executor import GenerationExecutor from .ipc import IpcQueue from .postproc_worker import (PostprocParams, PostprocWorker, PostprocWorkerConfig) -from .request import GenerationRequest, LoRARequest, PromptAdapterRequest +from .request import GenerationRequest, LoRARequest from .result import (GenerationResult, LogProbsResult, ResponseWrapper, compute_prompt_logprobs, get_metrics_dict) from .utils import (ErrorResponse, RequestError, bucket_responses_by_frontend, @@ -205,8 +203,6 @@ def setup_engine(self): self.max_seq_len = self.engine.max_seq_len self._lora_manager: Optional[LoraManager] = None - self._prompt_adapter_manager: Optional[PromptAdapterManager] = None - self._runtime_model_config: Optional[ModelConfig] = None if self._lora_config is not None: from tensorrt_llm._torch.pyexecutor.resource_manager import \ @@ -342,17 +338,13 @@ def _load_lora_adapter(self, lora_request: LoRARequest) -> None: uids=[adapter_id], ckpt_source=lora_request.ckpt_source) - def _load_prompt_adapter(self, - prompt_adapter_request: PromptAdapterRequest): - self._prompt_adapter_manager.load_from_ckpt( - [prompt_adapter_request.local_path], - model_config=self._runtime_model_config, - uids=[str(prompt_adapter_request.adapter_id)]) - def _enqueue_request(self, request: GenerationRequest, result_wait_queue=None) -> int: assert request.id is not None + if request.prompt_adapter_request is not None: + raise RequestError( + "Prompt adapters are not supported by the PyTorch executor.") py_lora_path = None py_lora_ckpt_source = None if self._lora_manager is not None and request.lora_request is not None: @@ -376,25 +368,14 @@ def _enqueue_request(self, # rode along on the wire (GenerationRequest._prompt_token_ids_i32), hand # THAT to the C++ Request ctor (memcpy) instead of the list -- this avoids # the O(ISL) list copy + element-wise nanobind cast on the GIL-held submit - # thread. No buffer (in-process, or prompt-adapter prepend) -> list path. + # thread. No buffer (in-process) -> list path. # If both forms exist, they are assumed to describe the same token ids; # in-place mutations of request.prompt_token_ids must clear the i32 buffer. i32_buf = getattr(request, "_prompt_token_ids_i32", None) - if i32_buf is not None and request.prompt_adapter_request is None: + if i32_buf is not None: prompt_token_ids = i32_buf else: prompt_token_ids = list(request.prompt_token_ids) - prompt_tuning_config = None - if request.prompt_adapter_request is not None: - self._load_prompt_adapter(request.prompt_adapter_request) - uid = str(request.prompt_adapter_request.adapter_id) - prompt_tuning_config = tllm.PromptTuningConfig( - self._prompt_adapter_manager.uid_to_weights[uid]) - vocab_size = self._runtime_model_config.vocab_size - pa_length = prompt_tuning_config.embedding_table.size(0) - prompt_token_ids = list(range( - vocab_size, vocab_size + pa_length)) + prompt_token_ids - # MULTIMODAL # NOTE: Since, we only support PyTorch backend for multimodal, we will send multimodal_data through the 'py_multimodal_data' field # except `multimodal_input` as it needs to go through the C++ runtime. @@ -493,7 +474,6 @@ def _deduce_max_tokens(request: GenerationRequest) -> int: request.sampling_params._get_stop_words(), embedding_bias=request.sampling_params.embedding_bias, lora_config=lora_config, - prompt_tuning_config=prompt_tuning_config, multimodal_input=multimodal_input, mrope_config=None, kv_cache_retention_config=request.kv_cache_retention_config, diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 486cade00086..1dc7697a76c3 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -735,7 +735,7 @@ def generate( lora_request (tensorrt_llm.executor.request.LoRARequest, Sequence[tensorrt_llm.executor.request.LoRARequest], optional): LoRA request to use for generation, if any. Defaults to None. prompt_adapter_request (tensorrt_llm.executor.request.PromptAdapterRequest, Sequence[tensorrt_llm.executor.request.PromptAdapterRequest], optional): - Prompt Adapter request to use for generation, if any. Defaults to None. + Reserved for compatibility. Non-None prompt adapters are unsupported by the PyTorch executor. Defaults to None. kv_cache_retention_config (tensorrt_llm.bindings.executor.KvCacheRetentionConfig, Sequence[tensorrt_llm.bindings.executor.KvCacheRetentionConfig], optional): Configuration for the request's retention in the KV Cache. Defaults to None. disaggregated_params (tensorrt_llm.disaggregated_params.DisaggregatedParams, Sequence[tensorrt_llm.disaggregated_params.DisaggregatedParams], optional): @@ -831,7 +831,7 @@ def generate_async( sampling_params (tensorrt_llm.sampling_params.SamplingParams, optional): The sampling params for the generation. Defaults to None. A default one will be used if not provided. lora_request (tensorrt_llm.executor.request.LoRARequest, optional): LoRA request to use for generation, if any. Defaults to None. - prompt_adapter_request (tensorrt_llm.executor.request.PromptAdapterRequest, optional): Prompt Adapter request to use for generation, if any. Defaults to None. + prompt_adapter_request (tensorrt_llm.executor.request.PromptAdapterRequest, optional): Reserved for compatibility. Non-None prompt adapters are unsupported by the PyTorch executor. Defaults to None. streaming (bool): Whether to use the streaming mode for the generation. Defaults to False. kv_cache_retention_config (tensorrt_llm.bindings.executor.KvCacheRetentionConfig, optional): Configuration for the request's retention in the KV Cache. Defaults to None. disaggregated_params (tensorrt_llm.disaggregated_params.DisaggregatedParams, optional): Disaggregated parameters. Defaults to None. diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor.py b/tests/unittest/_torch/pyexecutor/test_py_executor.py index 1109fbc43163..f4e9e6b5f365 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor.py @@ -61,6 +61,7 @@ ScheduledRequests, SerializableSchedulerOutput, ) +from tensorrt_llm.bindings import executor as trtllm from tensorrt_llm.bindings.executor import RequestType from tensorrt_llm.bindings.internal.batch_manager import LlmRequestType from tensorrt_llm.llmapi import DisaggScheduleStyle @@ -4623,3 +4624,51 @@ def test_first_token_response_carries_the_prefill_logits(monkeypatch, overlap): assert py_result.generation_logits is None py_result.append_generation_logits(first_logits + 10) # the next decode step lands assert torch.equal(response.result.generation_logits, first_logits.transpose(0, 1)) + + +@pytest.mark.parametrize("table_value", [0.0, 1.0]) +@pytest.mark.parametrize("extra_ids", [None, [1, 1, 0]]) +@pytest.mark.parametrize("v2", [False, True]) +@pytest.mark.parametrize( + "request_type", + [ + RequestType.REQUEST_TYPE_CONTEXT_AND_GENERATION, + RequestType.REQUEST_TYPE_CONTEXT_ONLY, + RequestType.REQUEST_TYPE_GENERATION_ONLY, + ], +) +def test_prompt_tuning_rejected_at_admission_with_valid_peers( + table_value, request_type, extra_ids, v2 +): + # Equal virtual IDs and extra IDs must never make either table eligible for + # model execution or KV reuse. Exercise the native pickle transport first. + import pickle + + from tensorrt_llm._torch.pyexecutor.llm_request import executor_request_to_llm_request + + source = trtllm.Request( + input_token_ids=[100, 101, 1], + max_tokens=2, + sampling_config=trtllm.SamplingConfig(num_return_sequences=2), + type=request_type, + prompt_tuning_config=trtllm.PromptTuningConfig(torch.full((2, 8), table_value), extra_ids), + ) + restored = pickle.loads(pickle.dumps(source)) + request = executor_request_to_llm_request(42, restored, [43], False) + valid = _make_admission_request(44) + executor = _make_admission_executor(None, [request, *request.child_requests, valid]) + executor._is_kv_manager_v2 = v2 + + assert request.orig_prompt_len == 3 + assert request.child_requests[0].orig_prompt_len == 3 + admitted = PyExecutor._fetch_and_activate_new_requests(executor) + + assert admitted == [valid] + assert executor.active_requests == [valid] + assert executor._handle_errors.call_count == 2 + for call, rejected in zip( + executor._handle_errors.call_args_list, [request, *request.child_requests] + ): + assert "Prompt tuning is not supported" in call.args[0] + assert call.kwargs == {"requests": [rejected], "charge_budget": False} + executor._validate_request_budget.assert_called_once_with(valid) diff --git a/tests/unittest/bindings/test_bindings_ut.py b/tests/unittest/bindings/test_bindings_ut.py index 9328f87cd3c4..f5c2204567c2 100644 --- a/tests/unittest/bindings/test_bindings_ut.py +++ b/tests/unittest/bindings/test_bindings_ut.py @@ -223,8 +223,6 @@ def test_llm_request(): "sampling_config": sampling_config, "input_tokens": [0, 1, 2], "is_streaming": True, - "prompt_embedding_table": torch.tensor((10, 10)), - "prompt_vocab_size": 2, } llm_request = _tb.internal.batch_manager.LlmRequest(**kwargs) @@ -233,9 +231,6 @@ def test_llm_request(): assert llm_request.sampling_config.beam_width == sampling_config.beam_width assert llm_request.streaming assert llm_request.seq_slot is None - assert torch.equal(llm_request.prompt_embedding_table, - kwargs["prompt_embedding_table"]) - assert llm_request.prompt_vocab_size == 2 assert llm_request.get_num_tokens(0) == 3 assert llm_request.max_beam_num_tokens == 3 diff --git a/tests/unittest/executor/test_base_worker.py b/tests/unittest/executor/test_base_worker.py index 7776a39902ca..bf6501cb4e2b 100644 --- a/tests/unittest/executor/test_base_worker.py +++ b/tests/unittest/executor/test_base_worker.py @@ -178,6 +178,7 @@ def raise_load_error(lora_request): request = type( "Request", (), { "id": 1, + "prompt_adapter_request": None, "lora_request": type("LoraRequest", (), {"adapter_id": 999})(), })() @@ -428,3 +429,14 @@ def test_generation_logprobs_need_no_worker_recomputation(): response = SimpleNamespace(client_id=7, result=SimpleNamespace(is_final=False)) assert _get_logprobs(worker, response) is None + + +@pytest.mark.cpu_only +def test_prompt_adapter_rejected_before_loading_or_changing_tokens() -> None: + request = SimpleNamespace(id=1, + prompt_adapter_request=object(), + prompt_token_ids=[10, 11]) + # An unsupported adapter must fail before accessing any worker resources. + with pytest.raises(RequestError, match="Prompt adapters are not supported"): + BaseWorker._enqueue_request(SimpleNamespace(), request) + assert request.prompt_token_ids == [10, 11] From dd1f0c0687c6b85b0acf6619dadf436e2a2f7732 Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:43:57 +0000 Subject: [PATCH 02/11] [None][refactor] retire native result logprob compatibility fields Audit all native Result logprob assignments and the shared response path. LlmRequest::createResult no longer produces these values; remaining writes only restore serialization/pickle state or populate test fixtures. Live sampling writes PyResult through the ordinary and beam-search samplers. No production repository caller supplies native Result logprobs. Remove cumLogProbs/logProbs from executor::Result, binary serialization, size accounting, nanobind properties, and pickle state. Adapt the shared response consumer to read logprobs from Python results while preserving native token/completion handling, prompt probabilities, beam indexing, streaming deltas, and postprocessor behavior. Replace synthetic native logprob fixtures with Python-owned payloads or absent-output expectations. This intentionally changes the native Result source/ABI and binary format, including serialized Responses and wrappers containing native result bytes. All binary producers and consumers must use matching builds. Native Result pickle state shrinks from 11 to 9 entries; old tuples are explicitly rejected. Python sampler and high-level completion logprob contracts remain intact. Validation: - Rebuilt matching RelWithDebInfo SM80 libraries/bindings and installed the editable package using run/run_build_wheel.sh --job_count 68. - 450 selected CPU request/executor/response/worker tests passed. - 58 binding tests passed, including the new pickle-layout rejection check. - 62 native response/serialization/request tests passed, including size accounting, context metadata, and request performance metric round trips. - 10 direct GPU sampler cases passed for ordinary, processed, speculative, and beam sampling logprobs. - Extended response tests exercise native/Python formats, greedy/beam, streaming/non-streaming, prompt logprobs, pickle transport, postprocessing, and existing immutable Python logprob snapshots. Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- cpp/include/tensorrt_llm/executor/executor.h | 6 -- cpp/tensorrt_llm/executor/serialization.cpp | 6 -- .../nanobind/executor/request.cpp | 26 ++++---- .../unit_tests/executor/responseTest.cpp | 8 +-- .../executor/serializeUtilsTest.cpp | 27 +++++--- tensorrt_llm/executor/result.py | 9 ++- .../_torch/pyexecutor/test_request_utils.py | 4 +- .../bindings/test_executor_bindings.py | 20 +++--- tests/unittest/llmapi/test_executor.py | 62 ++++++++++++++----- 9 files changed, 98 insertions(+), 70 deletions(-) diff --git a/cpp/include/tensorrt_llm/executor/executor.h b/cpp/include/tensorrt_llm/executor/executor.h index 6f1c03a1fcd5..8c5903f2d29d 100644 --- a/cpp/include/tensorrt_llm/executor/executor.h +++ b/cpp/include/tensorrt_llm/executor/executor.h @@ -732,12 +732,6 @@ struct Result /// @brief The output tokens for each beam BeamTokens outputTokenIds; - /// @brief The cumulative log probabilities. Size beamSize. - std::optional cumLogProbs; - - /// @brief The log probabilities for each generated token. Size [beamSize, outputLen] - std::optional> logProbs; - /// @brief The reason why the model stopped generating tokens for each beam in this request. Size [beamSize]. /// Currently only supported when beamSize is 1 and when using in-flight batching. std::vector finishReasons; diff --git a/cpp/tensorrt_llm/executor/serialization.cpp b/cpp/tensorrt_llm/executor/serialization.cpp index 0707e7a5d623..77796ee8c7bf 100644 --- a/cpp/tensorrt_llm/executor/serialization.cpp +++ b/cpp/tensorrt_llm/executor/serialization.cpp @@ -935,8 +935,6 @@ Result Serialization::deserializeResult(std::istream& is) Result result{}; result.isFinal = su::deserialize(is); result.outputTokenIds = su::deserialize(is); - result.cumLogProbs = su::deserialize>(is); - result.logProbs = su::deserialize>>(is); result.finishReasons = su::deserialize>(is); result.contextPhaseParams = su::deserialize>(is); result.decodingIter = su::deserialize(is); @@ -951,8 +949,6 @@ void Serialization::serialize(Result const& result, std::ostream& os) { su::serialize(result.isFinal, os); su::serialize(result.outputTokenIds, os); - su::serialize(result.cumLogProbs, os); - su::serialize(result.logProbs, os); su::serialize(result.finishReasons, os); su::serialize(result.contextPhaseParams, os); su::serialize(result.decodingIter, os); @@ -967,8 +963,6 @@ size_t Serialization::serializedSize(Result const& result) size_t totalSize = 0; totalSize += su::serializedSize(result.isFinal); totalSize += su::serializedSize(result.outputTokenIds); - totalSize += su::serializedSize(result.cumLogProbs); - totalSize += su::serializedSize(result.logProbs); totalSize += su::serializedSize(result.finishReasons); totalSize += su::serializedSize(result.contextPhaseParams); totalSize += su::serializedSize(result.decodingIter); diff --git a/cpp/tensorrt_llm/nanobind/executor/request.cpp b/cpp/tensorrt_llm/nanobind/executor/request.cpp index 53437735679b..a11d62bd4a3b 100644 --- a/cpp/tensorrt_llm/nanobind/executor/request.cpp +++ b/cpp/tensorrt_llm/nanobind/executor/request.cpp @@ -853,38 +853,34 @@ void initRequestBindings(nb::module_& m) auto resultSetstate = [](tle::Result& self, nb::tuple const& state) { - if (state.size() != 11) + if (state.size() != 9) { throw std::runtime_error("Invalid Result state!"); } tle::Result result; result.isFinal = nb::cast(state[0]); result.outputTokenIds = nb::cast>(state[1]); - result.cumLogProbs = nb::cast>>(state[2]); - result.logProbs = nb::cast>>>(state[3]); - result.finishReasons = nb::cast>(state[4]); - result.sequenceIndex = nb::cast(state[5]); - result.isSequenceFinal = nb::cast(state[6]); - result.decodingIter = nb::cast(state[7]); - result.avgDecodedTokensPerIter = nb::cast(state[8]); - result.contextPhaseParams = nb::cast>(state[9]); - result.requestPerfMetrics = nb::cast>(state[10]); + result.finishReasons = nb::cast>(state[2]); + result.sequenceIndex = nb::cast(state[3]); + result.isSequenceFinal = nb::cast(state[4]); + result.decodingIter = nb::cast(state[5]); + result.avgDecodedTokensPerIter = nb::cast(state[6]); + result.contextPhaseParams = nb::cast>(state[7]); + result.requestPerfMetrics = nb::cast>(state[8]); new (&self) tle::Result(result); }; auto resultGetstate = [](tle::Result const& self) { - return nb::make_tuple(self.isFinal, self.outputTokenIds, self.cumLogProbs, self.logProbs, self.finishReasons, - self.sequenceIndex, self.isSequenceFinal, self.decodingIter, self.avgDecodedTokensPerIter, - self.contextPhaseParams, self.requestPerfMetrics); + return nb::make_tuple(self.isFinal, self.outputTokenIds, self.finishReasons, self.sequenceIndex, + self.isSequenceFinal, self.decodingIter, self.avgDecodedTokensPerIter, self.contextPhaseParams, + self.requestPerfMetrics); }; nb::class_(m, "Result") .def(nb::init<>()) .def_rw("is_final", &tle::Result::isFinal) .def_rw("output_token_ids", &tle::Result::outputTokenIds) - .def_rw("cum_log_probs", &tle::Result::cumLogProbs, nb::arg("cum_log_probs").none()) - .def_rw("log_probs", &tle::Result::logProbs, nb::arg("log_probs").none()) .def_rw("finish_reasons", &tle::Result::finishReasons) .def_rw("sequence_index", &tle::Result::sequenceIndex) .def_rw("is_sequence_final", &tle::Result::isSequenceFinal) diff --git a/cpp/tests/unit_tests/executor/responseTest.cpp b/cpp/tests/unit_tests/executor/responseTest.cpp index a925333b3c60..05f599859c89 100644 --- a/cpp/tests/unit_tests/executor/responseTest.cpp +++ b/cpp/tests/unit_tests/executor/responseTest.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -58,17 +58,13 @@ TEST(ResponseTest, ctorInvalidErrorMsg) TEST(ResponseTest, responseWithResult) { IdType requestId = 1; - auto cumLogProbs = VecLogProbs{1.}; - auto logProbs = std::vector{{1.}}; auto finishReasons = std::vector{FinishReason::kEND_ID}; - auto result = Result{true, {{1}}, cumLogProbs, logProbs, finishReasons}; + auto result = Result{true, {{1}}, finishReasons}; auto response = Response(requestId, result); EXPECT_EQ(response.getRequestId(), requestId); auto resp_result = response.getResult(); EXPECT_EQ(resp_result.isFinal, result.isFinal); EXPECT_EQ(resp_result.outputTokenIds, result.outputTokenIds); - EXPECT_EQ(resp_result.logProbs, result.logProbs); - EXPECT_EQ(resp_result.cumLogProbs, result.cumLogProbs); EXPECT_EQ(resp_result.finishReasons, result.finishReasons); EXPECT_EQ(response.hasError(), false); diff --git a/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp b/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp index eccf392b3e95..289f39ea105e 100644 --- a/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp +++ b/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp @@ -149,13 +149,22 @@ void compareResult(texec::Result res, texec::Result res2) { EXPECT_EQ(res.isFinal, res2.isFinal); EXPECT_EQ(res.outputTokenIds, res2.outputTokenIds); - EXPECT_EQ(res.cumLogProbs, res2.cumLogProbs); - EXPECT_EQ(res.logProbs, res2.logProbs); EXPECT_EQ(res.finishReasons, res2.finishReasons); EXPECT_EQ(res.decodingIter, res2.decodingIter); EXPECT_EQ(res.avgDecodedTokensPerIter, res2.avgDecodedTokensPerIter); EXPECT_EQ(res.sequenceIndex, res2.sequenceIndex); EXPECT_EQ(res.isSequenceFinal, res2.isSequenceFinal); + EXPECT_EQ(res.contextPhaseParams.has_value(), res2.contextPhaseParams.has_value()); + if (res.contextPhaseParams.has_value()) + { + EXPECT_EQ(res.contextPhaseParams->getFirstGenTokens(), res2.contextPhaseParams->getFirstGenTokens()); + EXPECT_EQ(res.contextPhaseParams->getReqId(), res2.contextPhaseParams->getReqId()); + } + EXPECT_EQ(res.requestPerfMetrics.has_value(), res2.requestPerfMetrics.has_value()); + if (res.requestPerfMetrics.has_value()) + { + compareRequestPerfMetrics(*res.requestPerfMetrics, *res2.requestPerfMetrics); + } } void compareResponse(texec::Response res, texec::Response res2) @@ -413,10 +422,11 @@ TEST(SerializeUtilsTest, Nested) TEST(SerializeUtilsTest, ResultResponse) { - texec::Result res = texec::Result{false, {{1, 2, 3}}, texec::VecLogProbs{1.0, 2.0}, - std::vector{{1.1, 2.2}, {3.3, 4.4}}, - std::vector{texec::FinishReason::kLENGTH}, - texec::ContextPhaseParams({9, 37}, 0, VecTokens{1, 2}), 3, 1.5f, 2, true}; + texec::Result res + = texec::Result{false, {{1, 2, 3}}, std::vector{texec::FinishReason::kLENGTH}, + texec::ContextPhaseParams({9, 37}, 0, VecTokens{1, 2}), 3, 1.5f, 2, true}; + res.requestPerfMetrics = texec::RequestPerfMetrics{}; + res.requestPerfMetrics->lastIter = 17; { testSerializeDeserialize(res); } @@ -442,9 +452,8 @@ TEST(SerializeUtilsTest, VectorResponses) { if (i < 5) { - texec::Result res = texec::Result{false, {{i + 1, i + 2, i + 3}}, texec::VecLogProbs{1.0, 2.0}, - std::vector{{1.1, 2.2}, {3.3, 4.4}}, - std::vector{texec::FinishReason::kEND_ID}}; + texec::Result res = texec::Result{ + false, {{i + 1, i + 2, i + 3}}, std::vector{texec::FinishReason::kEND_ID}}; responsesIn.emplace_back(i, res); } else if (i < 10) diff --git a/tensorrt_llm/executor/result.py b/tensorrt_llm/executor/result.py index 30f0b0353674..43bcee12ff40 100644 --- a/tensorrt_llm/executor/result.py +++ b/tensorrt_llm/executor/result.py @@ -399,14 +399,19 @@ def _handle_sequence(self, output.token_ids.extend(decoder_output_prefix) output.token_ids.extend(output_token_ids[src_idx]) - if (cum_log_probs := response_tensors.cum_log_probs) is not None: + # Sampler logprobs belong to the Python result. Native results carry + # token/lifecycle metadata and can also reach this response handler. + has_sampler_outputs = not isinstance(response_tensors, tllm.Result) + if has_sampler_outputs and (cum_log_probs := + response_tensors.cum_log_probs) is not None: output.cumulative_logprob = cum_log_probs[src_idx] # prompt logprobs handling if logprobs_result and logprobs_result.prompt is not None: output.prompt_logprobs = logprobs_result.prompt # Generation logprobs are computed by the sampler. - if (log_probs := response_tensors.log_probs) is not None: + if has_sampler_outputs and (log_probs := + response_tensors.log_probs) is not None: # handle logprobs directly from response tensors given by sampler if decoder_output_prefix and self.sampling_params.use_beam_search: output.logprobs = [ diff --git a/tests/unittest/_torch/pyexecutor/test_request_utils.py b/tests/unittest/_torch/pyexecutor/test_request_utils.py index bfa853b8ca63..6def18f74a61 100644 --- a/tests/unittest/_torch/pyexecutor/test_request_utils.py +++ b/tests/unittest/_torch/pyexecutor/test_request_utils.py @@ -714,8 +714,8 @@ def test_python_logprob_response_snapshots(streaming, beam_width, simple_format) # all logprobs each iteration rather than emitting greedy streaming deltas. snapshot = pickle.loads(pickle.dumps(first_response)) native_result = first_response.result.get_result() - assert native_result.log_probs is None - assert native_result.cum_log_probs is None + assert not hasattr(native_result, "log_probs") + assert not hasattr(native_result, "cum_log_probs") assert snapshot.result.log_probs == first_probs assert snapshot.result.cum_log_probs == [-0.5] * beam_width request.add_new_tokens([5] * beam_width) diff --git a/tests/unittest/bindings/test_executor_bindings.py b/tests/unittest/bindings/test_executor_bindings.py index a2f3cf9809f0..d32de71c5045 100644 --- a/tests/unittest/bindings/test_executor_bindings.py +++ b/tests/unittest/bindings/test_executor_bindings.py @@ -312,8 +312,6 @@ def test_result(): result = trtllm.Result() result.is_final = True result.output_token_ids = [[1, 2, 3]] - result.cum_log_probs = [1.0, 2.0, 3.0] - result.log_probs = [[1.0, 2.0, 3.0]] result.finish_reasons = [trtllm.FinishReason.LENGTH] result.sequence_index = 1 result.is_sequence_final = True @@ -323,8 +321,6 @@ def test_result(): result.request_perf_metrics.last_iter = 33 assert result.is_final is True assert result.output_token_ids == [[1, 2, 3]] - assert result.cum_log_probs == [1.0, 2.0, 3.0] - assert result.log_probs == [[1.0, 2.0, 3.0]] assert result.finish_reasons == [trtllm.FinishReason.LENGTH] assert result.sequence_index == 1 assert result.is_sequence_final is True @@ -338,8 +334,6 @@ def test_result_pickle(): result = trtllm.Result() result.is_final = True result.output_token_ids = [[1, 2, 3]] - result.cum_log_probs = [1.0, 2.0, 3.0] - result.log_probs = [[1.0, 2.0, 3.0]] result.finish_reasons = [trtllm.FinishReason.LENGTH] result.sequence_index = 1 result.is_sequence_final = True @@ -350,12 +344,13 @@ def test_result_pickle(): 1]), [10, 20, 30], 1, "disagg_info_endpoint_24680") result.request_perf_metrics = trtllm.RequestPerfMetrics() result.request_perf_metrics.last_iter = 33 + assert len(result.__getstate__()) == 9 + assert not hasattr(result, "cum_log_probs") + assert not hasattr(result, "log_probs") result_str = pickle.dumps(result) result_copy = pickle.loads(result_str) assert result.is_final == result_copy.is_final assert result.output_token_ids == result_copy.output_token_ids - assert result.cum_log_probs == result_copy.cum_log_probs - assert result.log_probs == result_copy.log_probs assert result.finish_reasons == result_copy.finish_reasons assert result.sequence_index == result_copy.sequence_index assert result.is_sequence_final == result_copy.is_sequence_final @@ -370,6 +365,15 @@ def test_result_pickle(): assert result.request_perf_metrics.last_iter == result_copy.request_perf_metrics.last_iter +@pytest.mark.cpu_only +def test_result_rejects_retired_pickle_layout() -> None: + state = trtllm.Result().__getstate__() + old_state = (*state[:2], None, None, *state[2:]) + restored = trtllm.Result.__new__(trtllm.Result) + with pytest.raises(RuntimeError, match="Invalid Result state"): + restored.__setstate__(old_state) + + @pytest.mark.cpu_only def test_response(): request_id = 0 diff --git a/tests/unittest/llmapi/test_executor.py b/tests/unittest/llmapi/test_executor.py index 059c7bf27f2d..6dbe016150c0 100644 --- a/tests/unittest/llmapi/test_executor.py +++ b/tests/unittest/llmapi/test_executor.py @@ -3,12 +3,15 @@ import asyncio import datetime +import pickle import tempfile import threading import time from concurrent.futures import ProcessPoolExecutor +from contextlib import closing from pathlib import Path from queue import Empty, Queue +from types import SimpleNamespace from unittest.mock import Mock import pytest @@ -92,8 +95,6 @@ def consumer(queue: FusedIpcQueue): def create_rsp(id, finished: bool = False): result = tllm.Result() result.output_token_ids = [[id]] - result.log_probs = None - result.cum_log_probs = None if finished: result.finish_reasons = [tllm.FinishReason.END_ID] result.is_final = finished @@ -117,10 +118,15 @@ def test_GenerationResultBase(): @pytest.mark.parametrize("pytorch", [False, True]) @pytest.mark.parametrize("beam_search", [False, True]) -def test_result_stream_sequence_alignment(pytorch, beam_search): +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("postproc", [False, True]) +def test_result_stream_sequence_alignment(pytorch: bool, beam_search: bool, + streaming: bool, + postproc: bool) -> None: from tensorrt_llm._torch.pyexecutor.llm_request import (LlmResponse, LlmResult, PyResult) - from tensorrt_llm.executor.result import Logprob + from tensorrt_llm.executor.result import (Logprob, LogProbsResult, + ResponseWrapper) result = GenerationResultBase(id=0, sampling_params=SamplingParams( @@ -131,18 +137,20 @@ def test_result_stream_sequence_alignment(pytorch, beam_search): top_k=10, temperature=1.0, logprobs=1)) - for step in (1, 2): + worker = SimpleNamespace(_records={}, + _record_creator=lambda inp, tokenizer: result, + _tokenizer=None, + _post_processor_hook=None) + prompt_logprobs = [{7: Logprob(-0.75, rank=1)}] + for step in ((1, 2) if streaming else (2, )): for sequence in ([0] if beam_search else [0, 1]): sequences = [0, 1] if beam_search else [sequence] native = tllm.Result() native.output_token_ids = [ [100 + seq * 10 + pos for pos in range(1, step + 1)] - if beam_search else [100 + seq * 10 + step] for seq in sequences + if beam_search or not streaming else [100 + seq * 10 + step] + for seq in sequences ] - probs = [[-0.25] * len(tokens) - for tokens in native.output_token_ids] - native.log_probs = probs - native.cum_log_probs = [-0.25 * step] * len(sequences) native.sequence_index = sequence native.finish_reasons = [ tllm.FinishReason.LENGTH @@ -157,7 +165,7 @@ def test_result_stream_sequence_alignment(pytorch, beam_search): py_result.set_log_probs([[{ token: Logprob(logprob=-0.25, rank=1) } for token in tokens] for tokens in native.output_token_ids], - native.cum_log_probs) + [-0.25 * step] * len(sequences)) wrapped = LlmResult(native, py_result, is_final=native.is_final, @@ -169,19 +177,41 @@ def test_result_stream_sequence_alignment(pytorch, beam_search): response = tllm.Response(request_id=0, result=native, client_id=0) - result._handle_response(response) + # Exercise the same response wrapper and pickle boundary used by IPC + # and postprocessing, keeping prompt logprobs independent of Result. + response = pickle.loads( + pickle.dumps( + ResponseWrapper( + response, + logprobs=LogProbsResult(prompt=prompt_logprobs)))) + if postproc: + with closing(asyncio.new_event_loop()) as loop: + loop.run_until_complete( + PostprocWorker._handle_input( + worker, + PostprocWorker.Input( + rsp=response, + sampling_params=result.sampling_params, + streaming=streaming))) + else: + result._handle_response(response) for seq in sequences: output = result._outputs[seq] assert output.token_ids == [ 100 + seq * 10 + pos for pos in range(1, step + 1) ] - assert output.token_ids_diff == [100 + seq * 10 + step] + assert output.token_ids_diff == ([100 + seq * 10 + + step] if streaming else + output.token_ids) expected_logprobs = ([{ token: Logprob(logprob=-0.25, rank=1) - } for token in output.token_ids] if pytorch else [-0.25] * step) + } for token in output.token_ids] if pytorch else []) assert output.logprobs == expected_logprobs - assert output.logprobs_diff == expected_logprobs[-1:] - assert output.cumulative_logprob == -0.25 * step + assert output.logprobs_diff == ( + expected_logprobs[-1:] if streaming else expected_logprobs) + assert output.prompt_logprobs == prompt_logprobs + assert output.cumulative_logprob == (-0.25 * + step if pytorch else None) assert output.finish_reason == ("length" if step == 2 else None) if pytorch: assert result.time_breakdown_metrics == {"step": step} From f64845ccd23cd4837d546ba18038406727bf8435 Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:58:18 +0000 Subject: [PATCH 03/11] fix: set prompt tuning default in disaggregated request mocks Explicitly disable py_prompt_tuning on the six MagicMock requests used by pipelined-transfer validation tests. Otherwise MagicMock synthesizes a truthy attribute and the early PyExecutor prompt-tuning rejection prevents these tests from reaching their intended checks. Validation: python -m pytest -q tests/unittest/disaggregated/test_chunked_transfer.py (58 passed). Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- .../unittest/disaggregated/test_chunked_transfer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unittest/disaggregated/test_chunked_transfer.py b/tests/unittest/disaggregated/test_chunked_transfer.py index 4db5a7e30ecd..75a5da34dbfb 100644 --- a/tests/unittest/disaggregated/test_chunked_transfer.py +++ b/tests/unittest/disaggregated/test_chunked_transfer.py @@ -642,7 +642,7 @@ def test_pipelined_transfer_requires_gen_first_flow(): executor._validate_token_id_range = MagicMock() executor.sampler.validate_request = MagicMock() - request = MagicMock(kv_cache_retention_config=None) + request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) request.sampling_config = None request.py_beam_width = 1 request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY @@ -668,7 +668,7 @@ def test_pipelined_transfer_requires_chunked_prefill(): attn_runtime_features=SimpleNamespace(chunked_prefill=False) ) - request = MagicMock(kv_cache_retention_config=None) + request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY request.py_disaggregated_params = SimpleNamespace( schedule_style=DisaggScheduleStyle.GENERATION_FIRST @@ -690,7 +690,7 @@ def test_pipelined_transfer_rejects_pipeline_parallelism_for_context_request(): executor.model_engine.attn_runtime_features.chunked_prefill = True executor.dist.pp_size = 2 - request = MagicMock(kv_cache_retention_config=None) + request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) request.py_beam_width = 1 request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY request.py_disaggregated_params = SimpleNamespace( @@ -723,7 +723,7 @@ def test_pipelined_transfer_requires_single_beam_for_context_request(): executor.kv_cache_transceiver.pipeline_transfer_enabled = True executor.model_engine.attn_runtime_features.chunked_prefill = True - request = MagicMock(kv_cache_retention_config=None) + request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) request.py_beam_width = 2 request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY request.is_context_only_request = True @@ -746,7 +746,7 @@ def test_pipelined_transfer_allows_non_disaggregated_request(): executor._validate_token_id_range = MagicMock() executor.sampler.validate_request = MagicMock() - request = MagicMock(kv_cache_retention_config=None) + request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) request.sampling_config = None request.py_beam_width = 1 request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY @@ -771,7 +771,7 @@ def test_pipelined_transfer_allows_generation_only_request(): executor._validate_token_id_range = MagicMock() executor.sampler.validate_request = MagicMock() - request = MagicMock(kv_cache_retention_config=None) + request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) request.sampling_config = None request.py_beam_width = 2 request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_GENERATION_ONLY From 5e89f7c265ecf8dab5bdbf221dad4175eec148f1 Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:43:40 +0000 Subject: [PATCH 04/11] refactor: remove prompt tuning from executor requests Remove the unsupported prompt-tuning field from the internal executor Request constructor, accessors, storage, native serialization, and Python pickle layout. Stop carrying the Python rejection marker and remove both executor admission guards now that requests cannot configure prompt tuning. Remove obsolete rejection tests and the disaggregated MagicMock defaults. Extend existing binding coverage to reject the retired keyword and verify fields following it still round-trip through pickle. Validation: rebuilt the development wheel; 47 executor binding tests, 337 affected CPU Python tests, and 66 focused C++ request/conversion/block-key tests passed. Repository formatting and lint hooks passed. Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- .../tensorrt_llm/batch_manager/llmRequest.h | 2 - cpp/include/tensorrt_llm/executor/executor.h | 4 -- cpp/tensorrt_llm/executor/request.cpp | 31 ++++------- cpp/tensorrt_llm/executor/requestImpl.h | 28 +++------- cpp/tensorrt_llm/executor/serialization.cpp | 7 ++- .../nanobind/executor/request.cpp | 38 ++++++-------- .../unit_tests/batch_manager/blockKeyTest.cpp | 12 ++--- .../batch_manager/llmRequestTest.cpp | 24 --------- cpp/tests/unit_tests/executor/requestTest.cpp | 52 +------------------ tensorrt_llm/_torch/pyexecutor/llm_request.py | 5 -- tensorrt_llm/_torch/pyexecutor/py_executor.py | 3 -- .../_torch/pyexecutor/test_py_executor.py | 49 ----------------- .../bindings/test_executor_bindings.py | 16 ++++-- .../disaggregated/test_chunked_transfer.py | 12 ++--- 14 files changed, 59 insertions(+), 224 deletions(-) diff --git a/cpp/include/tensorrt_llm/batch_manager/llmRequest.h b/cpp/include/tensorrt_llm/batch_manager/llmRequest.h index dc08360d395a..9e20a094a6e6 100644 --- a/cpp/include/tensorrt_llm/batch_manager/llmRequest.h +++ b/cpp/include/tensorrt_llm/batch_manager/llmRequest.h @@ -263,8 +263,6 @@ class GenericLlmRequest } } - TLLM_CHECK_WITH_INFO( - !req.getPromptTuningConfig().has_value(), "Prompt tuning is not supported by the PyTorch executor."); auto multimodalInput = req.getMultimodalInput(); if (multimodalInput) { diff --git a/cpp/include/tensorrt_llm/executor/executor.h b/cpp/include/tensorrt_llm/executor/executor.h index 8c5903f2d29d..85a2f838783a 100644 --- a/cpp/include/tensorrt_llm/executor/executor.h +++ b/cpp/include/tensorrt_llm/executor/executor.h @@ -610,7 +610,6 @@ class Request /// @param badWords A list of bad words tokens. Each "word" can be composed of multiple tokens /// @param stopWords A list of stop words tokens. Each "word" can be composed of multiple tokens /// @param embeddingBias The embedding bias tensor. Expected shape is [vocab_size] - /// @param pTuningConfig The prompt tuning configuration /// @param multimodalInput The multimodal input {multimodalHashes, multimodalPositions, multimodalLengths, optional /// exact prompt runs} /// @param mRopeConfig The mrope configuration @@ -641,7 +640,6 @@ class Request std::optional> badWords = std::nullopt, std::optional> stopWords = std::nullopt, std::optional embeddingBias = std::nullopt, - std::optional pTuningConfig = std::nullopt, std::optional multimodalInput = std::nullopt, std::optional mRopeConfig = std::nullopt, std::optional loraConfig = std::nullopt, std::optional kvCacheRetentionConfig = std::nullopt, @@ -672,7 +670,6 @@ class Request [[nodiscard]] std::optional> getBadWords() const; [[nodiscard]] std::optional> getStopWords() const; [[nodiscard]] std::optional getEmbeddingBias() const; - [[nodiscard]] std::optional getPromptTuningConfig() const; [[nodiscard]] std::optional getMultimodalInput() const; [[nodiscard]] std::optional getMropeConfig() const; [[nodiscard]] std::optional getLoraConfig() const; @@ -699,7 +696,6 @@ class Request void setBadWords(std::list const& badWords); void setStopWords(std::list const& stopWords); void setEmbeddingBias(Tensor const& embeddingBias); - void setPromptTuningConfig(PromptTuningConfig const& pTuningConfig); void setMultimodalInput(MultimodalInput const& multimodalInput); void setMropeConfig(MropeConfig const& mRopeConfig); void setLoraConfig(LoraConfig const& loraConfig); diff --git a/cpp/tensorrt_llm/executor/request.cpp b/cpp/tensorrt_llm/executor/request.cpp index de550a0a3b87..d572c758ba6c 100644 --- a/cpp/tensorrt_llm/executor/request.cpp +++ b/cpp/tensorrt_llm/executor/request.cpp @@ -28,19 +28,18 @@ Request::Request(VecTokens inputTokenIds, SizeType32 maxTokens, bool streaming, OutputConfig const& outputConfig, std::optional const& endId, std::optional> positionIds, std::optional> badWords, std::optional> stopWords, std::optional embeddingBias, - std::optional pTuningConfig, std::optional multimodalInput, - std::optional mRopeConfig, std::optional loraConfig, - std::optional kvCacheRetentionConfig, std::optional encoderInputTokenIds, - std::optional clientId, bool returnAllGeneratedTokens, float priority, RequestType type, - std::optional contextPhaseParams, std::optional encoderInputFeatures, - std::optional encoderOutputLength, std::optional guidedDecodingParams, - std::optional allottedTimeMs, std::optional disaggRequestId, - std::optional cacheSalt) + std::optional multimodalInput, std::optional mRopeConfig, + std::optional loraConfig, std::optional kvCacheRetentionConfig, + std::optional encoderInputTokenIds, std::optional clientId, bool returnAllGeneratedTokens, + float priority, RequestType type, std::optional contextPhaseParams, + std::optional encoderInputFeatures, std::optional encoderOutputLength, + std::optional guidedDecodingParams, std::optional allottedTimeMs, + std::optional disaggRequestId, std::optional cacheSalt) : mImpl(std::make_unique(std::move(inputTokenIds), maxTokens, streaming, samplingConfig, outputConfig, endId, std::move(positionIds), std::move(badWords), std::move(stopWords), std::move(embeddingBias), - std::move(pTuningConfig), std::move(multimodalInput), std::move(mRopeConfig), std::move(loraConfig), - std::move(kvCacheRetentionConfig), std::move(encoderInputTokenIds), clientId, returnAllGeneratedTokens, - priority, type, std::move(contextPhaseParams), std::move(encoderInputFeatures), encoderOutputLength, + std::move(multimodalInput), std::move(mRopeConfig), std::move(loraConfig), std::move(kvCacheRetentionConfig), + std::move(encoderInputTokenIds), clientId, returnAllGeneratedTokens, priority, type, + std::move(contextPhaseParams), std::move(encoderInputFeatures), encoderOutputLength, std::move(guidedDecodingParams), allottedTimeMs, disaggRequestId, std::move(cacheSalt))) { } @@ -120,11 +119,6 @@ std::optional Request::getEmbeddingBias() const return mImpl->getEmbeddingBias(); } -std::optional Request::getPromptTuningConfig() const -{ - return mImpl->getPromptTuningConfig(); -} - std::optional Request::getMultimodalInput() const { return mImpl->getMultimodalInput(); @@ -245,11 +239,6 @@ void Request::setEmbeddingBias(Tensor const& embeddingBias) mImpl->setEmbeddingBias(embeddingBias); } -void Request::setPromptTuningConfig(PromptTuningConfig const& pTuningConfig) -{ - mImpl->setPromptTuningConfig(pTuningConfig); -} - void Request::setMultimodalInput(MultimodalInput const& multimodalInput) { mImpl->setMultimodalInput(multimodalInput); diff --git a/cpp/tensorrt_llm/executor/requestImpl.h b/cpp/tensorrt_llm/executor/requestImpl.h index 0b5359d024de..f9d3e0963c96 100644 --- a/cpp/tensorrt_llm/executor/requestImpl.h +++ b/cpp/tensorrt_llm/executor/requestImpl.h @@ -51,14 +51,13 @@ class Request::Impl OutputConfig outputConfig, std::optional const& endId, std::optional> positionIds, std::optional> badWords, std::optional> stopWords, std::optional embeddingBias, - std::optional pTuningConfig, std::optional multimodalInput, - std::optional mRopeConfig, std::optional loraConfig, - std::optional kvCacheRetentionConfig, std::optional encoderInputTokenIds, - std::optional clientId, bool returnAllGeneratedTokens, PriorityType priority, RequestType type, - std::optional contextPhaseParams, std::optional encoderInputFeatures, - std::optional encoderOutputLength, std::optional guidedDecodingParams, - std::optional allottedTimeMs, std::optional disaggRequestId, - std::optional cacheSalt = std::nullopt) + std::optional multimodalInput, std::optional mRopeConfig, + std::optional loraConfig, std::optional kvCacheRetentionConfig, + std::optional encoderInputTokenIds, std::optional clientId, bool returnAllGeneratedTokens, + PriorityType priority, RequestType type, std::optional contextPhaseParams, + std::optional encoderInputFeatures, std::optional encoderOutputLength, + std::optional guidedDecodingParams, std::optional allottedTimeMs, + std::optional disaggRequestId, std::optional cacheSalt = std::nullopt) : mInputTokenIds(std::move(inputTokenIds)) , mMaxNewTokens(maxNewTokens) , mStreaming(streaming) @@ -69,7 +68,6 @@ class Request::Impl , mBadWords(std::move(badWords)) , mStopWords(std::move(stopWords)) , mEmbeddingBias(checkEmbeddingBias(std::move(embeddingBias))) - , mPTuningConfig(std::move(pTuningConfig)) , mMultimodalInput(std::move(multimodalInput)) , mMropeConfig(std::move(mRopeConfig)) , mLoraConfig(std::move(loraConfig)) @@ -163,11 +161,6 @@ class Request::Impl return mEmbeddingBias; } - [[nodiscard]] std::optional getPromptTuningConfig() const - { - return mPTuningConfig; - } - [[nodiscard]] std::optional getMultimodalInput() const { return mMultimodalInput; @@ -288,11 +281,6 @@ class Request::Impl mEmbeddingBias = checkEmbeddingBias(embeddingBias); } - void setPromptTuningConfig(PromptTuningConfig const& pTuningConfig) - { - mPTuningConfig = pTuningConfig; - } - void setMultimodalInput(MultimodalInput const& multimodalInput) { mMultimodalInput = multimodalInput; @@ -407,7 +395,6 @@ class Request::Impl lambda(mBadWords); lambda(mStopWords); lambda(mEmbeddingBias); - lambda(mPTuningConfig); lambda(mMultimodalInput); lambda(mMropeConfig); lambda(mLoraConfig); @@ -436,7 +423,6 @@ class Request::Impl std::optional> mBadWords; std::optional> mStopWords; std::optional mEmbeddingBias; - std::optional mPTuningConfig; std::optional mMultimodalInput; std::optional mMropeConfig; std::optional mLoraConfig; diff --git a/cpp/tensorrt_llm/executor/serialization.cpp b/cpp/tensorrt_llm/executor/serialization.cpp index 77796ee8c7bf..7089e40c397d 100644 --- a/cpp/tensorrt_llm/executor/serialization.cpp +++ b/cpp/tensorrt_llm/executor/serialization.cpp @@ -755,7 +755,6 @@ Request Serialization::deserializeRequest(std::istream& is) auto badWords = su::deserialize>>(is); auto stopWords = su::deserialize>>(is); auto embeddingBias = su::deserialize>(is); - auto pTuningConfig = su::deserialize>(is); auto multimodalInput = su::deserialize>(is); auto mRopeConfig = su::deserialize>(is); auto loraConfig = su::deserialize>(is); @@ -778,9 +777,9 @@ Request Serialization::deserializeRequest(std::istream& is) return Request(std::move(inputTokenIds), maxNewTokens, streaming, samplingConfig, outputConfig, endId, std::move(positionIds), std::move(badWords), std::move(stopWords), std::move(embeddingBias), - std::move(pTuningConfig), std::move(multimodalInput), std::move(mRopeConfig), std::move(loraConfig), - std::move(kvCacheRetentionConfig), std::move(encoderInputTokenIds), clientId, returnAllGeneratedTokens, - priority, requestType, std::move(contextPhaseParams), std::move(encoderInputFeatures), encoderOutputLength, + std::move(multimodalInput), std::move(mRopeConfig), std::move(loraConfig), std::move(kvCacheRetentionConfig), + std::move(encoderInputTokenIds), clientId, returnAllGeneratedTokens, priority, requestType, + std::move(contextPhaseParams), std::move(encoderInputFeatures), encoderOutputLength, std::move(guidedDecodingParams), allottedTimeMs, disaggRequestId, std::move(cacheSalt)); } diff --git a/cpp/tensorrt_llm/nanobind/executor/request.cpp b/cpp/tensorrt_llm/nanobind/executor/request.cpp index a11d62bd4a3b..0a207ab55e8c 100644 --- a/cpp/tensorrt_llm/nanobind/executor/request.cpp +++ b/cpp/tensorrt_llm/nanobind/executor/request.cpp @@ -567,8 +567,8 @@ void initRequestBindings(nb::module_& m) reinterpret_cast(inputTokenIds.data()), inputTokenIds.size() * sizeof(VecTokens::value_type)); return nb::make_tuple(std::move(inputTokenIdsBytes), self.getMaxTokens(), self.getStreaming(), self.getSamplingConfig(), self.getOutputConfig(), self.getEndId(), self.getPositionIds(), - self.getBadWords(), self.getStopWords(), self.getEmbeddingBias(), self.getPromptTuningConfig(), - self.getMultimodalInput(), self.getMropeConfig(), self.getLoraConfig(), self.getKvCacheRetentionConfig(), + self.getBadWords(), self.getStopWords(), self.getEmbeddingBias(), self.getMultimodalInput(), + self.getMropeConfig(), self.getLoraConfig(), self.getKvCacheRetentionConfig(), self.getEncoderInputTokenIds(), self.getClientId(), self.getReturnAllGeneratedTokens(), self.getPriority(), self.getRequestType(), self.getContextPhaseParams(), self.getEncoderInputFeatures(), self.getEncoderOutputLength(), self.getGuidedDecodingParams(), self.getDisaggRequestId(), @@ -576,7 +576,7 @@ void initRequestBindings(nb::module_& m) }; auto requestSetstate = [](tle::Request& self, nb::tuple const& state) { - if (state.size() != 26) + if (state.size() != 25) { throw std::runtime_error("Invalid Request state!"); } @@ -598,16 +598,15 @@ void initRequestBindings(nb::module_& m) nb::cast>(state[5]), nb::cast>>(state[6]), nb::cast>>(state[7]), nb::cast>>(state[8]), nb::cast>(state[9]), - nb::cast>(state[10]), - nb::cast>(state[11]), - nb::cast>(state[12]), nb::cast>(state[13]), - nb::cast>(state[14]), - nb::cast>(state[15]), nb::cast>(state[16]), - nb::cast(state[17]), nb::cast(state[18]), nb::cast(state[19]), - nb::cast>(state[20]), - nb::cast>(state[21]), nb::cast>(state[22]), - nb::cast>(state[23]), std::nullopt, - nb::cast>(state[24]), nb::cast>(state[25])); + nb::cast>(state[10]), + nb::cast>(state[11]), nb::cast>(state[12]), + nb::cast>(state[13]), + nb::cast>(state[14]), nb::cast>(state[15]), + nb::cast(state[16]), nb::cast(state[17]), nb::cast(state[18]), + nb::cast>(state[19]), + nb::cast>(state[20]), nb::cast>(state[21]), + nb::cast>(state[22]), std::nullopt, + nb::cast>(state[23]), nb::cast>(state[24])); }; // Convert input_token_ids to VecTokens. Fast path: a 1-D contiguous int32 @@ -641,7 +640,6 @@ void initRequestBindings(nb::module_& m) tle::OutputConfig const& output_config, std::optional const& end_id, std::optional> position_ids, std::optional> bad_words, std::optional> stop_words, std::optional embedding_bias, - std::optional prompt_tuning_config, std::optional multimodal_input, std::optional mrope_config, std::optional lora_config, std::optional kv_cache_retention_config, @@ -655,11 +653,11 @@ void initRequestBindings(nb::module_& m) { new (self) tle::Request(toVecTokens(input_token_ids), max_tokens, streaming, sampling_config, output_config, end_id, std::move(position_ids), std::move(bad_words), std::move(stop_words), - std::move(embedding_bias), std::move(prompt_tuning_config), std::move(multimodal_input), - std::move(mrope_config), std::move(lora_config), std::move(kv_cache_retention_config), - std::move(encoder_input_token_ids), client_id, return_all_generated_tokens, priority, type, - std::move(context_phase_params), std::move(encoder_input_features), encoder_output_length, - std::move(guided_decoding_params), allotted_time_ms, disagg_request_id, std::move(cache_salt)); + std::move(embedding_bias), std::move(multimodal_input), std::move(mrope_config), + std::move(lora_config), std::move(kv_cache_retention_config), std::move(encoder_input_token_ids), + client_id, return_all_generated_tokens, priority, type, std::move(context_phase_params), + std::move(encoder_input_features), encoder_output_length, std::move(guided_decoding_params), + allotted_time_ms, disagg_request_id, std::move(cache_salt)); }, // clang-format off nb::arg("input_token_ids"), @@ -673,7 +671,6 @@ void initRequestBindings(nb::module_& m) nb::arg("bad_words") = nb::none(), nb::arg("stop_words") = nb::none(), nb::arg("embedding_bias") = nb::none(), - nb::arg("prompt_tuning_config") = nb::none(), nb::arg("multimodal_input") = nb::none(), nb::arg("mrope_config") = nb::none(), nb::arg("lora_config") = nb::none(), @@ -715,7 +712,6 @@ void initRequestBindings(nb::module_& m) .def_prop_rw("bad_words", &tle::Request::getBadWords, &tle::Request::setBadWords) .def_prop_rw("stop_words", &tle::Request::getStopWords, &tle::Request::setStopWords) .def_prop_rw("embedding_bias", &tle::Request::getEmbeddingBias, &tle::Request::setEmbeddingBias) - .def_prop_rw("prompt_tuning_config", &tle::Request::getPromptTuningConfig, &tle::Request::setPromptTuningConfig) .def_prop_rw("multimodal_input", &tle::Request::getMultimodalInput, &tle::Request::setMultimodalInput) .def_prop_rw("mrope_config", &tle::Request::getMropeConfig, &tle::Request::setMropeConfig) .def_prop_rw("lora_config", &tle::Request::getLoraConfig, &tle::Request::setLoraConfig) diff --git a/cpp/tests/unit_tests/batch_manager/blockKeyTest.cpp b/cpp/tests/unit_tests/batch_manager/blockKeyTest.cpp index a59d2f336e34..4cff9211b9d7 100644 --- a/cpp/tests/unit_tests/batch_manager/blockKeyTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/blockKeyTest.cpp @@ -190,8 +190,7 @@ TEST_F(BlockKeyTest, GenerateExtraKeysUsesExactMultimodalRuns) Request executorRequest({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, /*maxTokens=*/1, /*streaming=*/false, SamplingConfig(), OutputConfig(), /*endId=*/std::nullopt, /*positionIds=*/std::nullopt, /*badWords=*/std::nullopt, /*stopWords=*/std::nullopt, - /*embeddingBias=*/std::nullopt, - /*pTuningConfig=*/std::nullopt, multimodalInput); + /*embeddingBias=*/std::nullopt, multimodalInput); LlmRequest llmRequest(/*requestId=*/0, executorRequest); auto firstRunKeys = generateBlockHashExtraKeys(llmRequest, /*startTokenIdx=*/0, /*endTokenIdx=*/4); @@ -232,13 +231,11 @@ TEST_F(BlockKeyTest, GenerateExtraKeysFallsBackToLegacyForIncompleteRunMetadata) Request executorRequestA({101, 32000, 32000, 102, 103}, /*maxTokens=*/1, /*streaming=*/false, SamplingConfig(), OutputConfig(), /*endId=*/std::nullopt, /*positionIds=*/std::nullopt, /*badWords=*/std::nullopt, - /*stopWords=*/std::nullopt, /*embeddingBias=*/std::nullopt, - /*pTuningConfig=*/std::nullopt, multimodalInputA); + /*stopWords=*/std::nullopt, /*embeddingBias=*/std::nullopt, multimodalInputA); Request executorRequestB({101, 32000, 32000, 102, 103}, /*maxTokens=*/1, /*streaming=*/false, SamplingConfig(), OutputConfig(), /*endId=*/std::nullopt, /*positionIds=*/std::nullopt, /*badWords=*/std::nullopt, - /*stopWords=*/std::nullopt, /*embeddingBias=*/std::nullopt, - /*pTuningConfig=*/std::nullopt, multimodalInputB); + /*stopWords=*/std::nullopt, /*embeddingBias=*/std::nullopt, multimodalInputB); LlmRequest llmRequestA(/*requestId=*/0, executorRequestA); LlmRequest llmRequestB(/*requestId=*/1, executorRequestB); @@ -270,8 +267,7 @@ TEST_F(BlockKeyTest, GenerateExtraKeysFallsBackToLegacyForInvalidRunRange) Request executorRequest({101, 32000, 32000, 102, 103}, /*maxTokens=*/1, /*streaming=*/false, SamplingConfig(), OutputConfig(), /*endId=*/std::nullopt, /*positionIds=*/std::nullopt, /*badWords=*/std::nullopt, /*stopWords=*/std::nullopt, - /*embeddingBias=*/std::nullopt, - /*pTuningConfig=*/std::nullopt, multimodalInput); + /*embeddingBias=*/std::nullopt, multimodalInput); LlmRequest llmRequest(/*requestId=*/0, executorRequest); auto keys = generateBlockHashExtraKeys(llmRequest, /*startTokenIdx=*/0, /*endTokenIdx=*/4); diff --git a/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp b/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp index e7aa3808e7c4..48f17026414e 100644 --- a/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp @@ -130,17 +130,6 @@ TEST_F(LlmRequestTest, fromExecutorRequest) // No speculative decoding config, draft tokens should be empty EXPECT_EQ(llmReq.getNumDraftTokens(), 0); } - - // Unsupported tables must never reach model execution or cache allocation. - for (int const tableValue : {0, 1}) - { - texec::Request execReq(inputTokens, maxNewTokens); - auto embeddingTable = texec::Tensor::cpu(texec::DataType::kFP32, {4, 8}); - std::fill_n(static_cast(embeddingTable.getData()), 32, static_cast(tableValue)); - execReq.setPromptTuningConfig( - texec::PromptTuningConfig(embeddingTable, VecTokenExtraIds(inputTokens.size(), 1))); - EXPECT_THROW(tb::LlmRequest(requestId, execReq), tc::TllmException); - } } TEST_F(LlmRequestTest, extraIdsWithoutPromptTuning) @@ -201,19 +190,6 @@ TEST_F(LlmRequestTest, invalidExecRequest) lambdaErrMsgs.emplace_back(lambda, "beamWidth > 0"); } - // Invalid ptable shape - { - auto lambda = [&inputTokens, maxNewTokens, requestId]() - { - texec::Request execReq(inputTokens, maxNewTokens); - auto embeddingTable = texec::Tensor::cpu(texec::DataType::kFP32, {17, 32, 69}); - texec::PromptTuningConfig config(embeddingTable); - execReq.setPromptTuningConfig(config); - tb::LlmRequest llmReq(requestId, execReq); - }; - lambdaErrMsgs.emplace_back(lambda, "Expected prompt embedding table to have shape"); - } - for (auto& lambdaErrMsg : lambdaErrMsgs) { auto& lambda = lambdaErrMsg.first; diff --git a/cpp/tests/unit_tests/executor/requestTest.cpp b/cpp/tests/unit_tests/executor/requestTest.cpp index f34180032df4..19d8197c56ed 100644 --- a/cpp/tests/unit_tests/executor/requestTest.cpp +++ b/cpp/tests/unit_tests/executor/requestTest.cpp @@ -56,19 +56,6 @@ TEST(RequestTest, validInputs) EXPECT_EQ(request.getEmbeddingBias().value().getShape()[0], 20); } - { - auto request = Request({1, 1, 2}, 1); - SizeType32 vocabSize = 100; - SizeType32 hiddenSize = 64; - auto embeddingTable = Tensor::cpu(DataType::kFP32, {vocabSize, hiddenSize}); - PromptTuningConfig config(embeddingTable); - request.setPromptTuningConfig(config); - - EXPECT_TRUE(request.getPromptTuningConfig()); - EXPECT_EQ(request.getPromptTuningConfig().value().getEmbeddingTable().getShape().size(), 2); - EXPECT_EQ(request.getPromptTuningConfig().value().getEmbeddingTable().getShape()[0], vocabSize); - EXPECT_EQ(request.getPromptTuningConfig().value().getEmbeddingTable().getShape()[1], hiddenSize); - } { auto request = Request({1, 1, 2}, 1); IdType clientId = 1234; @@ -107,21 +94,6 @@ TEST(RequestTest, invalidInputs) lambdaErrMsgs.emplace_back(std::make_pair(lambda, ".size() == 1")); } - // Embedding table has wrong shape - { - auto lambda = []() - { - SizeType32 vocabSize = 100; - SizeType32 hiddenSize = 64; - auto embeddingTable = Tensor::cpu(DataType::kFP32, {vocabSize, hiddenSize, 64}); - PromptTuningConfig config(embeddingTable); - auto request = Request({1, 1, 2}, 1); - request.setPromptTuningConfig(config); - }; - lambdaErrMsgs.emplace_back( - std::make_pair(lambda, "Expected prompt embedding table to have shape [vocabSize, hiddenSize]")); - } - for (auto& lambdaErrMsg : lambdaErrMsgs) { auto& lambda = lambdaErrMsg.first; @@ -144,17 +116,9 @@ TEST(RequestTest, invalidInputs) TEST(RequestTest, serializeDeserialize) { - auto embeddingTable = Tensor::cpu(DataType::kFP32, {2, 2}); - float* data = reinterpret_cast(embeddingTable.getData()); - data[0] = 123; - data[1] = 456; - data[2] = 789; - data[3] = 10; - auto request = Request({1, 2, 3, 4}, 11, true, SamplingConfig(), OutputConfig(), 112, std::make_optional>({0, 1, 2, 3}), std::list{{1, 2, 3}, {2, 3, 4}}, - std::nullopt, std::nullopt, PromptTuningConfig(embeddingTable, VecTokenExtraIds({1, 2, 3, 4})), std::nullopt, - std::nullopt, std::nullopt, + std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, KvCacheRetentionConfig({KvCacheRetentionConfig::TokenRangeRetentionConfig(0, 1, 10)}, 10), std::nullopt, 1234, false, 0.5, RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION, std::nullopt, std::nullopt, std::nullopt, GuidedDecodingParams(GuidedDecodingParams::GuideType::kREGEX, "\\d+")); @@ -185,18 +149,4 @@ TEST(RequestTest, serializeDeserialize) EXPECT_TRUE(request.getGuidedDecodingParams().has_value()); EXPECT_TRUE(newRequest.getGuidedDecodingParams().has_value()); EXPECT_EQ(request.getGuidedDecodingParams(), newRequest.getGuidedDecodingParams()); - - EXPECT_TRUE(request.getPromptTuningConfig().has_value()); - EXPECT_TRUE(newRequest.getPromptTuningConfig().has_value()); - EXPECT_EQ(newRequest.getPromptTuningConfig()->getInputTokenExtraIds(), - request.getPromptTuningConfig()->getInputTokenExtraIds()); - auto newEmbeddingTable = newRequest.getPromptTuningConfig()->getEmbeddingTable(); - EXPECT_EQ(newEmbeddingTable.getShape().size(), embeddingTable.getShape().size()); - EXPECT_EQ(newEmbeddingTable.getDataType(), embeddingTable.getDataType()); - EXPECT_EQ(newEmbeddingTable.getMemoryType(), embeddingTable.getMemoryType()); - float* newData = reinterpret_cast(newEmbeddingTable.getData()); - EXPECT_EQ(data[0], newData[0]); - EXPECT_EQ(data[1], newData[1]); - EXPECT_EQ(data[2], newData[2]); - EXPECT_EQ(data[3], newData[3]); } diff --git a/tensorrt_llm/_torch/pyexecutor/llm_request.py b/tensorrt_llm/_torch/pyexecutor/llm_request.py index 2a1304d15a3d..e818d8a17ebc 100644 --- a/tensorrt_llm/_torch/pyexecutor/llm_request.py +++ b/tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -719,7 +719,6 @@ def __init__( logprobs_simple_format: bool = False, return_routed_experts: bool = False, **kwargs): - self.py_prompt_tuning: bool = False self.py_logits_post_processors = kwargs.pop("py_logits_post_processors", None) # Owned entirely on the Python side: the TorchSampler is the only @@ -1725,10 +1724,6 @@ def executor_request_to_llm_request( "py_return_routed_experts", False), ) - # Carry only the rejection marker through child construction. Admission - # reports the error per request before model or cache preparation. - llm_request.py_prompt_tuning = executor_request.prompt_tuning_config is not None - # Bad-words list for the TorchSampler path, kept in its native # list[list[int]] form (single- and multi-token words). This is the # TorchSampler's own input and is independent of any other sampler. diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 3ec2063a05c9..c70cda57a8be 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -7119,9 +7119,6 @@ def _configure_csa2_remote_tail(self, request: LlmRequest) -> None: request.py_csa2_remote_tail_split = split def _validate_request(self, request: LlmRequest): - if getattr(request, "py_prompt_tuning", False): - raise ValueError( - "Prompt tuning is not supported by the PyTorch executor.") self._configure_csa2_remote_tail(request) retention = request.kv_cache_retention_config if self._is_kv_manager_v2 and retention is not None: diff --git a/tests/unittest/_torch/pyexecutor/test_py_executor.py b/tests/unittest/_torch/pyexecutor/test_py_executor.py index f4e9e6b5f365..1109fbc43163 100644 --- a/tests/unittest/_torch/pyexecutor/test_py_executor.py +++ b/tests/unittest/_torch/pyexecutor/test_py_executor.py @@ -61,7 +61,6 @@ ScheduledRequests, SerializableSchedulerOutput, ) -from tensorrt_llm.bindings import executor as trtllm from tensorrt_llm.bindings.executor import RequestType from tensorrt_llm.bindings.internal.batch_manager import LlmRequestType from tensorrt_llm.llmapi import DisaggScheduleStyle @@ -4624,51 +4623,3 @@ def test_first_token_response_carries_the_prefill_logits(monkeypatch, overlap): assert py_result.generation_logits is None py_result.append_generation_logits(first_logits + 10) # the next decode step lands assert torch.equal(response.result.generation_logits, first_logits.transpose(0, 1)) - - -@pytest.mark.parametrize("table_value", [0.0, 1.0]) -@pytest.mark.parametrize("extra_ids", [None, [1, 1, 0]]) -@pytest.mark.parametrize("v2", [False, True]) -@pytest.mark.parametrize( - "request_type", - [ - RequestType.REQUEST_TYPE_CONTEXT_AND_GENERATION, - RequestType.REQUEST_TYPE_CONTEXT_ONLY, - RequestType.REQUEST_TYPE_GENERATION_ONLY, - ], -) -def test_prompt_tuning_rejected_at_admission_with_valid_peers( - table_value, request_type, extra_ids, v2 -): - # Equal virtual IDs and extra IDs must never make either table eligible for - # model execution or KV reuse. Exercise the native pickle transport first. - import pickle - - from tensorrt_llm._torch.pyexecutor.llm_request import executor_request_to_llm_request - - source = trtllm.Request( - input_token_ids=[100, 101, 1], - max_tokens=2, - sampling_config=trtllm.SamplingConfig(num_return_sequences=2), - type=request_type, - prompt_tuning_config=trtllm.PromptTuningConfig(torch.full((2, 8), table_value), extra_ids), - ) - restored = pickle.loads(pickle.dumps(source)) - request = executor_request_to_llm_request(42, restored, [43], False) - valid = _make_admission_request(44) - executor = _make_admission_executor(None, [request, *request.child_requests, valid]) - executor._is_kv_manager_v2 = v2 - - assert request.orig_prompt_len == 3 - assert request.child_requests[0].orig_prompt_len == 3 - admitted = PyExecutor._fetch_and_activate_new_requests(executor) - - assert admitted == [valid] - assert executor.active_requests == [valid] - assert executor._handle_errors.call_count == 2 - for call, rejected in zip( - executor._handle_errors.call_args_list, [request, *request.child_requests] - ): - assert "Prompt tuning is not supported" in call.args[0] - assert call.kwargs == {"requests": [rejected], "charge_budget": False} - executor._validate_request_budget.assert_called_once_with(valid) diff --git a/tests/unittest/bindings/test_executor_bindings.py b/tests/unittest/bindings/test_executor_bindings.py index d32de71c5045..c3faf28c170a 100644 --- a/tests/unittest/bindings/test_executor_bindings.py +++ b/tests/unittest/bindings/test_executor_bindings.py @@ -288,7 +288,6 @@ def test_request(): "bad_words": [[4, 5, 6]], "stop_words": [[7, 8, 9]], "embedding_bias": torch.ones(1), - "prompt_tuning_config": trtllm.PromptTuningConfig(torch.ones(100, 64)), "lora_config": trtllm.LoraConfig(1), "client_id": 1234, } @@ -302,10 +301,10 @@ def test_request(): assert attr_value == v assert isinstance(request.sampling_config, trtllm.SamplingConfig) assert isinstance(request.output_config, trtllm.OutputConfig) - assert isinstance(request.prompt_tuning_config, trtllm.PromptTuningConfig) - assert (request.prompt_tuning_config.embedding_table == torch.ones( - 100, 64)).all() assert isinstance(request.lora_config, trtllm.LoraConfig) + assert not hasattr(request, "prompt_tuning_config") + with pytest.raises(TypeError, match="incompatible function arguments"): + trtllm.Request(**kwargs, prompt_tuning_config=None) def test_result(): @@ -1036,7 +1035,10 @@ def test_request_pickle(): max_tokens=5, streaming=False, client_id=123, - sampling_config=samplingConfig) + sampling_config=samplingConfig, + cache_salt="request-pickle", + priority=0.75, + disagg_request_id=456) request_copy = pickle.loads(pickle.dumps(request)) assert request.sampling_config.beam_width == request_copy.sampling_config.beam_width @@ -1048,3 +1050,7 @@ def test_request_pickle(): assert request.output_config.return_log_probs == request_copy.output_config.return_log_probs assert request.guided_decoding_params == request_copy.guided_decoding_params assert request.kv_cache_retention_config == request_copy.kv_cache_retention_config + assert request_copy.client_id == 123 + assert request_copy.priority == 0.75 + assert request_copy.disagg_request_id == 456 + assert request_copy.cache_salt == "request-pickle" diff --git a/tests/unittest/disaggregated/test_chunked_transfer.py b/tests/unittest/disaggregated/test_chunked_transfer.py index 75a5da34dbfb..4db5a7e30ecd 100644 --- a/tests/unittest/disaggregated/test_chunked_transfer.py +++ b/tests/unittest/disaggregated/test_chunked_transfer.py @@ -642,7 +642,7 @@ def test_pipelined_transfer_requires_gen_first_flow(): executor._validate_token_id_range = MagicMock() executor.sampler.validate_request = MagicMock() - request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) + request = MagicMock(kv_cache_retention_config=None) request.sampling_config = None request.py_beam_width = 1 request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY @@ -668,7 +668,7 @@ def test_pipelined_transfer_requires_chunked_prefill(): attn_runtime_features=SimpleNamespace(chunked_prefill=False) ) - request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) + request = MagicMock(kv_cache_retention_config=None) request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY request.py_disaggregated_params = SimpleNamespace( schedule_style=DisaggScheduleStyle.GENERATION_FIRST @@ -690,7 +690,7 @@ def test_pipelined_transfer_rejects_pipeline_parallelism_for_context_request(): executor.model_engine.attn_runtime_features.chunked_prefill = True executor.dist.pp_size = 2 - request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) + request = MagicMock(kv_cache_retention_config=None) request.py_beam_width = 1 request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY request.py_disaggregated_params = SimpleNamespace( @@ -723,7 +723,7 @@ def test_pipelined_transfer_requires_single_beam_for_context_request(): executor.kv_cache_transceiver.pipeline_transfer_enabled = True executor.model_engine.attn_runtime_features.chunked_prefill = True - request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) + request = MagicMock(kv_cache_retention_config=None) request.py_beam_width = 2 request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY request.is_context_only_request = True @@ -746,7 +746,7 @@ def test_pipelined_transfer_allows_non_disaggregated_request(): executor._validate_token_id_range = MagicMock() executor.sampler.validate_request = MagicMock() - request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) + request = MagicMock(kv_cache_retention_config=None) request.sampling_config = None request.py_beam_width = 1 request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_CONTEXT_ONLY @@ -771,7 +771,7 @@ def test_pipelined_transfer_allows_generation_only_request(): executor._validate_token_id_range = MagicMock() executor.sampler.validate_request = MagicMock() - request = MagicMock(kv_cache_retention_config=None, py_prompt_tuning=False) + request = MagicMock(kv_cache_retention_config=None) request.sampling_config = None request.py_beam_width = 2 request.llm_request_type = LlmRequestType.LLMREQUEST_TYPE_GENERATION_ONLY From 2c48610c14ce680abb251eeb07259a6f33094d0a Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:52:38 +0000 Subject: [PATCH 05/11] refactor: remove unused PromptTuningConfig Executor requests no longer accept prompt tuning, leaving PromptTuningConfig without execution consumers. Delete the class, implementation, build entry, native serialization helpers, Python binding and pickle support, and the class-only binding test. Validation: development wheel rebuilt and binding stubs regenerated; the rebuilt module no longer exports PromptTuningConfig. All 46 executor binding tests and 61 focused native request/serialization tests passed. No tracked references to the deleted class remain. Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- cpp/include/tensorrt_llm/executor/executor.h | 21 ---------- .../tensorrt_llm/executor/serialization.h | 5 --- cpp/tensorrt_llm/executor/CMakeLists.txt | 1 - .../executor/promptTuningConfig.cpp | 41 ------------------- cpp/tensorrt_llm/executor/serialization.cpp | 22 ---------- cpp/tensorrt_llm/executor/serializeUtils.h | 6 --- .../nanobind/executor/request.cpp | 19 --------- .../bindings/test_executor_bindings.py | 6 --- 8 files changed, 121 deletions(-) delete mode 100644 cpp/tensorrt_llm/executor/promptTuningConfig.cpp diff --git a/cpp/include/tensorrt_llm/executor/executor.h b/cpp/include/tensorrt_llm/executor/executor.h index 85a2f838783a..f502c3466c59 100644 --- a/cpp/include/tensorrt_llm/executor/executor.h +++ b/cpp/include/tensorrt_llm/executor/executor.h @@ -252,27 +252,6 @@ class OutputConfig std::optional> additionalModelOutputs; }; -/// @brief Configuration for prompt tuning -class PromptTuningConfig -{ -public: - explicit PromptTuningConfig( - Tensor embeddingTable, std::optional inputTokenExtraIds = std::nullopt); - - [[nodiscard]] Tensor getEmbeddingTable() const; - - [[nodiscard]] std::optional getInputTokenExtraIds() const; - -private: - friend class Serialization; - /// @brief The prompt embedding table. Expected shape: [task vocab_size, hidden_size]. Data type must match model - /// weights. - Tensor mEmbeddingTable; - - /// @brief The input token extra ids for KV Cache reuse when p-tuning is enabled - std::optional mInputTokenExtraIds; -}; - /// @brief Multimodal input data class class MultimodalInput { diff --git a/cpp/include/tensorrt_llm/executor/serialization.h b/cpp/include/tensorrt_llm/executor/serialization.h index b8cb7ccf3234..0f1d5c4d4c1a 100644 --- a/cpp/include/tensorrt_llm/executor/serialization.h +++ b/cpp/include/tensorrt_llm/executor/serialization.h @@ -66,11 +66,6 @@ class Serialization static void serialize(AdditionalModelOutput const& additionalModelOutput, std::ostream& os); [[nodiscard]] static size_t serializedSize(AdditionalModelOutput const& additionalModelOutput); - // PromptTuningConfig - [[nodiscard]] static PromptTuningConfig deserializePromptTuningConfig(std::istream& is); - static void serialize(PromptTuningConfig const& config, std::ostream& os); - [[nodiscard]] static size_t serializedSize(PromptTuningConfig const& config); - // MultimodalInput [[nodiscard]] static MultimodalInput deserializeMultimodalInput(std::istream& is); static void serialize(MultimodalInput const& multimodalInput, std::ostream& os); diff --git a/cpp/tensorrt_llm/executor/CMakeLists.txt b/cpp/tensorrt_llm/executor/CMakeLists.txt index e040dd4e0111..fd98489627e2 100644 --- a/cpp/tensorrt_llm/executor/CMakeLists.txt +++ b/cpp/tensorrt_llm/executor/CMakeLists.txt @@ -80,7 +80,6 @@ add_library( loraConfig.cpp mropeConfig.cpp multimodalInput.cpp - promptTuningConfig.cpp request.cpp response.cpp serialization.cpp diff --git a/cpp/tensorrt_llm/executor/promptTuningConfig.cpp b/cpp/tensorrt_llm/executor/promptTuningConfig.cpp deleted file mode 100644 index 56e60c2b2ab9..000000000000 --- a/cpp/tensorrt_llm/executor/promptTuningConfig.cpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/executor/types.h" - -namespace tensorrt_llm::executor -{ -PromptTuningConfig::PromptTuningConfig(Tensor embeddingTable, std::optional inputTokenExtraIds) - : mEmbeddingTable(std::move(embeddingTable)) - , mInputTokenExtraIds(std::move(inputTokenExtraIds)) -{ - TLLM_CHECK_WITH_INFO(mEmbeddingTable.getShape().size() == 2, - "Expected prompt embedding table to have shape [vocabSize, hiddenSize]"); -} - -Tensor PromptTuningConfig::getEmbeddingTable() const -{ - return mEmbeddingTable; -} - -std::optional PromptTuningConfig::getInputTokenExtraIds() const -{ - return mInputTokenExtraIds; -} - -} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/serialization.cpp b/cpp/tensorrt_llm/executor/serialization.cpp index 7089e40c397d..ca5f63ea5a3f 100644 --- a/cpp/tensorrt_llm/executor/serialization.cpp +++ b/cpp/tensorrt_llm/executor/serialization.cpp @@ -230,28 +230,6 @@ size_t Serialization::serializedSize(AdditionalModelOutput const& additionalMode return totalSize; } -// PromptTuningConfig -PromptTuningConfig Serialization::deserializePromptTuningConfig(std::istream& is) -{ - auto tensor = su::deserialize(is); - auto inputTokenExtraIds = su::deserialize>(is); - return PromptTuningConfig{std::move(tensor), std::move(inputTokenExtraIds)}; -} - -void Serialization::serialize(PromptTuningConfig const& config, std::ostream& os) -{ - su::serialize(config.mEmbeddingTable, os); - su::serialize(config.mInputTokenExtraIds, os); -} - -size_t Serialization::serializedSize(PromptTuningConfig const& config) -{ - size_t totalSize = 0; - totalSize += su::serializedSize(config.mEmbeddingTable); - totalSize += su::serializedSize(config.mInputTokenExtraIds); - return totalSize; -} - // MultimodalInput MultimodalInput Serialization::deserializeMultimodalInput(std::istream& is) { diff --git a/cpp/tensorrt_llm/executor/serializeUtils.h b/cpp/tensorrt_llm/executor/serializeUtils.h index 0db6fb72c794..6a6854bba239 100644 --- a/cpp/tensorrt_llm/executor/serializeUtils.h +++ b/cpp/tensorrt_llm/executor/serializeUtils.h @@ -132,7 +132,6 @@ static_assert(hasSerializedSize(size_t())); static_assert(hasSerializedSize(size_t())); static_assert(hasSerializedSize(size_t())); static_assert(hasSerializedSize(size_t())); -static_assert(hasSerializedSize(size_t())); static_assert(hasSerializedSize(size_t())); static_assert(hasSerializedSize(size_t())); static_assert(hasSerializedSize(size_t())); @@ -248,7 +247,6 @@ static_assert(hasSerialize(nullptr)); static_assert(hasSerialize(nullptr)); static_assert(hasSerialize(nullptr)); static_assert(hasSerialize(nullptr)); -static_assert(hasSerialize(nullptr)); static_assert(hasSerialize(nullptr)); static_assert(hasSerialize(nullptr)); static_assert(hasSerialize(nullptr)); @@ -423,10 +421,6 @@ T deserialize(std::istream& is) { return Serialization::deserializeAdditionalModelOutput(is); } - else if constexpr (std::is_same_v) - { - return Serialization::deserializePromptTuningConfig(is); - } else if constexpr (std::is_same_v) { return Serialization::deserializeMultimodalInput(is); diff --git a/cpp/tensorrt_llm/nanobind/executor/request.cpp b/cpp/tensorrt_llm/nanobind/executor/request.cpp index 0a207ab55e8c..0cfdcfe615da 100644 --- a/cpp/tensorrt_llm/nanobind/executor/request.cpp +++ b/cpp/tensorrt_llm/nanobind/executor/request.cpp @@ -246,25 +246,6 @@ void initRequestBindings(nb::module_& m) .def("__getstate__", outputConfigGetstate) .def("__setstate__", outputConfigSetstate); - auto promptTuningConfigGetstate = [](tle::PromptTuningConfig const& self) - { return nb::make_tuple(self.getEmbeddingTable(), self.getInputTokenExtraIds()); }; - auto promptTuningConfigSetstate = [](tle::PromptTuningConfig& promptTuningConfig, nb::tuple const& state) - { - if (state.size() != 2) - { - throw std::runtime_error("Invalid PromptTuningConfig state!"); - } - new (&promptTuningConfig) - tle::PromptTuningConfig(nb::cast(state[0]), nb::cast>(state[1])); - }; - nb::class_(m, "PromptTuningConfig") - .def(nb::init>(), nb::arg("embedding_table"), - nb::arg("input_token_extra_ids") = nb::none()) - .def_prop_ro("embedding_table", &tle::PromptTuningConfig::getEmbeddingTable) - .def_prop_ro("input_token_extra_ids", &tle::PromptTuningConfig::getInputTokenExtraIds) - .def("__getstate__", promptTuningConfigGetstate) - .def("__setstate__", promptTuningConfigSetstate); - auto loraConfigGetstate = [](tle::LoraConfig const& self) { return nb::make_tuple(self.getTaskId(), self.getWeights(), self.getConfig()); }; auto loraConfigSetstate = [](tle::LoraConfig& loraConfig, nb::tuple const& state) diff --git a/tests/unittest/bindings/test_executor_bindings.py b/tests/unittest/bindings/test_executor_bindings.py index c3faf28c170a..0ce2d954b6a4 100644 --- a/tests/unittest/bindings/test_executor_bindings.py +++ b/tests/unittest/bindings/test_executor_bindings.py @@ -92,12 +92,6 @@ def test_output_config_pickle(): assert additional_model_output.gather_context == True -def test_prompt_tuning_config(): - embedding_table = torch.ones(100, 64) - config = trtllm.PromptTuningConfig(embedding_table) - assert (config.embedding_table == embedding_table).all() - - @pytest.mark.cpu_only def test_multimodal_input(): multimodal_hashes = [[1, 2, 3], [4, 5, 6]] From 0c24cc6f5345d48befc919b4c33abb6eeb515b57 Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:54:58 +0000 Subject: [PATCH 06/11] deprecate: mark unsupported prompt adapter requests Prompt adapters no longer have a supported execution path. Document prompt_adapter_request as deprecated across LLM, executor, and SMG entry points, and identify PromptAdapterRequest as a deprecated compatibility container. Emit DeprecationWarning from shared GenerationRequest construction for non-None adapters, while leaving omitted or None arguments warning-free and preserving the worker rejection. Keep signatures unchanged and document omitting the argument as the migration path. Validation: 75 CPU tests passed across base worker, generation request, and LLM API stability coverage (20 GPU tests deselected). Repository formatting and lint hooks passed. Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- tensorrt_llm/executor/executor.py | 6 +++++ tensorrt_llm/executor/request.py | 25 ++++++++++++++++++++- tensorrt_llm/grpc/smg/request_manager.py | 2 +- tensorrt_llm/llmapi/llm.py | 4 ++-- tests/unittest/executor/test_base_worker.py | 20 ++++++++++++++--- 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/tensorrt_llm/executor/executor.py b/tensorrt_llm/executor/executor.py index 48ecac324a97..7e8b565683a8 100644 --- a/tensorrt_llm/executor/executor.py +++ b/tensorrt_llm/executor/executor.py @@ -135,6 +135,9 @@ def generate_async( ) -> GenerationResult: """Generate output for the given prompt token ids in the asynchronous mode. Asynchronous generation accepts single prompt only. + + ``prompt_adapter_request`` is deprecated. Prompt adapters are unsupported; + omit this argument or pass None. """ assert isinstance(prompt_token_ids, np.ndarray) or isinstance( prompt_token_ids[0], int) @@ -181,6 +184,9 @@ def generate( ) -> Union[GenerationResult, List[GenerationResult]]: """Generate output for the given prompt token ids in the synchronous mode. Synchronous generation accepts either single prompt or batched prompts. + + ``prompt_adapter_request`` is deprecated. Prompt adapters are unsupported; + omit this argument or pass None. """ unbatched = isinstance(prompt_token_ids[0], int) diff --git a/tensorrt_llm/executor/request.py b/tensorrt_llm/executor/request.py index 65aa5a153349..d9dda2464fd0 100644 --- a/tensorrt_llm/executor/request.py +++ b/tensorrt_llm/executor/request.py @@ -1,4 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os +import warnings from collections.abc import Mapping from dataclasses import dataclass from typing import List, Optional, Union @@ -64,7 +80,7 @@ def ckpt_source(self): @dataclass(slots=True) class PromptAdapterRequest: """ - Request for a Prompt adapter. + Deprecated compatibility container for unsupported prompt adapters. """ prompt_adapter_name: str prompt_adapter_id: int @@ -114,6 +130,13 @@ def __init__( list]] = None, priority: float = DEFAULT_REQUEST_PRIORITY, ): + if prompt_adapter_request is not None: + warnings.warn( + "prompt_adapter_request is deprecated and will be removed in a future release. " + "Prompt adapters are unsupported; omit this argument or pass None.", + DeprecationWarning, + stacklevel=2, + ) if isinstance(prompt_token_ids, list): self.prompt_token_ids = prompt_token_ids elif isinstance(prompt_token_ids, (torch.Tensor, np.ndarray)): diff --git a/tensorrt_llm/grpc/smg/request_manager.py b/tensorrt_llm/grpc/smg/request_manager.py index 40e7ec09f822..460d19804b0b 100644 --- a/tensorrt_llm/grpc/smg/request_manager.py +++ b/tensorrt_llm/grpc/smg/request_manager.py @@ -84,7 +84,7 @@ async def generate( sampling_params: Sampling parameters (with detokenize=False!) streaming: Whether to stream results lora_request: Optional LoRA adapter request - prompt_adapter_request: Optional prompt adapter request + prompt_adapter_request: Deprecated; prompt adapters are unsupported. Omit or pass None. kv_cache_retention_config: KV cache retention config disaggregated_params: Disaggregated inference params multi_modal_data: Multimodal data dict (e.g. {"image": [PIL images]}) diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 1dc7697a76c3..2d9dabd427d9 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -735,7 +735,7 @@ def generate( lora_request (tensorrt_llm.executor.request.LoRARequest, Sequence[tensorrt_llm.executor.request.LoRARequest], optional): LoRA request to use for generation, if any. Defaults to None. prompt_adapter_request (tensorrt_llm.executor.request.PromptAdapterRequest, Sequence[tensorrt_llm.executor.request.PromptAdapterRequest], optional): - Reserved for compatibility. Non-None prompt adapters are unsupported by the PyTorch executor. Defaults to None. + Deprecated and scheduled for removal. Prompt adapters are unsupported by the PyTorch executor; omit this argument or pass None. kv_cache_retention_config (tensorrt_llm.bindings.executor.KvCacheRetentionConfig, Sequence[tensorrt_llm.bindings.executor.KvCacheRetentionConfig], optional): Configuration for the request's retention in the KV Cache. Defaults to None. disaggregated_params (tensorrt_llm.disaggregated_params.DisaggregatedParams, Sequence[tensorrt_llm.disaggregated_params.DisaggregatedParams], optional): @@ -831,7 +831,7 @@ def generate_async( sampling_params (tensorrt_llm.sampling_params.SamplingParams, optional): The sampling params for the generation. Defaults to None. A default one will be used if not provided. lora_request (tensorrt_llm.executor.request.LoRARequest, optional): LoRA request to use for generation, if any. Defaults to None. - prompt_adapter_request (tensorrt_llm.executor.request.PromptAdapterRequest, optional): Reserved for compatibility. Non-None prompt adapters are unsupported by the PyTorch executor. Defaults to None. + prompt_adapter_request (tensorrt_llm.executor.request.PromptAdapterRequest, optional): Deprecated and scheduled for removal. Prompt adapters are unsupported by the PyTorch executor; omit this argument or pass None. streaming (bool): Whether to use the streaming mode for the generation. Defaults to False. kv_cache_retention_config (tensorrt_llm.bindings.executor.KvCacheRetentionConfig, optional): Configuration for the request's retention in the KV Cache. Defaults to None. disaggregated_params (tensorrt_llm.disaggregated_params.DisaggregatedParams, optional): Disaggregated parameters. Defaults to None. diff --git a/tests/unittest/executor/test_base_worker.py b/tests/unittest/executor/test_base_worker.py index bf6501cb4e2b..71ad19753b2b 100644 --- a/tests/unittest/executor/test_base_worker.py +++ b/tests/unittest/executor/test_base_worker.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import time +import warnings from types import SimpleNamespace import pytest @@ -433,9 +434,22 @@ def test_generation_logprobs_need_no_worker_recomputation(): @pytest.mark.cpu_only def test_prompt_adapter_rejected_before_loading_or_changing_tokens() -> None: - request = SimpleNamespace(id=1, - prompt_adapter_request=object(), - prompt_token_ids=[10, 11]) + with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always") + GenerationRequest(prompt_token_ids=[10, 11], + sampling_params=SamplingParams(max_tokens=1)) + GenerationRequest(prompt_token_ids=[10, 11], + sampling_params=SamplingParams(max_tokens=1), + prompt_adapter_request=None) + assert not recorded + + with pytest.warns(DeprecationWarning, + match="prompt_adapter_request is deprecated"): + request = GenerationRequest( + prompt_token_ids=[10, 11], + sampling_params=SamplingParams(max_tokens=1), + prompt_adapter_request=object()) + request.id = 1 # An unsupported adapter must fail before accessing any worker resources. with pytest.raises(RequestError, match="Prompt adapters are not supported"): BaseWorker._enqueue_request(SimpleNamespace(), request) From 2ff46f7d8d0ef3247ff2435f022e7354e09d25b2 Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:57:19 +0000 Subject: [PATCH 07/11] docs: mark prompt adapter parameters deprecated in API reference Record status: deprecated for prompt_adapter_request on both committed LLM.generate and LLM.generate_async reference entries, matching their documented deprecation. Validation: all 38 LLM API stability tests passed. Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- tests/unittest/api_stability/references_committed/llm.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unittest/api_stability/references_committed/llm.yaml b/tests/unittest/api_stability/references_committed/llm.yaml index 3251caeee66c..fd42c8f8cfea 100644 --- a/tests/unittest/api_stability/references_committed/llm.yaml +++ b/tests/unittest/api_stability/references_committed/llm.yaml @@ -110,6 +110,7 @@ methods: NoneType] default: null prompt_adapter_request: + status: deprecated annotation: Union[tensorrt_llm.executor.request.PromptAdapterRequest, Sequence[tensorrt_llm.executor.request.PromptAdapterRequest], NoneType] default: null @@ -130,6 +131,7 @@ methods: annotation: Optional[tensorrt_llm.executor.request.LoRARequest] default: null prompt_adapter_request: + status: deprecated annotation: Optional[tensorrt_llm.executor.request.PromptAdapterRequest] default: null streaming: From a93cfbe158b836632535fa850312bcdd94aa3d53 Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:16:28 +0000 Subject: [PATCH 08/11] fix: roll back result registration on enqueue failure BaseWorker.submit must register its GenerationResult before enqueue so fast responses can find it. Remove that registration whenever enqueue fails, propagating the original exception unchanged and preserving other requests. Keep unsupported prompt-adapter rejection centralized in _enqueue_request, which also covers direct callers such as Ray workers. Extend CPU regressions to verify cleanup for rejected prompt adapters, RequestError, ValueError, and RuntimeError. Check successful registration, visibility during enqueue, original exception identity, and preservation of unrelated results and request-ID mappings. The test module is already listed in l0_cpu.yml. Validation: 41 CPU worker/request tests passed; 20 GPU tests deselected. Repository formatting and lint hooks passed. Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- tensorrt_llm/executor/base_worker.py | 8 +++- tests/unittest/executor/test_base_worker.py | 50 ++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index d40c907f52b7..217d3736cc40 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -556,7 +556,13 @@ def submit(self, request: GenerationRequest) -> GenerationResult: self._results[client_id] = result - request_id = self._enqueue_request(request) + enqueued = False + try: + request_id = self._enqueue_request(request) + enqueued = True + finally: + if not enqueued: + self._results.pop(client_id, None) # request_id returned from backend is necessary for the abort_request method. self._client_id_to_request_id[client_id] = request_id diff --git a/tests/unittest/executor/test_base_worker.py b/tests/unittest/executor/test_base_worker.py index 71ad19753b2b..2acc4020a9d2 100644 --- a/tests/unittest/executor/test_base_worker.py +++ b/tests/unittest/executor/test_base_worker.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 import time import warnings -from types import SimpleNamespace +from types import MethodType, SimpleNamespace +from unittest.mock import MagicMock import pytest import torch @@ -453,4 +454,51 @@ def test_prompt_adapter_rejected_before_loading_or_changing_tokens() -> None: # An unsupported adapter must fail before accessing any worker resources. with pytest.raises(RequestError, match="Prompt adapters are not supported"): BaseWorker._enqueue_request(SimpleNamespace(), request) + + worker = MagicMock(rank=0, + _results={}, + _client_id_to_request_id={}, + _handle_background_error=None) + worker._get_logprob_params.return_value = None + worker._enqueue_request.side_effect = lambda req: BaseWorker._enqueue_request( + worker, req) + with pytest.raises(RequestError, match="Prompt adapters are not supported"): + BaseWorker.submit(worker, request) + assert worker._results == {} + assert worker._client_id_to_request_id == {} assert request.prompt_token_ids == [10, 11] + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("error_type", + [None, RequestError, ValueError, RuntimeError]) +def test_submit_cleans_up_failed_enqueue( + error_type: type[Exception] | None) -> None: + request = GenerationRequest(prompt_token_ids=[10, 11], + sampling_params=SamplingParams(max_tokens=1)) + request.set_id(7) + peer_result = object() + worker = MagicMock(rank=0, + _results={8: peer_result}, + _client_id_to_request_id={8: 80}) + worker._handle_background_error = MethodType(lambda self: None, worker) + worker._get_logprob_params.return_value = None + error = error_type("enqueue failed") if error_type else None + + def enqueue(req: GenerationRequest) -> int: + assert worker._results[req.id]._generation_request is req + if error is not None: + raise error + return 70 + + worker._enqueue_request.side_effect = enqueue + if error is not None: + with pytest.raises(error_type, match="enqueue failed") as exc_info: + BaseWorker.submit(worker, request) + assert exc_info.value is error + assert worker._results == {8: peer_result} + assert worker._client_id_to_request_id == {8: 80} + else: + result = BaseWorker.submit(worker, request) + assert worker._results == {7: result, 8: peer_result} + assert worker._client_id_to_request_id == {7: 70, 8: 80} From da5430e95d904e2e5c863f8fa518ad21c9c97cd2 Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Wed, 28 Oct 2026 07:47:06 +0000 Subject: [PATCH 09/11] fix: drop retired prompt fields from child request construction The upstream child-request refactor introduced a delegating constructor that still forwarded mPromptEmbeddingTable and mPromptVocabSize after this branch removed those members and constructor parameters. Remove the stale arguments so child construction matches the remaining request inputs. Validation: rebuilt matching RelWithDebInfo SM80 libraries and bindings; 632 CPU request/executor/response/API tests, 46 executor binding tests, and 150 focused native request/child/block-key/result/serialization/scheduler/agent tests passed. All modified native test targets, including kvCacheManagerTest, compiled. GPU integration and sampler execution were unavailable in this container. Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- cpp/tensorrt_llm/batch_manager/llmRequest.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/tensorrt_llm/batch_manager/llmRequest.cpp b/cpp/tensorrt_llm/batch_manager/llmRequest.cpp index 5f5a73e15547..e6036a474881 100644 --- a/cpp/tensorrt_llm/batch_manager/llmRequest.cpp +++ b/cpp/tensorrt_llm/batch_manager/llmRequest.cpp @@ -248,13 +248,13 @@ void LlmRequest::validate( LlmRequest::LlmRequest(LlmRequest const& parent, RequestIdType requestId) : LlmRequest(requestId, parent.mMaxNewTokens, parent.mTokens.at(0), parent.mSamplingConfig, parent.mIsStreaming, - parent.mPromptEmbeddingTable, parent.mPromptVocabSize, copyOptionalInput(parent.mMultimodalHashes), - copyOptionalInput(parent.mMultimodalPositions), copyOptionalInput(parent.mMultimodalLengths), - copyOptionalInput(parent.mMultimodalUuids), parent.mLoraTaskId, parent.mLoraWeights, parent.mLoraConfig, - parent.mKvCacheRetentionConfig, parent.mDraftTokens ? *parent.mDraftTokens : VecTokens{}, - parent.mExcludeInputFromOutput, copyOptionalInput(parent.mEncoderTokens), parent.mPriority, - parent.mEncoderInputLength, parent.mEncoderOutputLength, parent.mLlmRequestType, - copyOptionalInput(parent.mInputTokenExtraIds), parent.mReturnPerfMetrics, parent.mContextPhaseParams, + copyOptionalInput(parent.mMultimodalHashes), copyOptionalInput(parent.mMultimodalPositions), + copyOptionalInput(parent.mMultimodalLengths), copyOptionalInput(parent.mMultimodalUuids), parent.mLoraTaskId, + parent.mLoraWeights, parent.mLoraConfig, parent.mKvCacheRetentionConfig, + parent.mDraftTokens ? *parent.mDraftTokens : VecTokens{}, parent.mExcludeInputFromOutput, + copyOptionalInput(parent.mEncoderTokens), parent.mPriority, parent.mEncoderInputLength, + parent.mEncoderOutputLength, parent.mLlmRequestType, copyOptionalInput(parent.mInputTokenExtraIds), + parent.mReturnPerfMetrics, parent.mContextPhaseParams, parent.mReturnPerfMetrics ? std::make_optional(parent.mPerfMetrics.timingMetrics.arrivalTime) : std::nullopt, parent.mAgentHierarchy, copyOptionalInput(parent.mMultimodalItemRunCuOffsets), copyOptionalInput(parent.mMultimodalRunPositions), copyOptionalInput(parent.mMultimodalRunLengths), From 66787639f6a1b17f3134237fa81bfe577e31a75a Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:02:27 +0000 Subject: [PATCH 10/11] fix: reject prompt adapters before fire-and-forget RPC submission RpcExecutorMixin.submit sent unsupported prompt adapters with need_response=False. The worker rejection could not reach the proxy result, leaving generation pending indefinitely. Reject non-None adapters locally with RequestError before allocating a client ID, dispatching RPC, or registering a result. Add CPU regression coverage for immediate rejection, no remote submit, unchanged request ID, and preservation of existing results. A valid-request control verifies the ordinary fire-and-forget path and result registration remain intact. The test module is already covered by l0_cpu.yml. Validation: reproduced the missing rejection before the fix; 100 CPU worker/request/RPC/routing/API tests passed, 20 GPU tests deselected. Repository formatting and lint hooks passed. Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- tensorrt_llm/executor/rpc_proxy_mixin.py | 19 +++++++++- tests/unittest/executor/test_base_worker.py | 39 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/executor/rpc_proxy_mixin.py b/tensorrt_llm/executor/rpc_proxy_mixin.py index b3779b2ecbae..2cb03f5ede5f 100644 --- a/tensorrt_llm/executor/rpc_proxy_mixin.py +++ b/tensorrt_llm/executor/rpc_proxy_mixin.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import asyncio import atexit import os @@ -13,7 +28,7 @@ from .result import GenerationResult from .rpc import RPCClient from .rpc.rpc_common import get_unique_ipc_addr -from .utils import ErrorResponse, is_llm_response +from .utils import ErrorResponse, RequestError, is_llm_response class RpcExecutorMixin: @@ -81,6 +96,8 @@ def _run_main_loop_task(): atexit.register(self.shutdown) def submit(self, request: GenerationRequest) -> GenerationResult: + if request.prompt_adapter_request is not None: + raise RequestError("Prompt adapters are not supported by the PyTorch executor.") request.set_id(self._get_next_client_id()) logprob_params = self._get_logprob_params(request) diff --git a/tests/unittest/executor/test_base_worker.py b/tests/unittest/executor/test_base_worker.py index 2acc4020a9d2..09714f1f63a7 100644 --- a/tests/unittest/executor/test_base_worker.py +++ b/tests/unittest/executor/test_base_worker.py @@ -20,6 +20,7 @@ from tensorrt_llm.executor.base_worker import BaseWorker from tensorrt_llm.executor.request import GenerationRequest, LoRARequest from tensorrt_llm.executor.result import LogProbsResult +from tensorrt_llm.executor.rpc_proxy_mixin import RpcExecutorMixin from tensorrt_llm.executor.utils import RequestError from tensorrt_llm.llmapi.llm_args import TorchLlmArgs from tensorrt_llm.sampling_params import LogprobParams, SamplingParams @@ -502,3 +503,41 @@ def enqueue(req: GenerationRequest) -> int: result = BaseWorker.submit(worker, request) assert worker._results == {7: result, 8: peer_result} assert worker._client_id_to_request_id == {7: 70, 8: 80} + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("unsupported_adapter", [False, True]) +def test_rpc_submit_rejects_prompt_adapters_before_dispatch( + unsupported_adapter: bool) -> None: + if unsupported_adapter: + with pytest.warns(DeprecationWarning, + match="prompt_adapter_request is deprecated"): + request = GenerationRequest( + prompt_token_ids=[10, 11], + sampling_params=SamplingParams(max_tokens=1), + prompt_adapter_request=object(), + ) + else: + request = GenerationRequest( + prompt_token_ids=[10, 11], + sampling_params=SamplingParams(max_tokens=1)) + peer_result = object() + proxy = MagicMock(_results={8: peer_result}, _handle_background_error=None) + proxy._get_next_client_id.return_value = 7 + proxy._get_logprob_params.return_value = None + + if unsupported_adapter: + with pytest.raises(RequestError, + match="Prompt adapters are not supported"): + RpcExecutorMixin.submit(proxy, request) + proxy.rpc_client.submit.assert_not_called() + proxy._get_next_client_id.assert_not_called() + assert request.id is None + assert proxy._results == {8: peer_result} + else: + result = RpcExecutorMixin.submit(proxy, request) + proxy.rpc_client.submit.assert_called_once_with(request) + proxy.rpc_client.submit.return_value.remote.assert_called_once_with( + need_response=False) + assert request.id == 7 + assert proxy._results == {7: result, 8: peer_result} From cbabd8a658a113afbd69629ba4662fd34fee428d Mon Sep 17 00:00:00 2001 From: Robin Kobus <19427718+Funatiq@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:41:06 +0000 Subject: [PATCH 11/11] fix: reject prompt adapters in Ray submit override RayExecutor overrides RpcExecutorMixin.submit and bypassed its prompt-adapter preflight validation. Raise RequestError in the Ray override before allocating a client ID, issuing fire-and-forget RPC, or creating a pending result. Extend the CPU regression to invoke the actual RayExecutor.submit override as well as the RPC mixin. Load the Ray executor with scoped mocks for optional cluster import dependencies so the test runs without Ray installed. Verify rejection does not dispatch or alter IDs/results, and valid requests still submit normally. The existing test module is covered by l0_cpu.yml. Validation: reproduced the missing Ray rejection before the fix; 108 CPU request/worker/RPC/routing/Ray-stub/API tests passed, 20 GPU tests deselected. Repository formatting and lint hooks passed. Signed-off-by: Robin Kobus <19427718+Funatiq@users.noreply.github.com> --- tensorrt_llm/executor/ray/executor.py | 4 ++ tests/unittest/executor/test_base_worker.py | 46 +++++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/executor/ray/executor.py b/tensorrt_llm/executor/ray/executor.py index 844f978d14b0..c9ed7e1e6b90 100644 --- a/tensorrt_llm/executor/ray/executor.py +++ b/tensorrt_llm/executor/ray/executor.py @@ -44,6 +44,7 @@ from ..request import GenerationRequest from ..result import GenerationResult from ..rpc_proxy_mixin import RpcExecutorMixin +from ..utils import RequestError from .gpu_worker import RayGPUWorker, RayWorkerWrapper __all__ = [ @@ -282,6 +283,9 @@ def submit(self, request: "GenerationRequest") -> "GenerationResult": Low-level API to the executor. Return a "future" GenerationResult which can be waited. Forwards the request to the workers through RPC. """ + if request.prompt_adapter_request is not None: + raise RequestError( + "Prompt adapters are not supported by the PyTorch executor.") if request.id is None: request.set_id(self._get_next_client_id()) logprob_params = self._get_logprob_params(request) diff --git a/tests/unittest/executor/test_base_worker.py b/tests/unittest/executor/test_base_worker.py index 09714f1f63a7..e7170c7048c0 100644 --- a/tests/unittest/executor/test_base_worker.py +++ b/tests/unittest/executor/test_base_worker.py @@ -1,8 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import importlib +import importlib.util +import sys import time import warnings -from types import MethodType, SimpleNamespace +from pathlib import Path +from types import MethodType, ModuleType, SimpleNamespace from unittest.mock import MagicMock import pytest @@ -505,10 +509,46 @@ def enqueue(req: GenerationRequest) -> int: assert worker._client_id_to_request_id == {7: 70, 8: 80} +def _load_ray_executor(monkeypatch: pytest.MonkeyPatch) -> type: + """Load the real Ray executor with cluster-only imports stubbed.""" + importlib.import_module("tensorrt_llm.executor.ray.utils") + dependencies = { + name: ModuleType(name) + for name in ( + "ray", + "ray.util", + "ray.util.placement_group", + "ray.util.scheduling_strategies", + "tensorrt_llm.executor.ray.gpu_worker", + ) + } + placement = dependencies["ray.util.placement_group"] + placement.get_current_placement_group = MagicMock() + placement.placement_group = MagicMock() + dependencies[ + "ray.util.scheduling_strategies"].PlacementGroupSchedulingStrategy = MagicMock( + ) + gpu_worker = dependencies["tensorrt_llm.executor.ray.gpu_worker"] + gpu_worker.RayGPUWorker = MagicMock() + gpu_worker.RayWorkerWrapper = MagicMock() + for name, module in dependencies.items(): + monkeypatch.setitem(sys.modules, name, module) + source = Path(base_worker.__file__).parent / "ray" / "executor.py" + spec = importlib.util.spec_from_file_location( + "tensorrt_llm.executor.ray._submit_test", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.RayExecutor + + @pytest.mark.cpu_only +@pytest.mark.parametrize("executor_kind", ["rpc", "ray"]) @pytest.mark.parametrize("unsupported_adapter", [False, True]) def test_rpc_submit_rejects_prompt_adapters_before_dispatch( + monkeypatch: pytest.MonkeyPatch, executor_kind: str, unsupported_adapter: bool) -> None: + executor_class = RpcExecutorMixin if executor_kind == "rpc" else _load_ray_executor( + monkeypatch) if unsupported_adapter: with pytest.warns(DeprecationWarning, match="prompt_adapter_request is deprecated"): @@ -529,13 +569,13 @@ def test_rpc_submit_rejects_prompt_adapters_before_dispatch( if unsupported_adapter: with pytest.raises(RequestError, match="Prompt adapters are not supported"): - RpcExecutorMixin.submit(proxy, request) + executor_class.submit(proxy, request) proxy.rpc_client.submit.assert_not_called() proxy._get_next_client_id.assert_not_called() assert request.id is None assert proxy._results == {8: peer_result} else: - result = RpcExecutorMixin.submit(proxy, request) + result = executor_class.submit(proxy, request) proxy.rpc_client.submit.assert_called_once_with(request) proxy.rpc_client.submit.return_value.remote.assert_called_once_with( need_response=False)