[None][feat] ModelingV2: GLM-5.3-Flash block-FP8 target for sm_10x tp4

.github/CODEOWNERS

@@ -524,10 +524,15 @@
 /tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency @dongfengy @Tabrizian @xwang233
 /tests/unittest/_torch/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency @dongfengy @Tabrizian @xwang233
 
-# GLM-5.3-Flash (glm5_next)
-/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next @zhaoyangwang-nvidia
-/tests/unittest/_torch/modeling_v2/models/glm5_next @zhaoyangwang-nvidia
-/tests/integration/defs/accuracy/modeling_v2/glm5_next/ @zhaoyangwang-nvidia
+# GLM-5.3-Flash family (glm5_next): routing and shared code are shared by its target owners.
+/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next @zhaoyangwang-nvidia @ruocheng-nv
+/tests/unittest/_torch/modeling_v2/models/glm5_next @zhaoyangwang-nvidia @ruocheng-nv
+/tests/integration/defs/accuracy/modeling_v2/glm5_next/ @zhaoyangwang-nvidia @ruocheng-nv
+
+# GLM-5.3-Flash block-FP8 sm_10x tp4 target: source, ops, native code, and tests.
+/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4 @ruocheng-nv
+/tests/unittest/_torch/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4 @ruocheng-nv
+/tests/integration/defs/accuracy/modeling_v2/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4 @ruocheng-nv
 
 # Uncomment only on release branches (e.g. release/0.19):
 # * @NVIDIA/trt-llm-release-branch-approval

tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/__init__.py

@@ -1,3 +1,3 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-"""Glm5NextForConditionalGeneration targets."""
+"""Glm5NextForConditionalGeneration targets (GLM-5.3-Flash) and the routing between them."""

tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/csrc/CMakeLists.txt

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved. SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+add_modeling_v2_family(glm5_next)
+
+add_modeling_v2_target(
+  glm5_next glm_5_3_flash_fp8__sm_10x__tp4
+  ARCHS 100f
+  SOURCES *.cu *.cpp
+  REQUIRES MULTI_DEVICE)

tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4/README.md

@@ -0,0 +1,91 @@
+<!-- SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
+<!-- SPDX-License-Identifier: Apache-2.0 -->
+
+# GLM-5.3-Flash FP8 on 4x B200: deployment
+
+This target (`glm_5_3_flash_fp8__sm_10x__tp4`) serves the GLM-5.3-Flash FP8 checkpoint on one node of four
+B200 (sm_100) GPUs with tensor parallelism 4 and MTP-3 speculative decoding. Routing does not select it on
+sm_103 (B300 / GB300), where it has not run. Compared with a standard
+GLM-5.3-Flash deployment, it needs `modeling_v2: require` in the server config and a larger `--max_num_tokens`; its
+CUDA kernels are in the glm5_next family library the build compiles.
+
+## 1. Build
+
+Build or install TensorRT LLM as usual, for a CUDA architecture list that includes `100` and with multi-device
+support (the default): the build then compiles the target's kernels into `libth_modeling_v2_glm5_next.so`.
+
+## 2. Hardware and checkpoint
+
+- One node, 4x B200, NVLink.
+- The GLM-5.3-Flash FP8 checkpoint (128 x 128 block-FP8 experts and MLA projections).
+
+## 3. Server config
+
+`config.yaml`:
+
+```yaml
+modeling_v2: require          # select this ModelingV2 target; startup fails if the setup does not match it
+speculative_config:
+  decoding_type: MTP
+  max_draft_len: 3            # one-model MTP-3 on the checkpoint's MTP layer
+kv_cache_config:
+  dtype: fp8
+  free_gpu_memory_fraction: 0.5
+  enable_block_reuse: false
+cuda_graph_config:
+  enable_padding: true
+  max_batch_size: 64
+enable_chunked_prefill: true
+enable_attention_dp: false
+disable_mm_encoder: true
+moe_config:
+  backend: AUTO
+```
+
+`free_gpu_memory_fraction` can be raised for more KV-cache capacity.
+
+## 4. Launch
+
+```bash
+trtllm-serve /path/to/GLM-5.3-Flash \
+  --backend pytorch --tp_size 4 --ep_size 1 \
+  --max_batch_size 64 --max_num_tokens 32768 --max_seq_len 131072 \
+  --trust_remote_code --config config.yaml
+```
+
+What differs from a standard deployment:
+
+- `--max_num_tokens 32768`: long prompts prefill in larger chunks. The measurements below use it; the default
+  token budget has not been measured on this target.
+- `--max_seq_len 131072` is the model's context length. Prompts of up to 13,312 tokens have been measured on this
+  target (the operations' certified cells stop there); longer ones are untested.
+- Without the glm5_next family library (a build whose CUDA architectures include no `100`, or without multi-device
+  support) the target is not selected, and `modeling_v2: require` fails at startup.
+- Keep the all-reduce strategy at its default (AUTO, or ONESHOT): the target runs the one-shot all-reduce, whose
+  workspace its fused kernels share, and routing declines other strategies.
+
+## 5. Check
+
+- The server log shows `modeling_v2: ModelingV2Glm53FlashFp8Sm10xTp4 built`.
+
+## 6. Measured
+
+One node of 4x B200, the config and launch above, against the built-in GLM-5.3-Flash model (`modeling_v2: off`)
+with the same settings on the same node. Throughput: 13,312-token prompts, 33 output tokens (ignore_eos,
+temperature 0), the median of two runs of 32 requests at C1 and 64 or 128 at C8 / C32 / C64.
+
+| | built-in | this target |
+| --- | --- | --- |
+| C1 TTFT p50 | 271 ms | 205 ms |
+| C1 output tok/s/user p50 | 397 | 492 |
+| C8 output tok/s | 126 | 175 |
+| C32 output tok/s | 135 | 193 |
+| C64 output tok/s | 135 | 198 |
+| KV-cache memory per rank (`free_gpu_memory_fraction` 0.5) | 45.2 GiB | 44.0 GiB |
+| GSM8K, accuracy harness, no speculation | 93.2% | 92.5% |
+| GSM8K, accuracy harness, MTP-3 | 92.3% | 93.1% |
+| MTP-3 acceptance length (accuracy harness) | 3.382 | 3.380 |
+| GSM8K, 400 questions through the server, MTP-3 | 98.5% | 99.0% |
+
+The accuracy-harness rows are `test_accuracy.py` (5-shot, no chat template, the CI protocol); the last row asks
+the server above zero-shot through its OpenAI chat endpoint (greedy, chat template), a different protocol.

tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""GLM-5.3-Flash block-FP8 on sm_100: tensor-parallel attention and experts over 4 ranks."""
+
+from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops
+
+native_ops = load_target_ops(__name__)  # torch.ops.mv2_glm5_next_glm_5_3_flash_fp8_sm_10x_tp4

tensorrt_llm/_torch/_experimental/modeling_v2/models/glm5_next/glm_5_3_flash_fp8__sm_10x__tp4/checkpoint.py

@@ -0,0 +1,91 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Reads this rank's share of checkpoint tensors and records what was read.
+
+The engine hands ``load_weights`` a mapping of checkpoint keys to tensors or lazy
+safetensors slices. ``Checkpoint.read`` cuts a lazy entry before it is materialized, so a
+rank reads only its rows or columns of a split weight, and records every key it touches:
+after loading, every key must have been read except the ones the target predicts it does
+not load (the vision tower, and the MTP layer without speculative decoding).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Mapping, Optional
+
+import torch
+
+#: An index into a checkpoint tensor: one slice per leading axis.
+Cut = tuple[slice, ...]
+
+#: The side of the checkpoint's square FP8 scale blocks (``weight_scale_inv``).
+BLOCK = 128
+
+
+class Checkpoint:
+    def __init__(self, weights: Mapping[str, Any], device: torch.device) -> None:
+        self.weights = weights
+        self.device = device
+        self.read_keys: set[str] = set()
+
+    def read(
+        self, key: str, cut: Optional[Cut] = None, dtype: Optional[torch.dtype] = None
+    ) -> torch.Tensor:
+        """``key``'s tensor (``cut`` of it, read from the lazy entry), on the device.
+
+        ``dtype`` converts after the read; ``None`` keeps the checkpoint's dtype (FP8
+        payloads and FP32 block scales stay as stored).
+        """
+        if key not in self.weights:
+            raise KeyError(f"checkpoint has no tensor {key!r}")
+        entry = self.weights[key]
+        if cut is not None:
+            tensor = entry[cut]
+        else:
+            tensor = entry if torch.is_tensor(entry) else entry[...]
+        self.read_keys.add(key)
+        tensor = tensor.to(device=self.device)
+        if dtype is not None and tensor.dtype != dtype:
+            tensor = tensor.to(dtype)
+        return tensor.contiguous()
+
+    def check_coverage(self, skipped: Iterable[str]) -> None:
+        """Every key was read or is predicted to be skipped; nothing predicted was read."""
+        skipped = set(skipped)
+        keys = set(self.weights.keys())
+        unread = sorted(keys - self.read_keys - skipped)
+        if unread:
+            raise ValueError(f"{len(unread)} checkpoint tensors were not loaded, e.g. {unread[:5]}")
+        both = sorted(self.read_keys & skipped)
+        if both:
+            raise ValueError(f"tensors predicted to be skipped were loaded: {both[:5]}")
+
+
+def rows(start: int, end: int) -> Cut:
+    """The cut of rows ``[start, end)``."""
+    return (slice(start, end),)
+
+
+def cols(start: int, end: int) -> Cut:
+    """The cut of columns ``[start, end)`` of a 2-D tensor."""
+    return (slice(None), slice(start, end))
+
+
+def share(total: int, tp_size: int, tp_rank: int) -> tuple[int, int]:
+    """This rank's contiguous ``[start, end)`` of ``total`` split evenly over ``tp_size``."""
+    if total % tp_size:
+        raise ValueError(f"{total} does not split evenly over {tp_size} ranks")
+    size = total // tp_size
+    return tp_rank * size, (tp_rank + 1) * size
+
+
+def dequantize_block_fp8(
+    weight: torch.Tensor, scale: torch.Tensor, block: int = BLOCK
+) -> torch.Tensor:
+    """BF16 ``weight`` from a 128 x 128 block-FP8 weight and its FP32 block scales."""
+    n, k = weight.shape
+    assert scale.shape == (-(-n // block), -(-k // block)), (
+        f"block scales {tuple(scale.shape)} for a {n} x {k} weight"
+    )
+    full = scale.float().repeat_interleave(block, 0)[:n].repeat_interleave(block, 1)[:, :k]
+    return (weight.float() * full).to(torch.bfloat16)