From 179f9495f985ed180617815f6dac6c85c5620c91 Mon Sep 17 00:00:00 2001 From: Dongfeng Yu Date: Tue, 3 Nov 2026 17:10:44 -0800 Subject: [PATCH 1/5] [None][perf] Add Rubin C1 GLM megakernel to the staircase Signed-off-by: Dongfeng Yu --- .../models/glm_moe_dsa/csrc/CMakeLists.txt | 35 + .../_megakernel.py | 699 ++ .../csrc/glmMegakernel.cu | 6766 ++++++++++++++ .../csrc/glmMegakernel.h | 30 + .../csrc/glmMegakernelOp.cpp | 155 + .../csrc/glmMegakernelStages.cuh | 8183 +++++++++++++++++ .../fake_ops.py | 16 + .../modeling.py | 12 + .../accuracy/modeling_v2/test_glm_moe_dsa.py | 5 +- 9 files changed, 15900 insertions(+), 1 deletion(-) create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_megakernel.py create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.h create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelStages.cuh diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/csrc/CMakeLists.txt b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/csrc/CMakeLists.txt index 0209f1974205..fb682545f69a 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/csrc/CMakeLists.txt +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/csrc/CMakeLists.txt @@ -26,6 +26,41 @@ add_modeling_v2_target( ARCHS 100f SOURCES *.cu *.cpp) +# Preserve the C1 megakernel specialization and native Rubin code generation. +set(mega_source + "${CMAKE_CURRENT_SOURCE_DIR}/../glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu") +set_property( + SOURCE "${mega_source}" + APPEND + PROPERTY COMPILE_DEFINITIONS + GLM52_FC2_SPARSE_WIDE=1 + GLM52_OWNER_FC1_OVERLAP=1 + GLM52_ROUTER_LG=2 + GLM52_CAND_FILTER_ROLL=0 + GLM52_SEL_BATCH_MMA=1 + GLM52_LEAN=1 + GLM52_FLAGS_MODE=5 + GLM52_IO_MODE=3 + GLM52_STAMPS_MODE=1 + GLM52_SETUP_MODE=1 + GLM52_LISTS_MODE=2 + GLM52_TOPK_UNION=1 + GLM52_TOPK_RSH=1 + GLM52_STATIC_FLAGS=2752ll + GLM52_SMEM_MAIN=200704 +) +set_property( + SOURCE "${mega_source}" + APPEND + PROPERTY COMPILE_OPTIONS --maxrregcount=200 --ftz=false --prec-div=true + --prec-sqrt=true) +if(107 IN_LIST CMAKE_CUDA_ARCHITECTURES_ORIG) + set_property( + SOURCE "${mega_source}" + APPEND + PROPERTY COMPILE_OPTIONS --generate-code=arch=compute_107a,code=sm_107a) +endif() + add_modeling_v2_target( glm_moe_dsa glm_5_2_nvfp4__sm_107__low_latency ARCHS 100f diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_megakernel.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_megakernel.py new file mode 100644 index 000000000000..a4b3b1ebdb67 --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/_megakernel.py @@ -0,0 +1,699 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Persistent TP16 decoder-layer kernel for one request and eight verify tokens. + +The weights and multicast buffers belong to this target instance. Each launch clears the next +of three workspace copies, so every eligible forward replaces the same complete MoE-layer sequence. +""" + +from __future__ import annotations + +import os +import weakref +from typing import TYPE_CHECKING + +import torch + +from tensorrt_llm._torch.distributed.ops import ( + _get_mnnvl_workspace_comm, + _make_mnnvl_mcast_buffer, + _mnnvl_workspace_all_succeeded, +) +from tensorrt_llm._utils import prefer_pinned +from tensorrt_llm.logger import logger +from tensorrt_llm.mapping import Mapping + +from . import native_ops + +if TYPE_CHECKING: + from .modeling import DecodeTarget + +# Parameter block fields, in the order of glm52::Param. +_FIELDS = [ + "Magic", + "KvBase", + "StrideRows", + "LayerRows", + "MaxBlocks", + "Rope", + "WFa", + "WQb", + "WLg", + "WO", + "WR", + "WGu", + "WD", + "GammaKv", + "GammaPost", + "GammaNext", + "RouterBias", + "W13", + "S13", + "W2", + "S2", + "Alpha1", + "Alpha2", + "Arena", + "PartO", + "PartStats", + "Fc2out", + "H1Mc", + "H1Uc", + "H2Mc", + "H2Uc", + "HopBytes", + "Rank", + "Counters", + "State", + "Err", + "Roles", + "RoleSmem", + "RoleIssue", + "MaxRoles", + "TimeoutNs", + "Stamps", + "Flags", + "Copy", + "NumCopies", + "Leader", + "IdxWq", + "IdxKnW", + "IdxKnB", + "IdxRope", + "IdxPool", + "IdxBlockStride", + "IdxTmap", + "IdxBlocks", + "IdxCand", + "TopkOut", + "IdxDebug", + "IdxDebugLd", + "IdxDebug2", + "TkUc", + "TkMc", +] +_P = {name: i for i, name in enumerate(_FIELDS)} + +# Stage codes (glm52::Stage). +_ST = dict( + FusedA=1, + Qb=2, + KvAppend=3, + Attn=4, + Merge=5, + Oproj=7, + Clear=8, + H1=9, + RouterGemv=10, + SharedGu=12, + Moe=13, + Owner=14, + H2=15, + OwnerShared=16, + IdxK=17, + IdxQ=18, + IdxSel=19, + OprojLg=20, +) +_CTAS = 212 +_MAX_ROLES = 16 +_SMEM_MAIN = 200704 +_SMEM_FIXED = 16384 + 3072 + 3072 # reduction, residual and small areas after the unit regions +# The attention's working set sits at the top of shared memory (into the reduction area, unused by the attention), so +# the o_proj weights of the same CTA can land at [0, 128 KiB) while it runs. +_ATTN_OFF = 131072 +_HOP_BYTES = 8 * 16 * 6144 * 2 +_TK_BYTES = 8 * 2048 * 4 # the leader's top-k exchange per launch copy (glm52::kTkBytes) +# The router logits by linearity (glm52::kRouterLg, GLM52_ROUTER_LG; the library reports its mode): the ranks' partial +# logits per launch copy (glm52::kLgBytes: FP32 [16 ranks][16 units][8 tokens][16 experts]), in the multicast workspace +# after the top-k exchange's copies; the logit units' weights (K = 2048 O columns + the rank's 384 residual columns). +_LG_BYTES = 16 * 16 * 8 * 16 * 4 +_LG_UNITS = 16 +_LG_CHUNKS = 38 +_LG_OPROJ_FIRST = ( + 148 # glm52::kLgOprojFirst: logit unit u rides on o_proj unit 148 + u (CTAs 148-163) +) +_ARENA_BYTES = ( + 2174720 + 8 * 6144 * 2 + 3 * 8 * 4 * 512 * 2 + 8 * 6144 * 2 + 8 * 4 * 4 +) # glm52::kArenaBytes (x, O's +# three copies, h', x' sum-of-squares partials) + +_FLAGS = 35520 +_FLAG_EARLY_X = 32768 +_FLAG_NEXT_CHAINED = 32 + +_HOP_SLOT_CLEAR_UNITS = 2 * _HOP_BYTES // 16384 +_HOP_CLEAR_UNITS = (2 * _HOP_BYTES + _TK_BYTES) // 16384 +_CLEAR_UNITS = _HOP_CLEAR_UNITS + (_ARENA_BYTES + 16383) // 16384 +# Shared-memory bytes of a unit's region (weights, or the working set of the dynamic stages). +_UNIT_BYTES = { + _ST["FusedA"]: 96 * 2048, + _ST["Qb"]: 3 * 32 * 2048, + _ST["Attn"]: 128 * 592 + 8 * 592 + 8 * 144 + 2 * 64 * 4 + 8 * 16 * 4, + _ST["Oproj"]: 2 * 32 * 2048, + _ST["RouterGemv"]: 24 * 2048, + _ST["SharedGu"]: 2 * 24 * 2048, + _ST["Moe"]: 3 * 55296 + 32768 + 1024, + _ST["IdxQ"]: 2 * 32 * 2048, + _ST["IdxSel"]: 200704, # glm52::kSelTotal (the row shard's logits past the K rounds' region) + _ST["OprojLg"]: 3 + * 32 + * 2048, # o_proj's two tiles, then the logit tile's O part (its residual part: global) +} +# Stages whose shared memory is written by their own code (their region must be free when they start). +_DYNAMIC = {_ST["Attn"], _ST["Moe"], _ST["IdxSel"]} + + +def pack_bf16(w: torch.Tensor) -> torch.Tensor: + """[M, K] BF16 -> [M/16, K/64, 2048] bytes: per (tile, chunk) pieces p = (row half, element half) of 32 threads x + 16 B; thread (g, c) holds row g's (pieces 0-1) and row g + 8's (2-3) elements [64 q + 16 c, +16).""" + m, k = w.shape + assert w.dtype == torch.bfloat16 and m % 16 == 0 and k % 64 == 0 + v = w.contiguous().view(m // 16, 2, 8, k // 64, 4, 2, 8) # tile, rh, g, q, c, eh, e8 + v = v.permute(0, 3, 1, 5, 2, 4, 6) # tile, q, rh, eh, g, c, e8 + return v.contiguous().view(torch.uint8).reshape(m // 16, k // 64, 2048) + + +def swizzle_64b(t: torch.Tensor) -> torch.Tensor: + """[..., R, 64] bytes -> the SWIZZLE_64B layout of 8-row x 64 B atoms: 16-byte chunk c of row r goes to chunk + c ^ ((r >> 1) & 3) of its row.""" + *lead, rows, width = t.shape + assert width == 64 + v = t.reshape(-1, rows, 4, 16) + r = torch.arange(rows, device=t.device).view(rows, 1) + dst = torch.arange(4, device=t.device).view(1, 4) ^ ((r >> 1) & 3) + out = torch.empty_like(v) + out.scatter_(2, dst.view(1, rows, 4, 1).expand(v.shape[0], rows, 4, 16), v) + return out.view(*lead, rows, 64) + + +def _u8(t: torch.Tensor) -> torch.Tensor: + return t.contiguous().view(torch.uint8) + + +def build_roles(leader=False, smem_main=_SMEM_MAIN, qb_late=False, slice_merge=False, lg=0): + """lg: GLM52_ROUTER_LG of the build (1: the logit units too; 2: and no router GEMV units).""" + lists = [[] for _ in range(_CTAS)] + + def add(cta, stage, unit): + lists[cta].append((_ST[stage], unit)) + + if leader: + # fused-A tiles 0-173 (164-173: the indexer K and head-weight rows); q_b' on CTAs 174-211 and after fused-A + # on 164-173; the indexer Q (128 quarter-head units) after fused-A on CTAs 0-127; then every CTA's IdxSel + # (key pages c, c + 212, ...; all CTAs meet in its top-k). + for c in range(174): + add(c, "FusedA", c) + if not qb_late: + for u in range(48): + add(174 + u if u < 38 else 164 + (u - 38), "Qb", u) + add(163, "KvAppend", 0) + add(162, "IdxK", 0) + for u in range(128): + add(u, "IdxQ", u) + for c in range(_CTAS): + add(c, "IdxSel", c) + if qb_late: + # After IdxSel on the non-selector CTAs (the selectors 204-211 run the top-k's last steps). + for u in range(48): + add(164 + u if u < 40 else 128 + (u - 40), "Qb", u) + else: + for c in range(_CTAS): + if c < 164: + add(c, "FusedA", c) + else: + add(c, "Qb", c - 164) + add(163, "KvAppend", 0) + for u in range(128): + add(u, "Attn", u) + if not slice_merge: + for u in range(32): + add(164 + u, "Merge", u) + for u in range(192): + # The router logits by linearity: the o_proj units on CTAs 148-163 (no attention, shared-expert or q_b' unit) + # also run logit unit u - 148 on the same O (its weights land after fused-A with o_proj's), then the CTA's MoE + # warm-up and the ranks' sum. + lgu = lg and _LG_OPROJ_FIRST <= u < _LG_OPROJ_FIRST + _LG_UNITS + add(u, "OprojLg" if lgu else "Oproj", u) + for u in range(_HOP_SLOT_CLEAR_UNITS, _CLEAR_UNITS): + add(u if u < _HOP_CLEAR_UNITS else u - _HOP_CLEAR_UNITS, "Clear", u) + for u in range(32): + add(180 + u, "H1", u) + if lg < 2: + for u in range(64): + add(u, "RouterGemv", u) + for u in range(32): + add(65 + u, "SharedGu", u) + # On CTAs [100, 148): the kernel's MoE work cut gives these CTAs a smaller share (glm52::kOsFirst, kOsCtas). + for j in range(48): + add(100 + j, "OwnerShared", j) + # MoE unit = 1 + the CTA's shared-expert unit (run inside the MoE stage with kFlagOsInMoe), or 0. + for c in range(_CTAS): + add(c, "Moe", c - 99 if 100 <= c < 148 else 0) + for j in range(48): + add(100 + j, "Owner", j) + for u in range(32): + add(180 + u, "H2", u) + + roles = torch.zeros(_CTAS, _MAX_ROLES, dtype=torch.int32) + smem = torch.zeros(_CTAS, _MAX_ROLES, dtype=torch.int32) + issue = torch.full((_CTAS, _MAX_ROLES), -2, dtype=torch.int32) + for c, units in enumerate(lists): + assert len(units) <= _MAX_ROLES, (c, units) + placed = [] # (offset, size, index) + for i, (stage, unit) in enumerate(units): + roles[c, i] = (stage << 16) | unit + size = _UNIT_BYTES.get(stage, 0) + if size == 0: + continue + if stage in _DYNAMIC: + off = _ATTN_OFF if stage == _ST["Attn"] else 0 + blocking = max( + [j for (o, s, j) in placed if o < off + size and off < o + s], default=-1 + ) + else: + cands = sorted({0, *[((o + s + 1023) // 1024) * 1024 for (o, s, j) in placed]}) + best = None + for off_c in cands: + if off_c + size > smem_main: + continue + blk = max( + [j for (o, s, j) in placed if o < off_c + size and off_c < o + s], + default=-1, + ) + if best is None or blk < best[0]: + best = (blk, off_c) + assert best is not None, (c, stage) + blocking, off = best + issue[c, i] = blocking + smem[c, i] = off + placed.append((off, size, i)) + return roles, smem, issue + + +class _Shared: + """Buffers reused by a complete sequence of decoder-layer launches.""" + + def __init__(self, mapping: Mapping) -> None: + ( + self.magic, + num_params, + arena_bytes, + clear_units, + smem_bytes, + num_counters, + cand_cap, + self.max_keys, + self.router_lg, + lg_bytes, + ) = native_ops.glm_megakernel_info() + assert num_params == len(_FIELDS), "megakernel parameter layout mismatch" + assert arena_bytes == _ARENA_BYTES and clear_units == _CLEAR_UNITS + assert smem_bytes == _SMEM_MAIN + _SMEM_FIXED and lg_bytes == _LG_BYTES + self.copies = 3 + self.tables = { + leader: build_roles(leader, qb_late=True, lg=self.router_lg) for leader in (False, True) + } + device = torch.device("cuda", torch.cuda.current_device()) + + def empty(num_bytes: int) -> torch.Tensor: + return torch.full((num_bytes // 4,), -0.0, dtype=torch.float32, device=device) + + self.arena = empty(self.copies * _ARENA_BYTES) + self.part_o = empty(8 * 16 * 4 * 512 * 4) + self.part_stats = empty(8 * 16 * 4 * 2 * 4) + self.fc2out = empty(64 * 8 * 6144 * 2) + self.counters = torch.zeros(self.copies * num_counters, dtype=torch.int32, device=device) + self.idx_cand = torch.zeros( + self.copies * 8 * cand_cap * 2, dtype=torch.int32, device=device + ) + self.state = torch.zeros(4, dtype=torch.int32, device=device) + self.err = torch.zeros(16 + 5 * _CTAS, dtype=torch.int32, pin_memory=prefer_pinned()) + self.leader_topk: dict[int, torch.Tensor] = {} + self.comm = _get_mnnvl_workspace_comm(mapping) + error = None + try: + total = self.copies * (2 * _HOP_BYTES + _TK_BYTES + _LG_BYTES) + self.handle = _make_mnnvl_mcast_buffer( + self.comm, total, mapping, use_fabric_handle=mapping.is_multi_node() + ) + self.uc = self.handle.get_uc_buffer(mapping.tp_rank, (total // 4,), torch.float32, 0) + self.mc = self.handle.get_mc_buffer((total // 4,), torch.float32, 0) + self.uc.fill_(-0.0) + torch.cuda.synchronize() + except Exception as exc: + # All ranks must reach the agreement even if one allocation failed. + error = exc + if not _mnnvl_workspace_all_succeeded(self.comm, error is None): + raise RuntimeError( + "GLM megakernel workspace construction failed on at least one rank" + ) from error + self.mutable = [ + self.arena, + self.part_o, + self.part_stats, + self.fc2out, + self.counters, + self.state, + self.idx_cand, + self.uc, + self.mc, + self.err, + ] + + +class _Layer: + """Repacked weights and capture-stable native parameters for one MoE layer.""" + + def __init__( + self, + decode: DecodeTarget, + layer_idx: int, + shared: _Shared, + copy: int, + chained: bool, + next_chained: bool, + ) -> None: + self.layer = layer = decode.core.layers[layer_idx] + self.layer_idx = layer_idx + self.mapping = decode.mapping + self.shared = shared + self.copy = copy + self.flags = _FLAGS + if next_chained and self.flags & _FLAG_EARLY_X: + self.flags |= _FLAG_NEXT_CHAINED + if not chained: + self.flags &= ~_FLAG_EARLY_X + self.pdl = os.environ.get("TRTLLM_ENABLE_PDL", "0") == "1" + self.params_host = None + self.params = None + self.step_params = {} + attn = layer.self_attn + self.indexer = attn.mqa.indexer + self.leader = self.indexer is not None + self.leader_idx = max( + ( + j + for j in range(layer_idx) + if decode.core.layers[j].self_attn.mqa.indexer is not None + ), + default=None, + ) + device = attn.o_proj.weight.device + moe = layer.mlp + backend = getattr(moe.experts, "backend", moe.experts) + w_fa = attn.kv_a_proj_with_mqa.weight + w_qb = decode._q_b_absorbed[layer_idx] + w_vb = attn.v_b_proj + w_o = attn.o_proj.weight + assert tuple(w_fa.shape) == (2624, 6144) + assert tuple(w_qb.shape) == (2304, 2048) + assert tuple(w_vb.shape) == (4, 256, 512) + assert tuple(w_o.shape) == (6144, 1024) + w_ov = torch.cat( + [w_o[:, 256 * h : 256 * h + 256].float() @ w_vb[h].float() for h in range(4)], dim=1 + ) + if self.leader: + idx = self.indexer + w_fa = decode._fused_a_indexer_weight[layer_idx] + self.w_idx = pack_bf16(decode._wq_b_folded[layer_idx]) + self.idx_kn_w = idx.k_norm.weight.float().contiguous() + self.idx_kn_b = idx.k_norm.bias.float().contiguous() + assert abs(float(idx.k_norm.variance_epsilon) - 1e-6) < 1e-12 + assert not idx.rotary_emb.is_neox + cs = idx.rotary_emb.rotary_cos_sin + self.idx_rope = cs.float().contiguous().view(cs.shape[0], -1) + assert self.idx_rope.shape[1] == 64 + assert idx.n_heads == 32 and idx.head_dim == 128 and idx.index_topk == 2048 + assert not idx.use_fp4 + self.topk_out = torch.full((8, 2048), -1, dtype=torch.int32, device=device) + shared.leader_topk[layer_idx] = self.topk_out + self.w_fa = pack_bf16(w_fa) + self.w_qb = pack_bf16(w_qb) + self.w_o = pack_bf16(w_ov.to(torch.bfloat16)) + del w_ov + self.gamma_kv = attn.kv_a_layernorm.weight.contiguous() + self.rope = attn.mqa.rotary_cos_sin.contiguous().view(-1) + self.w_r = pack_bf16(moe.gate.weight.to(torch.bfloat16)) + self.router_bias = moe.gate.e_score_correction_bias.to(torch.bfloat16).contiguous() + gu = moe.shared_experts.gate_up_proj.weight + dn = moe.shared_experts.down_proj.weight + assert tuple(gu.shape) == (256, 6144) and tuple(dn.shape) == (6144, 128) + self.w_gu = pack_bf16(gu) + self.w_d = pack_bf16(dn) + w13 = _u8(backend.w3_w1_weight).reshape(256, 2, 128, 48, 64).permute(0, 1, 3, 2, 4) + w13 = swizzle_64b(w13.contiguous()).reshape(256, 2, 8, 6 * 8192) + s13 = _u8(backend.w3_w1_weight_scale).reshape(256, 2, 8, 12 * 512) + self.w13 = torch.cat([w13, s13], dim=3).contiguous() + self.s13 = torch.zeros(1, dtype=torch.uint8, device=device) + w2 = swizzle_64b(_u8(backend.w2_weight).reshape(256, 48, 128, 64)).reshape(256, 8, 6 * 8192) + s2 = _u8(backend.w2_weight_scale).reshape(256, 8, 6 * 2 * 512) + self.w2 = torch.cat([w2, s2], dim=2).contiguous() + self.s2 = torch.zeros(1, dtype=torch.uint8, device=device) + del w13, s13, w2, s2 + assert getattr(backend, "fc31_act_scale", None) is None + scales = [] + for name in ("fc31_input_scale", "fc2_input_scale"): + value = getattr(backend, name, None) + value = ( + torch.ones(1, device=device) + if value is None + else value.detach().float().reshape(-1).to(device) + ) + assert value.numel() == 1 and bool(torch.isfinite(value).all()) and float(value) > 0 + scales.append(value) + self.alpha1 = torch.cat([backend.fc31_alpha.float().reshape(256), scales[0]]).contiguous() + self.alpha2 = torch.cat([backend.fc2_alpha.float().reshape(256), scales[1]]).contiguous() + self.gamma_post = layer.post_attention_layernorm.weight.contiguous() + self.gamma_next = layer.next_layer_layernorm.weight.contiguous() + self.w_lg = None + if shared.router_lg: + rank = int(self.mapping.tp_rank) + tf32 = torch.backends.cuda.matmul.allow_tf32 + torch.backends.cuda.matmul.allow_tf32 = False + try: + wg = moe.gate.weight.float() * self.gamma_post.float().view(1, -1) + wov = torch.cat( + [w_o[:, 256 * h : 256 * h + 256].float() @ w_vb[h].float() for h in range(4)], + dim=1, + ) + product = torch.cat([wg @ wov, wg[:, 384 * rank : 384 * rank + 384]], dim=1) + finally: + torch.backends.cuda.matmul.allow_tf32 = tf32 + assert tuple(product.shape) == (256, 64 * _LG_CHUNKS) + self.w_lg = pack_bf16(product.to(torch.bfloat16)) + + def bind(self, manager) -> None: + """Rebuild raw cache addresses after profiling replaces the KV manager, outside capture.""" + shared = self.shared + device = self.w_fa.device + kv_pool = manager.get_unique_primary_pool() + if self.leader: + kbuf = manager.get_indexer_k_cache_buffers(self.layer_idx) + assert kbuf.dtype == torch.uint8 and tuple(kbuf.shape[1:]) == (64, 1, 132) + assert kbuf.stride()[1:] == (132, 132, 1) and kbuf.stride(0) % 128 == 0 + self.idx_block_stride = int(kbuf.stride(0)) + rows = ((kbuf.shape[0] - 1) * self.idx_block_stride + 8448) // 128 + self.idx_tmap = native_ops.glm_megakernel_indexer_tmap(kbuf, rows).to(device) + local_layer = manager.layer_offsets[self.layer_idx] + scale, layer_offset = manager.get_primary_pool_page_index_params(local_layer) + assert manager.tokens_per_block == 64 + self.stride_rows = int(scale) * 64 + self.layer_rows = int(layer_offset) * 64 + self.kv_base = kv_pool.data_ptr() + self.layer.self_attn.mqa._ensure_rope_table_size(manager.max_seq_len) + self.rope = self.layer.self_attn.mqa.rotary_cos_sin.contiguous().view(-1) + roles, role_smem, role_issue = shared.tables[self.leader] + tables = torch.cat([roles.reshape(-1), role_smem.reshape(-1), role_issue.reshape(-1)]) + wt_offset = len(_FIELDS) + tables.numel() // 2 + p = torch.zeros( + wt_offset + _CTAS * _MAX_ROLES * 3, dtype=torch.int64, pin_memory=prefer_pinned() + ) + p[len(_FIELDS) : wt_offset] = tables.view(torch.int64) + + def put(name: str, value: int | torch.Tensor) -> None: + p[_P[name]] = value.data_ptr() if isinstance(value, torch.Tensor) else value + + for name, value in ( + ("Magic", shared.magic), + ("KvBase", self.kv_base), + ("StrideRows", self.stride_rows), + ("LayerRows", self.layer_rows), + ("Rope", self.rope), + ("WFa", self.w_fa), + ("WQb", self.w_qb), + ("WO", self.w_o), + ("WR", self.w_r), + ("WGu", self.w_gu), + ("WD", self.w_d), + ("GammaKv", self.gamma_kv), + ("GammaPost", self.gamma_post), + ("GammaNext", self.gamma_next), + ("RouterBias", self.router_bias), + ("W13", self.w13), + ("S13", self.s13), + ("W2", self.w2), + ("S2", self.s2), + ("Alpha1", self.alpha1), + ("Alpha2", self.alpha2), + ("Arena", shared.arena), + ("PartO", shared.part_o), + ("PartStats", shared.part_stats), + ("Fc2out", shared.fc2out), + ("Counters", shared.counters), + ("State", shared.state), + ("Err", shared.err), + ("IdxCand", shared.idx_cand), + ("Leader", int(self.leader)), + ("NumCopies", shared.copies), + ("H1Uc", shared.uc.data_ptr()), + ("H1Mc", shared.mc.data_ptr()), + ("H2Uc", shared.uc.data_ptr() + shared.copies * _HOP_BYTES), + ("H2Mc", shared.mc.data_ptr() + shared.copies * _HOP_BYTES), + ("TkUc", shared.uc.data_ptr() + 2 * shared.copies * _HOP_BYTES), + ("TkMc", shared.mc.data_ptr() + 2 * shared.copies * _HOP_BYTES), + ("Copy", self.copy), + ("HopBytes", _HOP_BYTES), + ("Rank", self.mapping.tp_rank), + ("MaxRoles", _MAX_ROLES), + ("TimeoutNs", 20_000_000_000), + ("Flags", self.flags), + ): + put(name, value) + if self.w_lg is not None: + put("WLg", self.w_lg) + if self.leader: + for name, value in ( + ("IdxWq", self.w_idx), + ("IdxKnW", self.idx_kn_w), + ("IdxKnB", self.idx_kn_b), + ("IdxRope", self.idx_rope), + ("IdxPool", kbuf), + ("IdxTmap", self.idx_tmap), + ("TopkOut", self.topk_out), + ("IdxBlockStride", self.idx_block_stride), + ): + put(name, value) + self.params = p.to(device) + native_ops.glm_megakernel_weight_table(self.params, wt_offset) + p[wt_offset:] = self.params[wt_offset:].cpu() + self.params_host = p + self.step_params = {} + + def forward( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + metadata, + fallback_topk: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + if self.leader: + topk = self.topk_out + indexer_blocks = metadata.indexer_k_cache_block_offsets[0] + else: + topk = self.shared.leader_topk.get(self.leader_idx, fallback_topk) + indexer_blocks = None + block_table = metadata.block_table[0] + kv_lens = metadata.kv_lens_cuda_runtime + max_blocks = metadata.block_table.shape[1] + key = (None if indexer_blocks is None else indexer_blocks.data_ptr(), max_blocks) + entry = self.step_params.get(key) + if entry is None: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError("Warm up the GLM megakernel metadata before CUDA graph capture") + if indexer_blocks is not None: + self.params_host[_P["IdxBlocks"]] = indexer_blocks.data_ptr() + self.params_host[_P["MaxBlocks"]] = max_blocks + params = self.params_host.to(self.params.device) + # Keep the metadata view alive for every cached raw pointer. + entry = (params, indexer_blocks) + self.step_params[key] = entry + # The executor owns KV pools and may release and rebuild them between forwards. + manager = metadata.kv_cache_manager + kv_cache = [manager.get_unique_primary_pool()] + if self.leader: + kv_cache.append(manager.get_indexer_k_cache_buffers(self.layer_idx)) + return native_ops.glm_megakernel( + entry[0], + hidden_states, + residual, + topk, + block_table, + kv_lens, + self.shared.mutable, + kv_cache, + self.pdl, + ) + + +class GlmMegakernel: + """Opt-in replacement of all C1 MoE decoder layers for an eligible verify step.""" + + def __init__(self, decode: DecodeTarget, layer_indices: list[int]) -> None: + self.shared = _Shared(decode.mapping) + self.layers = {} + self.manager = None + indices = set(layer_indices) + for ordinal, idx in enumerate(layer_indices): + self.layers[idx] = _Layer( + decode, + idx, + self.shared, + ordinal % self.shared.copies, + idx - 1 in indices, + idx + 1 in indices, + ) + logger.info(f"[glm5_megakernel] prepared {len(self.layers)} MoE layers for TP16/B1/K7") + + @classmethod + def create(cls, decode: DecodeTarget) -> GlmMegakernel | None: + if decode.mapping.tp_size != 16 or os.environ.get("TRTLLM_GLM5_MEGAKERNEL", "0") != "1": + return None + if not prefer_pinned(): + raise RuntimeError( + "GLM megakernel requires pinned host memory for its CUDA error record" + ) + config = decode.core.model_config.pretrained_config + indices = [ + idx + for idx in range(config.first_k_dense_replace, config.num_hidden_layers) + if idx % config.moe_layer_freq == 0 + ] + props = torch.cuda.get_device_properties(torch.cuda.current_device()) + if ( + not indices + or len(indices) % 3 + or (props.major, props.minor) != (10, 7) + or props.multi_processor_count != _CTAS + ): + raise ValueError( + "GLM megakernel requires complete three-layer groups on SM107 with 212 SMs" + ) + return cls(decode, indices) + + def begin(self, hidden_states: torch.Tensor, metadata, spec_metadata) -> bool: + if tuple(hidden_states.shape) != (8, 6144) or hidden_states.dtype != torch.bfloat16: + return False + if metadata.num_contexts != 0 or metadata.num_seqs != 1 or metadata.num_tokens != 8: + return False + if getattr(metadata, "in_mtp_draft_loop", False): + return False + if spec_metadata is not None and any( + spec_metadata.is_layer_capture(i) for i in self.layers + ): + return False + manager = metadata.kv_cache_manager + if ( + manager is None + or manager.tokens_per_block != 64 + or manager.max_seq_len > self.shared.max_keys + ): + return False + if self.manager is None or self.manager() is not manager: + if torch.cuda.is_current_stream_capturing(): + return False + for impl in self.layers.values(): + impl.bind(manager) + self.manager = weakref.ref(manager) + logger.info("[glm5_megakernel] bound KV cache for TP16/B1/K7 decode") + return True diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu new file mode 100644 index 000000000000..1b3f4ae4e616 --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu @@ -0,0 +1,6766 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// One GLM-5.2-NVFP4 plain decoder layer (a MoE layer that reuses its leader's top-k) as one kernel, for one shape: +// Rubin VR200 (212 SMs), attention TP16 + MoE TP16, 8 query rows (batch 1, MTP 7), FP8 KV in 64-token blocks. +// +// The launch is 212 CTAs of 256 threads, one per SM. Every CTA runs a list of work units fixed on the host (stage, +// unit, shared-memory offset, and after which unit its weights are requested), so the weights of a unit are in shared +// memory before its input exists whenever its slot is free early. +// +// Data is its own flag. Every activation buffer starts as the empty word 0x80000000 (FP32 -0.0) in every 32-bit word; +// producers store whole words (a payload word equal to the empty word is stored as 0, the same value in every format +// used here), consumers poll exactly the words they read. Buffers read by one consumer are cleared by it after use; +// the others live in an arena with C >= 3 copies (kPNumCopies): launch n (of the replaced layers, in order) uses copy +// n % C and clears copy (n + 1) % C for the next launch (its readers, launch n + 1 - C <= n - 2, finished before launch +// n began). The counters come in C sets the same way. The two all-reduces over the 16 GPUs work the same way across +// GPUs: producers store their shard through the multicast address into every GPU's slot buffer. The stores are weak and +// the polls volatile: PTX promises a whole-word read only between strong accesses, so this relies on the hardware +// storing aligned 32-bit words whole, and no system-scope fence follows the multicast stores (fence.sc.sys there cost +// +52..+58 us per slice). +// +// Stages, in dependency order: fused-A (q_a | c_kv | k_pe), q_b' (absorbed, q_a RMSNorm folded, RoPE, E4M3 Q), the +// KV append, the sparse attention over the 2048 selected rows (per row 16 blocks of 128), the merge, o_proj with v_b +// folded in (W_OV, K = 4 heads x 512; publishes the attention all-reduce), the attention all-reduce + residual + +// RMSNorm + NVFP4 quantization, the router GEMV and the noaux_tc top-8, the shared expert, the routed experts (FC1 +// blocks with SwiGLU + NVFP4, FC2 eighths, dealt over all CTAs), the top-8 combine + shared add (publishes the MoE +// all-reduce), and the MoE all-reduce + residual + next RMSNorm. +// +// Every GEMM is mma.sync m16n8k16 with M = weight rows and N = the 8 rows: weights are repacked on the host into the +// fragment order of the MMA (BF16 directly, NVFP4 with their E4M3 block scales applied to exact FP16 values), with a +// K permutation inside each 64-wide chunk that lets every thread read its activation fragment as 32 contiguous bytes. + +#include "glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.h" + +#include +#include +#include +#include +#include +#include + +#include + +#define GLM52_STAMPS(c) static_cast(nullptr) +#define GLM52_FLAGS(c) specializedFlags(c) + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels::glm_moe_dsa::megakernel +{ + +namespace glm52 +{ + +// ---------------------------------------------------------------------------------------------------------------- +// Shape +// ---------------------------------------------------------------------------------------------------------------- +constexpr int kThreads = 256; +constexpr int kWarps = 8; +constexpr int kCtas = 212; +constexpr int kTok = 8; +constexpr int kHidden = 6144; +constexpr int kRanks = 16; +constexpr int kHeads = 4; +constexpr int kFaRows = 2624; +constexpr int kFaChunks = 96; +constexpr int kQbRows = 2304; +constexpr int kQbChunks = 32; +constexpr int kQbUnitTiles = 3; +constexpr int kTopK = 2048; +constexpr int kRowBytes = 576; +constexpr int kSmemRowBytes = kRowBytes + 16; +constexpr int kAttnBlocks = 16; +constexpr int kAttnKeys = kTopK / kAttnBlocks; +constexpr int kPRowBytes = kAttnKeys + 16; +constexpr int kDv = 512; +constexpr int kOChunks = 32; +constexpr int kRChunks = 96; +constexpr int kExperts = 256; +constexpr int kColsPerHopUnit = kHidden / 4; +constexpr int kBf16Block = 2048; +constexpr float kPScale = 448.f; +constexpr float kEps = 1e-5f; +constexpr uint32_t kEmpty = 0x80000000u; + +// Routed experts: 16E FC1 pieces (g = 16 slot + 8 tile + K piece: 128 rows x 6 K blocks of 128) and 8E FC2 pieces +// (j = 8 slot + eighth: output rows [768 eighth, +768) = 6 tiles of 128 rows), each list cut into 212 equal runs, CTA c +// taking FC1 pieces [16E c / 212, 16E (c + 1) / 212) and then FC2 pieces [8E c / 212, 8E (c + 1) / 212): every +// CTA's FC1 work is one round of 4-5 pieces (E = 57), so all FC1 partials are in at about the same time. A piece is +// 48 KiB of E2M1 in SWIZZLE_64B atoms (128 rows x 64 B per K block of 128) and its 12 K steps of block scales (512 B +// each, the 128x4 layout). +constexpr int kPieceW = 6 * 128 * 64; // 49,152 +constexpr int kPieceS = 12 * 512; // 6,144 +constexpr int kPieceBytes = kPieceW + kPieceS; // 55,296 +constexpr int kRingSlots = 3; +constexpr int kHfp4Bytes = kTok * 3072; // h in E2M1, [8][3072] +constexpr int kHsfBytes = kTok * 384; // its E4M3 block scales, [8][384] + +// One all-reduce buffer (one launch copy): [8 tokens][16 ranks][6144] BF16. +constexpr int kHopBytes = kTok * kRanks * kHidden * 2; +// The leader's top-k exchange (one launch copy, multicast like the all-reduce buffers): int32 [8 rows][2048], each row +// published by the even rank of the pair that selects it (the row shard; unused with kFlagIdxAllRows). +constexpr int kTkBytes = kTok * kTopK * 4; +// GLM52_ROUTER_LG, the router logits by linearity: W_r RMSNorm_post(h') x rms = W_r diag(gamma) (res_in + sum_r W_OV,r +// O_r), so each rank's 16 logit units (experts [16 u, +16)) multiply [O_r | res_in's columns [384 r, +384)] by the +// host-made M_r = W_r diag(gamma) [W_OV,r | I] (K = 2048 + 384) as soon as O is merged, and publish the FP32 partials +// through the multicast like the hops: [16 ranks][16 units][8 tokens][16 experts] per launch copy, after the top-k +// exchange's copies in the same workspace. The same unit then sums its slot column u of the 16 ranks in rank order into +// kArLg and empties it (single reader); the routing scales by 1 / rms as before. +constexpr int kRouterLg = GLM52_ROUTER_LG; +constexpr int kLgUnits = 16; +constexpr int kLgResChunks = kHidden / kRanks / 64; // 6: the rank's 384 residual columns +constexpr int kLgChunks = kOChunks + kLgResChunks; // 38 K chunks of 64 per unit (host layout) +constexpr int kLgSlotBytes = kTok * 16 * 4; // 512 +constexpr int kLgBytes = kRanks * kLgUnits * kLgSlotBytes; // 131,072 per launch copy +constexpr int kLgOprojFirst = 148; // logit unit u rides on o_proj unit kLgOprojFirst + u (host) + +// Arena of multi-reader activation buffers, per launch copy (byte offsets). +constexpr int kArY = 0; // BF16 [8][2624] fused-A output +constexpr int kArQ8 = 41984; // E4M3 [8][2304] Q +constexpr int kArKv = 60416; // E4M3 [8][576] this step's KV rows +constexpr int kArO = 65024; // BF16 [8][4][512] merged attention output +// Leader layers (the DSA indexer) in the 16 KiB v_b no longer needs (folded into o_proj): +constexpr int kArIdxKw = 97792; // BF16 [8][160] fused-A rows 2624-2783: indexer K (128), head weights (32) +constexpr int kArIdxAmax = 100352; // FP32 [8][32][4] indexer Q: |max| of each head's four 32-row quarters +constexpr int kArIdxKNew = 104448; // E4M3 [8][128] this step's indexer keys +constexpr int kArIdxKNewS = 105472; // FP32 [8] their scales +constexpr int kArPStats = 105504; // FP32 [8][16][4][2] kFlagSliceMerge: the attention blocks' (max, sum) per head +static_assert(kArIdxKNewS + 32 <= kArPStats && kArPStats + 8 * 16 * 4 * 8 <= 114176, "arena layout"); +constexpr int kArLg = 105600; // FP32 [8][256] GLM52_ROUTER_LG: the router logits (16 ranks' partials summed, + // before the 1 / rms scale); shares kArPStats' space (no kFlagSliceMerge with it) +static_assert(kArIdxKNewS + 32 <= kArLg && kArLg % 128 == 0 && kArLg + 8 * 256 * 4 <= 114176, "arena layout"); +constexpr int kArH = 114176; // BF16 [8][6144] h (attention all-reduce, RMSNorm) +constexpr int kArHfp4 = 212480; // E2M1 [8][3072] NVFP4(h) +constexpr int kArHsf = 237056; // E4M3 [8][384] its block scales (right after the E2M1 values) +constexpr int kArSsq1 = 240128; // FP32 [8][4] sum-of-squares partials, attention all-reduce +constexpr int kArSsq2 = 240256; // FP32 [8][4] sum-of-squares partials, MoE all-reduce +constexpr int kArLogit = 240384; // FP32 [4][8][256] router logits per K quarter +constexpr int kArRoute = 273152; // int32 [256] routing (E, slot experts, slot of (t, k), weights) +constexpr int kArSh = 274176; // FP32 [4][8][32][8] shared gate_up partials per K quarter +constexpr int kArP1 = 306944; // FP32 [212][2][128][8] FC1 partials (CTA, run, row, token); kFlagFc2Direct uses + // [212][2][8][4][16][2] (CTA, run, token, feature block, feature, up/gate) +constexpr int kArIdxQ8 = 2043648; // E4M3 [8][32][128] indexer Q (RoPE'd, per-head UE8M0 scale), leader layers +constexpr int kArIdxW = 2076416; // FP32 [8][32] indexer head weights x q_scale x 128^-0.5 x 32^-0.5 +constexpr int kArIdxBnd = 2077440; // u32 [215][8] per CTA and row: a lower bound of its 10th-largest key; T, T2, + // E (or u64 [8][212]: per row and CTA, its lanes' 9th and 10th maxima) +constexpr int kArIdxTopk = 2109184; // int32 [8][2048] the leader's top-k again (its attention polls these words) +// BF16 [8][32][128]: the leader's indexer Q before E4M3 under the row shard (its IdxSel CTAs quantize it; the row shard +// leaves the top-k's arena copy unused: its attention reads the exchange). +constexpr int kArIdxQb = kArIdxTopk; +static_assert(8 * 32 * 128 * 2 == 8 * 2048 * 4, "the BF16 indexer Q in the top-k's arena copy"); +constexpr int kArX = 2174720; // BF16 [8][6144] this launch's x, written by the previous launch's H2 +static_assert(kArIdxBnd + (212 + 3) * 8 * 4 <= kArIdxTopk, "arena layout"); // the CTAs' words, then the rows' T, T2, E +static_assert(kArIdxBnd + 212 * 8 * 8 <= kArIdxTopk, "arena layout"); // or per (row, CTA) one 64-bit word +static_assert(kArIdxTopk + 8 * 2048 * 4 <= kArX, "arena layout"); +constexpr int kORepBytes = kTok * 4 * 512 * 2; +constexpr int kArORep = kArX + kTok * kHidden * 2; +constexpr int kArHp = kArORep + 3 * kORepBytes; // BF16 [8][6144] kFlagHPrime: h' = res x gamma, before the RMS scale +constexpr int kArXSsq = kArHp + kTok * kHidden * 2; // FP32 [8][4] kFlagXPrime: this launch's x' sum-of-squares partials +constexpr int kArenaBytes = kArXSsq + kTok * 4 * 4; + +// Clear units (16 KiB): the next launch's hop buffers (2 x kHopBytes) and top-k exchange, then its arena copy. +constexpr int kClearUnitBytes = 16384; +constexpr int kHopClearUnits = (2 * kHopBytes + kTkBytes) / kClearUnitBytes; +static_assert((2 * kHopBytes) % kClearUnitBytes == 0 && kTkBytes % kClearUnitBytes == 0, "clear units"); +constexpr int kArenaClearUnits = (kArenaBytes + kClearUnitBytes - 1) / kClearUnitBytes; +constexpr int kClearUnits = kHopClearUnits + kArenaClearUnits; +// Every clear unit signals kCClear (the owners and the next launch's fused-A wait for kCountedClear of them): that +// fused-A stores its rows (kArY; kArIdxKw in leaders) before its grid-dependency wait, so its wait for the count is +// what orders those clears first, as it does the clears of x it polls. +constexpr int kCountedClearUnits = kHopClearUnits + kArenaClearUnits; + +// ---------------------------------------------------------------------------------------------------------------- +// Leader layers: the DSA indexer (32 heads x 128, keys in a paged FP8 cache) and its exact top-2048 +// ---------------------------------------------------------------------------------------------------------------- +constexpr int kIdxHeads = 32; +constexpr int kIdxDim = 128; +constexpr int kIdxRows = kIdxHeads * kIdxDim; // 4096 rows of the indexer wq_b +constexpr int kIdxQUnitRows = 32; // an IdxQ unit: two 16-row tiles, a quarter of one head +constexpr int kIdxQChunks = 32; // K = 2048 in 64-wide chunks +constexpr int kFaTilesLeader = 174; // fused-A with the indexer rows: 2784 = 174 x 16 +constexpr int kIdxKwCols = 160; +static_assert(kFaTilesLeader * 16 == kFaRows + kIdxKwCols, "leader fused-A tiles"); +constexpr int kIdxPageKeys = 64; +constexpr int kIdxPageBytes = kIdxPageKeys * (kIdxDim + 4); // 8448: 64 keys x 128 B, then their 64 FP32 scales +static_assert(kIdxPageBytes % 128 == 0 && kIdxRows == 128 * kIdxQUnitRows, "indexer layout: 128 IdxQ units"); +constexpr int kIdxMaxPages = 16; // pages per IdxSel round (its K region's) +constexpr int kSelBatchPages = kIdxMaxPages / 2; // the row shard's page batches (half the K region each) +#if GLM52_TOPK_UNION +constexpr int kCandList = 1 << 20; // candidates per row (list A): any row's keys (kIdxMaxKeys) +constexpr int kCandCap = kCandList + 8192; // per row: list A, then the union's 13,568 words +#else +constexpr int kCandList = 1 << 18; // candidates per list (list A, then list B): any row's keys +constexpr int kCandCap = 2 * kCandList; // top-k candidates per row: list A, then list B +#endif +static_assert(kCtas * kIdxMaxPages * kIdxPageKeys + 8 <= kCandList, "a list holds every key of a row"); +constexpr int kCandSmem = 16384; // the candidates the shared-memory selection copies (A, then B) +constexpr int kCandSmemList = kCandSmem / 2; // per list; a list taken past it is selected in global memory +constexpr uint32_t kTightGap = 3u << 17; // T2 = E - this (order-key units: 2.3-4.7 % of the value) +constexpr uint32_t kGallop = 1u << 17; // the selector's first step from E (0.8-1.6 %) +constexpr int kSelSmall = 512; // a selection window this small is ranked directly +constexpr int kSelWin = 64; // idxSelectRow: the search's window ranked directly +constexpr int kHistBins = 64; // per row: list A's histogram, bins from T2 +constexpr int kHistShift = 14; // its bins' width in order-key units (0.1-0.2 % of the value) +constexpr uint32_t kHistSpan = static_cast(kHistBins) << kHistShift; // [T2, T2 + this): ~E - 3 % .. E + 5 % +constexpr int kIdxQUnits = 128; // IdxQ units = quarter heads, on CTAs [0, 128) +constexpr int kIdxSelBase = 204; // the CTA selecting row r: kIdxSelBase + r +constexpr int kH1Cta0 = 180; // the H1 / H2 units' CTAs: 180-211 (host build_roles) +constexpr float kIdxWeightScale = 0.015625f; // 128^-0.5 * 32^-0.5 +// IdxSel's shared memory (from its offset, 1024-aligned): the K pages (SWIZZLE_128B atoms; once a 128-key tile's +// MMAs are done, its first 4 KiB hold the tile's logits, FP32 [8 tokens][128 keys]), Q in E4M3 ([256 (token, head) +// rows][128] SWIZZLE_128B), the pages' key scales, the head weights. The top-k histograms use the reduction area. +constexpr int kSelK = 0; +constexpr int kSelQ8 = kSelK + kIdxMaxPages * 8192; // 32 KiB +constexpr int kSelKs = kSelQ8 + 256 * 128; // FP32 [16][64] +constexpr int kSelW = kSelKs + kIdxMaxPages * 256; // FP32 [8][32] +constexpr int kSelBytes = kSelW + 1024; +constexpr int kSelLaneMax + = 8192; // row shard: the tile epilogues' largest keys, u32 [8 warps][32 lanes], reduction area +constexpr int kSelLaneMax2 = 9216; // row shard: their second largest keys, u32 [8 warps][32 lanes] +static_assert(kSelQ8 % 1024 == 0 && kSelBytes <= 200704, "IdxSel shared memory"); +// The row shard: the row's logits, FP32 [local key] (key 64 j + r of the CTA's page j), past the rounds' region; IdxSel +// takes kSelTotal bytes (the host's unit size). +constexpr int kSelLog = kSelBytes; +constexpr int kSelTotal = 200704; +constexpr int kIdxMaxPagesRs = (kSelTotal - kSelLog) / (4 * 2 * kIdxPageKeys) * 2; // a CTA's pages, whole tiles: 124 +// The longest context the indexer takes (its keys: the cached ones and the step's kTok). kFlagPageBalance (always on) +// puts kIdxBalDeficit pages fewer on each IdxQ CTA than on each CTA without an IdxQ unit (idxPageOf), at most +// kIdxMaxPagesRs on a CTA; a candidate list holds every key of a row. +constexpr int kIdxBalDeficit = 7; +constexpr int kIdxBalCap = (kCtas - kIdxQUnits) * kIdxBalDeficit + kCtas * (kIdxMaxPagesRs - kIdxBalDeficit); +constexpr int kIdxMaxKeys = kIdxBalCap * kIdxPageKeys < kCandList ? kIdxBalCap * kIdxPageKeys : kCandList; +// RS-H's lattice (rsBin): bins of 1/128 binade (the order key's top 16 bits) for |logit| in [2^-13, 2^10) on each sign +// (kRsW bins per sign), each sign's two tails, and +0.0 / -0.0 exactly, in key order: [0] logit <= -2^10, [1, kRsW] +// the negative window, [kRsW + 1] (-2^-13, -0.0), [kRsW + 2] -0.0, [kRsW + 3] +0.0, [kRsW + 4] (+0.0, 2^-13), +// [kRsW + 5, 2 kRsW + 4] the positive window, [2 kRsW + 5] >= 2^10. The window covers every 2048th logit seen so far +// (synthetic rows 0.0028-0.83, served rows -24..-100); a cut in a tail bin is still exact (the global selection). +// The global histogram's bin b sits at word 4 (b / kRsPer + kThreads * (b % kRsPer / 4)) + b % 4: thread t's six +// coalesced 16-byte polls hold its bins [kRsPer t, + kRsPer) in order. +constexpr int kRsW = 23 * 128; +constexpr uint32_t kRsN0 = 0x3B80u; // the negative window's top-16 keys [kRsN0, kRsN0 + kRsW) +constexpr uint32_t kRsP0 = 0xB900u; // the positive window's [kRsP0, kRsP0 + kRsW) +constexpr uint32_t kRsBinNZ = kRsW + 2; +constexpr uint32_t kRsBinPZ = kRsW + 3; +constexpr int kRsPer = 24; +constexpr int kRsBins = kRsPer * kThreads; +static_assert(2 * kRsW + 6 <= kRsBins && kRsPer % 4 == 0, "RS-H lattice"); +static_assert(kRsN0 + kRsW == 0x4700u && kRsP0 + kRsW == 0xC480u, "RS-H window: |logit| in [2^-13, 2^10)"); +// Each CTA adds to the global histogram only its bins from the one holding its kRsLocalK-th largest key up (its keys +// counted in shared memory during the tile epilogues): ~27k adds per row whatever the context. Per-key adds of every +// key backed up on the hot bins (one-GPU test: 12 us at 335k, 32 us at 900k on served-like rows; same-address adds +// serialize at ~7-9 ns). Served rows hold at most ~69 of the row's top 2048 in one CTA (one hot page). +constexpr uint32_t kRsLocalK = 128; +constexpr uint32_t kRsPairs = 512; // a cut bin of at most this many keys is ranked by the row CTA in shared memory +constexpr uint32_t kRsDirect = 1; // descriptor modes: the cut bin's keys go straight into the row (taken whole, or +constexpr uint32_t kRsRank = 2; // equal keys), ranked by the row CTA, or selected exactly in global memory; +constexpr uint32_t kRsGlobal = 3; // kRsMore: the cut lies below a CTA's added bins, every CTA adds the rest first +constexpr uint32_t kRsMore = 4; + +// Stage codes of a work unit (role entry = stage << 16 | unit). +enum Stage : int +{ + kStNone = 0, + kStFusedA = 1, + kStQb = 2, + kStKvAppend = 3, + kStAttn = 4, + kStMerge = 5, + kStOproj = 7, + kStClear = 8, + kStH1 = 9, + kStRouterGemv = 10, + kStSharedGu = 12, + kStMoe = 13, + kStOwner = 14, + kStH2 = 15, + kStOwnerShared = 16, + kStIdxK = 17, // leader: this step's indexer keys (LayerNorm, RoPE, E4M3) into the cache and the arena + kStIdxQ = 18, // leader: 32 rows of the indexer Q (q_a norm folded, RoPE), BF16, and their |max| per head quarter + kStIdxSel = 19, // leader: the logits of the CTA's key pages, then the exact top-2048 with every other CTA + kStOprojLg = 20, // GLM52_ROUTER_LG: o_proj unit u (CTAs 148-163), then on the same O this rank's partial router + // logits of experts [16 (u - 148), +16) with its residual columns, multicast, and their ranks' sum +}; + +// Counters (u32, the next copy zeroed in every launch's prologue): edges that cannot carry their own flag. +enum Counter : int +{ + kCClear = 0, // hop-buffer clear units done (ordered before the MoE all-reduce's publish) + kCExit = 1, + kCMoeDone = 2, // CTAs past their MoE unit (kFlagHoldExit) + kCXReady = 3, // (unused) + // Leader: the top-k rows done (polled by every CTA past its part of the top-k: a 128-byte line of its own, away + // from the candidate counts the selectors poll). + kCIdxDone = 16, + // Leader top-k: per row the bound T (its complement, stored by the row's selector for the record), and per row a + // 64-bit pair: the CTAs that have taken their candidate slots and list A's candidates ((CTAs << 24) | count), then + // list B's candidates (low 18 bits) and, with the row shard, list A's keys in its histogram (high 14 bits). + kCIdxT = 32, + kCIdxCand = kCIdxT + kTok, + kCIdxTies = kCIdxCand + 2 * kTok, // per row: keys equal to T2 taken from list A's pool (past each CTA's free ones) + kCIdxTiesB = kCIdxTies + kTok, // per row: keys equal to T taken from list B's pool + kCIdxWritten = kCIdxTiesB + kTok, // CTAs done storing candidates (a release after the join; the selector acquires) + kCIdxHist = 128, // per row: list A's histogram (kHistBins bins of keys - T2, from T2); the row shard uses its row's + // Optional cooperative bin emission, one row per rank. This tail is included in the C ABI counter count and + // the existing next-copy clear; no word is reused from a live histogram or another row's candidate metadata. + kCIdxParState = kCIdxHist + kTok * kHistBins, // 0: pending, 1: original selector, 2: descriptor published + kCIdxParCount, // sure positions in low 16 bits, boundary entries in high 16 + kCIdxParDone, // eight CTAs' release-published completed scans + kCIdxParN, + kCIdxParWin, // uint4: the histogram window + kCIdxParKey = kCIdxParWin + 4, + kCIdxParPos = kCIdxParKey + kSelWin, + kNumCounters = (kCIdxParPos + kSelWin + 31) & ~31, // preserve the original 128-byte alignment of every copy +}; + +static_assert(kCIdxWritten < kCIdxHist, "counters"); + +// Parameter block (int64 words, filled by the host). +enum Param : int +{ + kPMagic, + kPKvBase, + kPStrideRows, + kPLayerRows, + kPMaxBlocks, + kPRope, + kPWFa, + kPWQb, + kPWLg, // GLM52_ROUTER_LG: W_r diag(gamma_post) [W_OV of this rank | this rank's 384 residual columns], packed + // BF16 [16 tiles][38 chunks][2048 B] (the slot of the folded-away v_b weights) + kPWO, + kPWR, + kPWGu, + kPWD, + kPGammaKv, + kPGammaPost, + kPGammaNext, + kPRouterBias, + kPW13, + kPS13, + kPW2, + kPS2, + kPAlpha1, + kPAlpha2, + kPArena, + kPPartO, + kPPartStats, + kPFc2out, + kPH1Mc, + kPH1Uc, + kPH2Mc, + kPH2Uc, + kPHopBytes, + kPRank, + kPCounters, + kPState, + kPErr, + kPRoles, + kPRoleSmem, + kPRoleIssue, + kPMaxRoles, + kPTimeoutNs, + kPStamps, + kPFlags, + kPCopy, // this launch's copy of the arena, hop buffers and counters (launch index mod kPNumCopies) + kPNumCopies, // copies in the rotation (the launches per replay are a multiple of it) + kPLeader, // 1: a leader layer (the role table has the indexer stages; the attention waits for the own top-k) + kPIdxWq, // indexer wq_b with the q_a norm folded in, packed BF16 [256 tiles][32 chunks][2048 B] + kPIdxKnW, // indexer K LayerNorm weight, FP32 [128] + kPIdxKnB, // its bias, FP32 [128] + kPIdxRope, // indexer cos/sin, FP32 [positions][64] = [cos 0-31 | sin 0-31] + kPIdxPool, // this layer's indexer K cache: block b (kIdxPageBytes) at kPIdxPool + b * kPIdxBlockStride + kPIdxBlockStride, // bytes between the layer's blocks (the pool interleaves the indexer layers), a multiple of 128 + kPIdxTmap, // the pool as a 2-D tensor map [rows of 128 B], SWIZZLE_128B boxes of 64 rows (CUtensorMap) + kPIdxBlocks, // the request's indexer K cache block table, int32 + kPIdxCand, // top-k boundary candidates, uint2 (key, position) [copies][8][kCandCap] + kPTopkOut, // the leader's top-k, int32 [8][2048] (the next plain layers' input) + kPIdxDebug, // check mode: the leader's logits, FP32 [8][kPIdxDebugLd] (0: off) + kPIdxDebugLd, + kPIdxDebug2, // check mode: CTA 0's first key page (64 x 128 B, unswizzled), its Q8 (256 x 128 B), its tile 0's D + // (FP32 [128 keys][256]); 0: off + kPTkUc, // the top-k exchange (kTkBytes per launch copy): this GPU's copy, and its multicast address + kPTkMc, + kPNumParams, +}; + +// Variant bits (kPFlags): wait on one canary word per producer unit before the bulk poll of an activation many CTAs +// read at once (the bulk poll of every waiting CTA otherwise re-reads the whole region every round trip). +enum Flag : int64_t +{ + kFlagFc2Masked = 1ll << 38, // skip arithmetic/shuffles for tokens outside the uniform expert mask + kFlagMergeDry = 1ll << 39, // merge: its code once on zeros (no stores) before the poll, so the real pass runs warm + kFlagHopSelfClear + = 1ll << 37, // (always on) the H1 units empty the 16 hop-1 slots they read and the previous hop-2's + // of their columns (single readers); the host schedules no clear units for the hop + // buffers (each signalled with a release: ~1 us, and their stores after o_proj's) + kFlagXFrag = 1ll << 40, // (always on) x' in the arena in fused-A's fragment order: per 64-column chunk two + // 512-byte halves, lane l's 16 bytes of half h at 16 l (each poll instruction reads 4 + // whole lines once, not half of 8 lines twice) + kFlagPageBalance = 1ll << 55, // leader: more key pages on the CTAs without an IdxQ unit (idxPageOf) + kFlagAttnIlp = 1ll << 56, // attention: four Q.K accumulator chains, two P.V output blocks per iteration + kFlagSliceMerge = 1ll << 58, // attention unit (t, b) merges dims [32 b, +32) of row t itself (attnSliceMerge): no + // merge units (the host layout drops them) + kFlagXPrime + = 1ll << 51, // (always on) H2 publishes the next launch's x' = res' x gamma_next (kArX) and its per-unit sum of + // squares (kArXSsq) before its exchange; that launch's fused-A (early x) scales its + // outputs by 1 / rms (every output row is linear in x): the exchange leaves the x path + kFlagWarmInHop = 1ll << 47, // (always on) H1 CTAs (180-211): the MoE warm-up (kFlagWarmMoe) in H1's hop-1 wait, not + // between H1 and the routing (those CTAs set the MoE tail) + kFlagSelFast = 1ll << 42, // (always on) leader top-k (row shard, kFlagIdxParallel): every selector CTA takes + // the counts and the window itself, the boundary keys are polled 64-bit slots (no + // release counters) + kFlagHopClearLate + = 1ll << 32, // (always on) the H1 units' hop-slot self-clear at the start of the same CTA's H2 unit (it waits for + // hop 2 there), not at H1's end (~1.4 of H1's 1.9 us, before those CTAs' MoE) + kFlagLandProbe = 1ll << 41, // stamped runs: sub-stamp 128 when the setup's weight requests went out, 129 when the + // first fused-A / q_b' unit's weights landed (thread 0 waits for them before its poll) + kFlagFirstEarly = 1ll << 50, // (always on) thread 0 requests the first unit's weights (setup-issued ones) first, + // from the tables in one round trip, before the parameter block and the setup (the + // CTAs that start last, on the SMs the previous layer's H2 units held, get q_b''s + // weights ~0.6-0.9 us after its input otherwise) + kFlagHPrime = 1ll << 48, // (always on) H1 publishes h' = res x gamma (kArHp) before its sum-of-squares exchange; + // the router and the shared gate_up run on it, the routing and the shared expert's combine + // scale by 1 / rms (from kArSsq1): the exchange leaves the routing path +#if GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5 + kFlagSparseP1 = 1ll << 54, // direct P1 only: publish/gather routed tokens, zero inactive FC2 operands +#endif + kFlagCanaryMoe = 1, // MoE CTAs: the router units' logits, then the routing, then NVFP4(h) + kFlagCanaryOproj = 2, // o_proj: the v_b units + kFlagCanaryH = 4, // router GEMV and shared gate_up: the H1 units of the K quarter + kFlagCanaryOwner = 8, // owner: the shared gate_up units + kFlagHoldExit = 16, // CTAs exit only once every CTA's MoE unit is done (the next kernel's prologue loads then + // stay out of the MoE weight stream) + kFlagNextChained = 32, // (set by the host per launch) the next launch is this kernel's and reads x' from the arena + // (kFlagEarlyX): H2 leaves out the normalized x and its sum-of-squares exchange + kFlagEvictFirst = 64, // weight bulk copies with the L2 evict-first policy + kFlagWarmMoe = 128, // MoE: one FC1 and one FC2 MMA batch on stale operands before the routing (the MMA code + // paths' first run otherwise lands on the expert pieces' critical path) + kFlagOsInMoe = 512, // the shared-expert unit runs inside the MoE stage after the first expert requests (its + // own unit before the MoE stage returns at once) + kFlagNoStamps = 2048, // no timing stamps (the timed comparison with the stock layer) + kFlagMoeL2 = 4096, // MoE: the CTA's pieces beyond the ring are prefetched into L2 right after the ring's requests + // (they then land from L2 when a ring slot frees, not from HBM a round trip later) + kFlagMoeL2Normal = 8192, // those prefetches with the default L2 policy (else evict-first with kFlagEvictFirst) + kFlagL2Deferred = 16384, // leader, at entry: the CTA's indexer Q weights (requested once fused-A frees their slot) + kFlagEarlyX = 32768, // the previous launch is this kernel: fused-A polls x from the arena (its H2 wrote it + // there), before the grid-dependency wait + kFlagCheckX = 65536, // with kFlagEarlyX: CTA 0 compares that x with the kernel argument after the wait (trap 98) + kFlagOsPrefetch = 131072, // MoE CTAs with a shared-expert unit: its down weights into L2 before the expert requests + kFlagIdxQEarly = 262144, // leader fused-A: its IdxQ unit's weights into L2 once its own landed, and into the slot + // as soon as its MMAs are done (not after the unit) + kFlagTopkOneList = 524288, // leader top-k without the bounds' estimate: one list (keys >= T), plain bisection + kFlagMergeClearFirst = 1048576, // merge units clear the attention partials before their output (else after it) + kFlagOwnerClearFirst = 2097152, // owner units clear the routed rows they read before their shard (else after it) + kFlagIdxAllRows = 4194304, // leader: every rank scores and selects all 8 rows (else the row shard: rank r scores + // only row r / 2 (logits N = 32), the even rank of the pair publishes its selection to + // every rank through the top-k exchange, and the attention takes all 8 rows from there) + kFlagTopkNoHist = 8388608, // the selectors without list A's histogram: the search starts from the whole list, not + // from the bin of the row's need0-th key + kFlagTopkNoBin + = 16777216, // with the histogram's bin window: the packed search from it (else one pass, idxSelectBin) + kFlagTopkTCta = 33554432, // T, T2 and E from one CTA (the lanes' top four, ten warp rounds, two hops; else every + // CTA takes them from the words of the lanes' maxima itself) + kFlagIdxTilePairs + = 536870912, // row shard: the logits' tiles two at a time in 256-column TMEM buffers (as all rows) + kFlagBoundLoop = 1073741824, // row shard: the bound's lane maxima from a pass over the logits (else from the tile + // epilogues) + kFlagBinCopy = 268435456, // the bin selector reads list A from its bulk copy (else straight from L2 in its pass) + kFlagMoeAllFirst = 67108864, // MoE: the first three pieces requested at once (else piece 0 first, pieces 1-2 after + // the FC2 groups' inputs) + kFlagBinNoPreload = 256, // the direct bin pass loads its first pairs itself (else the caller, before the window) + kFlagBinVolatile = 1024, // the direct bin pass reads the list with volatile loads (else ld.global.cg) + kFlagRstdShared = 1ll << 33, // (always on) MoE (kFlagHPrime): the routing polls the tokens' sum-of-squares partials + // with the logits and leaves each token's 1 / rms in the MoE list, where the shared + // expert inside the MoE stage reads it (else four L2 round trips per thread there) + kFlagCandWarps + = 1ll << 36, // (always on) leader (row shard): the CTA's candidates counted and written by all 8 warps (warp w: its + // lanes' keys m in [w nPer / 8, (w + 1) nPer / 8)), one slot atomic per CTA + kFlagUnionLean + = 1ll << 31, // (always on) leader union T: on CTA 127 (an IdxQ CTA, the fewest key pages) while it holds fewer + // than CTAs 128-211's 14, else on CTA 204; the union's slots zeroed after T's publish + kFlagUnionSide = 1ll << 27, // (always on) leader union T: pass 1 over the side of zero that holds the need-th + // largest key (served rows: a few positive logits, the 2048th negative; the window + // below the largest key held too few and a full-range fallback pass ran) + kFlagHop1Lines = 1ll << 34, // (always on) hop 1's buffer as [rank][o_proj unit][token][32 columns]: an o_proj + // unit's 8 tokens are one 512 B block (full lines in the multicast; else 8 half-lines) + kFlagPageMap2 = 1ll << 35, // leader: the key pages by the CTA groups' IdxSel starts, for every context length + // (idxPageMap; else kFlagPageBalance's map, tuned at 147k) + kFlagFc2Direct = 1ll << 46, // FC1 partials as [token][block][feature][up/gate]; FC2 sums its own features in + // registers, without the shared-memory partial transpose + kFlagIdxParallel = 1ll << 49, // even row shards: CTAs 204..211 emit a small-bin selection cooperatively +}; + +constexpr int64_t kMagic = 0x474c4d3532504c32LL; // "GLM52PL2" +constexpr int kMaxRoles = 16; +// Stamps per CTA (u64 %globaltimer): [role][start, inputs in (thread 0, last poll), weights in, end], then entry, PDL +// wait return, exit, MoE weights requested, then kSubStamps points inside q_b' (0-4), the attention (5-11) and the MoE +// (12-15, piece p landed / consumed at 16 + 2p / 17 + 2p, the FC1 partials of the CTA's item k in at 32 + k; FC2 group +// g < 2 polling its partials from 73 + g to 75 + g; piece p < 8 waited for from 80 + p); the SM clock (%clock64) of +// sub-stamps 40-71 at 96 + k - 40 (cycles against ns: the SM's speed in a step). +constexpr int kSubStamps = 136; +constexpr int kStampsPerCta = kMaxRoles * 4 + 4 + kSubStamps; + +// Shared memory: unit regions [0, kSmemMain), then fixed areas. Past the 227 KB opt-in limit (VR200: up to +// 334,848 B per block) the kernel runs in the oversized shared-memory mode (8 KB of L1 left). +#ifndef GLM52_SMEM_MAIN +#define GLM52_SMEM_MAIN 200704 +#endif +constexpr int kSmemMain = GLM52_SMEM_MAIN; +constexpr int kSmemRed = kSmemMain; +constexpr int kSmemRedBytes = 16384; +constexpr int kSmemRes = kSmemRed + kSmemRedBytes; +constexpr int kSmemResBytes = 3072; +constexpr int kSmemSmall = kSmemRes + kSmemResBytes; +#if GLM52_SETUP_MODE == 0 +constexpr int kSmemSmallBytes = 2688; +#else +constexpr int kSmemSmallBytes = 3072; +#endif +constexpr int kSmemBytes = kSmemSmall + kSmemSmallBytes; + +// Small area layout (bytes from kSmemSmall). +constexpr int kSmallBars = 0; // kMaxRoles unit barriers + 8 attention + kRingSlots MoE ring + MoE MMAs + + // leader IdxSel: its key pages, its two TMEM buffers' MMAs, the row shard's odd + // page batches +constexpr int kBarIdxK = kMaxRoles + 8 + kRingSlots + 1; +constexpr int kBarIdxK2 = kBarIdxK + 3; +constexpr int kNumBars = kBarIdxK2 + 1; +static_assert(kNumBars * 8 <= 256, "barriers"); +constexpr int kSmallRoles = 256; // 3 x kMaxRoles int32 +constexpr int kSmallScalars = 512; // misc (128 floats) +constexpr int kSmallMoe = 1024; // MoE work list (int32) +constexpr int kSmallParams = 2048; // the parameter block (int64), copied at entry +constexpr int kSmallCtx = 2560; // the kernel context (struct Ctx) +static_assert(kSmallParams + kPNumParams * 8 <= kSmallCtx, "parameter block does not fit"); +#if GLM52_SETUP_MODE == 1 +constexpr int kSmallWeights = 2688; +static_assert(kSmallWeights + kMaxRoles * 3 * 8 <= kSmemSmallBytes, "weight table row does not fit"); +#endif + +// MoE work list (int32 words at kSmallMoe): E, piece count, the CTA's FC1 and FC2 piece ranges, the slot of every +// (token, k), the slot experts, the routing weights, the slots' token masks (bytes). +constexpr int kMoeE = 0; +constexpr int kMoePieces = 1; +constexpr int kMoeF1A = 2; +constexpr int kMoeF1B = 3; +constexpr int kMoeF2A = 4; +constexpr int kMoeF2B = 5; +constexpr int kMoeWarmed = 6; // nonzero: an earlier unit of the CTA ran the MoE warm-up (zeroed at entry) +constexpr int kMoeRcp = 8; // [8] float bits: the tokens' 1 / rms (kFlagRstdShared) +constexpr int kMoeSlotOfTk = 32; // [64] +constexpr int kMoeSlotExpert = 96; // [64] +constexpr int kMoeWeight = 160; // [64] float bits +constexpr int kMoeSlotMask = 224; // [64] bytes + +// Route block layout (int32 words) in the arena; the vectors are 16-byte aligned. +constexpr int kRouteE = 0; +constexpr int kRouteSlotExpert = 4; // [64] +constexpr int kRouteSlotOfTk = 68; // [8][8] +constexpr int kRouteWeight = 132; // [8][8] float bits + +// The MoE cut's weights per CTA class (W, OS, SG, H1: see moeCutWeight), written with kFc2ListsPacked by gen_cut.py. +constexpr int kFc2ListsCut[4] = {6, 4, 5, 5}; +#if GLM52_LISTS_MODE >= 1 +__device__ __constant__ __align__(16) uint32_t kFc2ListsPacked[4160][3] = { + {0x1105c168u, 0xa118722eu, 0x00000612u}, + {0x4a20ce48u, 0x32e156a0u, 0x00000d36u}, + {0x0882c0a8u, 0x40883817u, 0x000002e1u}, + {0x1f8e4688u, 0xd9509a46u, 0x00000612u}, + {0x3d9c8d28u, 0xd2511883u, 0x00000a04u}, + {0x58aad4c8u, 0x6b197eb8u, 0x00000d36u}, + {0x0581c068u, 0xd85c260fu, 0x000001e0u}, + {0x15098448u, 0xc8d8642eu, 0x000003d1u}, + {0x25918828u, 0xe960a84fu, 0x00000612u}, + {0x391b0cc8u, 0x4a0cfa78u, 0x000008f4u}, + {0x4e261288u, 0x5aa148a0u, 0x00000af5u}, + {0x5e2e1668u, 0x7b298cc1u, 0x00000d36u}, + {0x04014048u, 0xa0441c0bu, 0x00000170u}, + {0x0f870348u, 0x58a04a22u, 0x000002e1u}, + {0x1b8d0628u, 0x10fc7839u, 0x00000462u}, + {0x28134928u, 0xf16cae54u, 0x00000612u}, + {0x371a4ca8u, 0xf9ecec72u, 0x00000833u}, + {0x48231108u, 0xea692e94u, 0x00000a04u}, + {0x54a99468u, 0xa2c55cabu, 0x00000b85u}, + {0x612fd768u, 0x833592c6u, 0x00000d36u}, + {0x03010048u, 0x80341609u, 0x00000120u}, + {0x0c85c288u, 0x10803c1bu, 0x00000251u}, + {0x160a44e8u, 0xa8c8602eu, 0x00000371u}, + {0x1f0f0728u, 0x49188640u, 0x000004b2u}, + {0x2a1449c8u, 0xf970b257u, 0x00000612u}, + {0x3599cc88u, 0xc9d4e46eu, 0x000007c3u}, + {0x43a0d008u, 0xa2451c8au, 0x00000974u}, + {0x4f26d328u, 0x3a9144a0u, 0x00000a95u}, + {0x582b9568u, 0xd2e16ab2u, 0x00000bd5u}, + {0x6330d808u, 0x8b3996c8u, 0x00000d36u}, + {0x0280c028u, 0x682c1207u, 0x000000f0u}, + {0x0a84c228u, 0xe06c3217u, 0x000001e0u}, + {0x12088408u, 0x60a85026u, 0x000002e1u}, + {0x1a0c8608u, 0xd8e46e36u, 0x000003d1u}, + {0x221047e8u, 0x692c9046u, 0x000004f2u}, + {0x2b150a28u, 0xf974b658u, 0x00000612u}, + {0x34998c68u, 0xa9c8de6cu, 0x00000783u}, + {0x401f4f68u, 0x62250c83u, 0x000008f4u}, + {0x4b251248u, 0xf2713498u, 0x00000a04u}, + {0x53291448u, 0x6aad52a8u, 0x00000af5u}, + {0x5aacd628u, 0xfaf174b8u, 0x00000c15u}, + {0x64319868u, 0x8b3d9acau, 0x00000d36u}, + {0x0200c028u, 0x58241006u, 0x000000d0u}, + {0x090401c8u, 0xc05c2a13u, 0x000001a0u}, + {0x0f874388u, 0x30904421u, 0x00000271u}, + {0x160ac528u, 0x98c45e2eu, 0x00000341u}, + {0x1c8e06c8u, 0x00f87a3bu, 0x00000422u}, + {0x24118888u, 0x8138984au, 0x00000522u}, + {0x2c158a88u, 0x0178b85au, 0x00000613u}, + {0x34194c68u, 0x91bcda6au, 0x00000743u}, + {0x3e1e4ee8u, 0x320d027eu, 0x00000884u}, + {0x48235168u, 0xc2592892u, 0x00000994u}, + {0x4f275368u, 0x2a8d42a0u, 0x00000a65u}, + {0x55aa9508u, 0x92c15eadu, 0x00000b45u}, + {0x5d2e16c8u, 0x13017cbcu, 0x00000c46u}, + {0x653218c8u, 0x93419cccu, 0x00000d36u}, + {0x02008028u, 0x50200e05u, 0x000000b0u}, + {0x078381a8u, 0xa8502411u, 0x00000170u}, + {0x0d868308u, 0x087c3c1cu, 0x00000221u}, + {0x13894488u, 0x60ac5228u, 0x000002e1u}, + {0x190c45e8u, 0xc0dc6a34u, 0x00000391u}, + {0x1f0f0768u, 0x2108823fu, 0x00000462u}, + {0x25924908u, 0x91409e4du, 0x00000542u}, + {0x2c95cac8u, 0x0178b85bu, 0x00000613u}, + {0x33994c68u, 0x81b8d669u, 0x00000723u}, + {0x3c1d8e88u, 0x09fcfa7bu, 0x00000834u}, + {0x452210a8u, 0x92411c8cu, 0x00000944u}, + {0x4ca5d2c8u, 0xf275369au, 0x00000a04u}, + {0x5228d428u, 0x52a54ea6u, 0x00000ab5u}, + {0x582b95a8u, 0xb2d166b1u, 0x00000b85u}, + {0x5eaed728u, 0x230980bfu, 0x00000c66u}, + {0x65b258e8u, 0x93419ccdu, 0x00000d36u}, + {0x01808028u, 0x481c0c05u, 0x000000a0u}, + {0x07030168u, 0x9848200fu, 0x00000140u}, + {0x0c05c2a8u, 0xe8703619u, 0x000001e0u}, + {0x11084408u, 0x38984a24u, 0x00000291u}, + {0x168ac548u, 0x90c05e2eu, 0x00000331u}, + {0x1b8d8688u, 0xe0ec7238u, 0x000003d1u}, + {0x209007e8u, 0x39188843u, 0x00000492u}, + {0x27130968u, 0xa148a24fu, 0x00000552u}, + {0x2d160ae8u, 0x017cba5cu, 0x00000613u}, + {0x33190c68u, 0x71b0d468u, 0x00000703u}, + {0x3b1d0e48u, 0xe9f0f478u, 0x000007f3u}, + {0x42a0d028u, 0x6a2d1287u, 0x000008f4u}, + {0x4a24d228u, 0xca612e96u, 0x000009b4u}, + {0x4fa75388u, 0x228942a0u, 0x00000a55u}, + {0x54aa14c8u, 0x72b556aau, 0x00000af5u}, + {0x59ac9628u, 0xcae16cb5u, 0x00000bb5u}, + {0x5faf9788u, 0x331186c1u, 0x00000c76u}, + {0x66329928u, 0x93419ecdu, 0x00000d36u}, + {0x01808028u, 0x40180a04u, 0x00000090u}, + {0x0602c148u, 0x88401e0du, 0x00000120u}, + {0x0b050268u, 0xd0643017u, 0x000001b0u}, + {0x0f878388u, 0x18884220u, 0x00000251u}, + {0x1409c4c8u, 0x68b05429u, 0x000002e1u}, + {0x188c05e8u, 0xb0d46832u, 0x00000371u}, + {0x1d8e4708u, 0xf8f87a3cu, 0x00000401u}, + {0x2290c848u, 0x51248e46u, 0x000004b2u}, + {0x281389a8u, 0xa950a451u, 0x00000572u}, + {0x2d964b08u, 0x017cbc5cu, 0x00000613u}, + {0x33190c68u, 0x69acd267u, 0x000006e3u}, + {0x3a1c8e08u, 0xd9e4ee75u, 0x000007c3u}, + {0x41200fc8u, 0x421d0a83u, 0x000008a4u}, + {0x47a39188u, 0xaa512691u, 0x00000974u}, + {0x4d265308u, 0xfa79389bu, 0x00000a04u}, + {0x51a89428u, 0x429d4ca4u, 0x00000a95u}, + {0x56aad548u, 0x8ac15eaeu, 0x00000b25u}, + {0x5b2d5688u, 0xe2e972b8u, 0x00000bd5u}, + {0x60b017e8u, 0x3b1988c3u, 0x00000c86u}, + {0x66b2d948u, 0x93459eceu, 0x00000d36u}, + {0x01808028u, 0x38180a04u, 0x00000080u}, + {0x05828128u, 0x78381a0cu, 0x00000100u}, + {0x0a048228u, 0xc05c2c15u, 0x00000190u}, + {0x0e06c348u, 0x007c3c1du, 0x00000211u}, + {0x1208c448u, 0x48a04c25u, 0x000002a1u}, + {0x168b0568u, 0x88c05e2eu, 0x00000321u}, + {0x1a8d0668u, 0xc8e06e36u, 0x000003a1u}, + {0x1f0f4788u, 0x1104803fu, 0x00000432u}, + {0x239188a8u, 0x612c9448u, 0x000004d2u}, + {0x289409e8u, 0xb154a853u, 0x00000582u}, + {0x2d968b28u, 0x017cbc5du, 0x00000613u}, + {0x32990c68u, 0x61a8d067u, 0x000006d3u}, + {0x391c0de8u, 0xc1dcea73u, 0x000007a3u}, + {0x3f9f4f68u, 0x2a0d0480u, 0x00000864u}, + {0x45a29108u, 0x8a411c8du, 0x00000934u}, + {0x4b255288u, 0xda693097u, 0x000009c4u}, + {0x4fa793a8u, 0x1a8942a0u, 0x00000a45u}, + {0x53a994a8u, 0x5aa952a8u, 0x00000ac5u}, + {0x582bd5c8u, 0xa2cd64b1u, 0x00000b55u}, + {0x5cae16c8u, 0xf2f576bau, 0x00000bf5u}, + {0x61b09828u, 0x431d8cc4u, 0x00000c96u}, + {0x66b31968u, 0x9345a0ceu, 0x00000d36u}, + {0x01004008u, 0x30140803u, 0x00000070u}, + {0x05024108u, 0x7034180bu, 0x000000f0u}, + {0x09044208u, 0xb0542813u, 0x00000170u}, + {0x0d064307u, 0xf074381bu, 0x00000000u}, + {0x108803e8u, 0x28904622u, 0x00000261u}, + {0x148a04e8u, 0x68b0562au, 0x000002e1u}, + {0x188c05e8u, 0xa8d06632u, 0x00000361u}, + {0x1c8e06e7u, 0xe8f0763au, 0x00000001u}, + {0x200fc7c8u, 0x29108441u, 0x00000462u}, + {0x249208e8u, 0x7134984bu, 0x000004f2u}, + {0x29944a08u, 0xb958aa54u, 0x00000582u}, + {0x2e16cb47u, 0x097cbc5du, 0x00000003u}, + {0x3298cc48u, 0x59a4d066u, 0x000006c3u}, + {0x381bcdc8u, 0xb1d4e672u, 0x00000783u}, + {0x3e1ecf28u, 0x1200fe7du, 0x00000834u}, + {0x442190a8u, 0x6a311489u, 0x000008f4u}, + {0x49a49208u, 0xba592a94u, 0x00000984u}, + {0x4da69328u, 0xfa793a9cu, 0x00000a04u}, + {0x51a89428u, 0x3a994aa4u, 0x00000a85u}, + {0x55aa9527u, 0x7ab95aacu, 0x00000005u}, + {0x592c5608u, 0xbad568b3u, 0x00000b85u}, + {0x5dae9728u, 0x02fd7abcu, 0x00000c16u}, + {0x6230d848u, 0x4b218ec6u, 0x00000ca6u}, + {0x67335968u, 0x9345a0cfu, 0x00000d36u}, + {0x01004008u, 0x30140803u, 0x00000070u}, + {0x05024107u, 0x7034180bu, 0x00000000u}, + {0x088401e7u, 0xa8502612u, 0x00000000u}, + {0x0c05c2c7u, 0xe06c3419u, 0x00000000u}, + {0x0f8783a7u, 0x18884220u, 0x00000001u}, + {0x13094487u, 0x50a45027u, 0x00000001u}, + {0x168b0567u, 0x88c05e2eu, 0x00000001u}, + {0x1a0cc648u, 0xc0dc6c35u, 0x00000391u}, + {0x1d8e8738u, 0xf8f87a3cu, 0x00000401u}, + {0x21908828u, 0x39188a44u, 0x00000482u}, + {0x25928928u, 0x81389a4cu, 0x00000512u}, + {0x2a14ca48u, 0xc15cac55u, 0x00000592u}, + {0x2e16cb47u, 0x0980bc5du, 0x00000003u}, + {0x3298cc48u, 0x51a4ce66u, 0x000006b3u}, + {0x379b8da8u, 0xa9cce471u, 0x00000763u}, + {0x3d1e4ee8u, 0xf9f8fa7bu, 0x00000813u}, + {0x42a0d048u, 0x52250e86u, 0x000008b4u}, + {0x47a391a8u, 0xa24d2491u, 0x00000954u}, + {0x4c25d2c7u, 0xe26d3499u, 0x00000004u}, + {0x4fa793a7u, 0x1a8942a0u, 0x00000005u}, + {0x53295488u, 0x52a550a7u, 0x00000ab5u}, + {0x56ab1578u, 0x8ac15eaeu, 0x00000b25u}, + {0x5b2d1667u, 0xd2e570b7u, 0x00000005u}, + {0x5eaf1768u, 0x0b017ebeu, 0x00000c26u}, + {0x63315868u, 0x532590c7u, 0x00000cb6u}, + {0x67335988u, 0x9345a0cfu, 0x00000d36u}, + {0x01004007u, 0x30140803u, 0x00000000u}, + {0x048200e7u, 0x6830160au, 0x00000000u}, + {0x0803c1c6u, 0x004c2411u, 0x00000000u}, + {0x0b054287u, 0xd0643017u, 0x00000000u}, + {0x0e870367u, 0x08803e1eu, 0x00000001u}, + {0x11888437u, 0x38984a24u, 0x00000001u}, + {0x150a4507u, 0x70b4582bu, 0x00000001u}, + {0x188c05e6u, 0x00d06632u, 0x00000000u}, + {0x1b8d86a7u, 0xd8e87238u, 0x00000001u}, + {0x1f0f4787u, 0x1104803fu, 0x00000002u}, + {0x22910868u, 0x49208e46u, 0x000004a2u}, + {0x26930968u, 0x89409e4eu, 0x00000522u}, + {0x2a950a68u, 0xc960ae56u, 0x000005a2u}, + {0x2e970b67u, 0x0980be5eu, 0x00000003u}, + {0x3218cc48u, 0x49a0ce65u, 0x000006a3u}, + {0x371b4d88u, 0x99c8e26fu, 0x00000743u}, + {0x3c1dcec8u, 0xe9f0f679u, 0x000007e3u}, + {0x41205008u, 0x3a190883u, 0x00000884u}, + {0x4622d128u, 0x8a411c8du, 0x00000924u}, + {0x4aa51267u, 0xca612e96u, 0x00000004u}, + {0x4e26d347u, 0x027d3c9du, 0x00000005u}, + {0x51a89426u, 0x02994aa4u, 0x00000000u}, + {0x54aa14e7u, 0x6ab156aau, 0x00000005u}, + {0x582bd5c7u, 0xa2cd64b1u, 0x00000005u}, + {0x5bad96a8u, 0xdae972b8u, 0x00000bc5u}, + {0x5faf97a8u, 0x1b0982c0u, 0x00000c46u}, + {0x63b198a8u, 0x5b2992c8u, 0x00000cc6u}, + {0x67b399a7u, 0x9b49a2d0u, 0x00000006u}, + {0x01004007u, 0x30140803u, 0x00000000u}, + {0x0401c0d7u, 0x602c1409u, 0x00000000u}, + {0x078381a6u, 0x00482210u, 0x00000000u}, + {0x0a850266u, 0x00602e16u, 0x00000000u}, + {0x0d868326u, 0x00783a1cu, 0x00000000u}, + {0x108803e7u, 0x28904622u, 0x00000001u}, + {0x138984b7u, 0x58a85228u, 0x00000001u}, + {0x170b4586u, 0x00c4602fu, 0x00000000u}, + {0x1a0cc646u, 0x00dc6c35u, 0x00000000u}, + {0x1d0e4706u, 0x00f4783bu, 0x00000000u}, + {0x200fc7c7u, 0x210c8441u, 0x00000002u}, + {0x239188a7u, 0x59289248u, 0x00000002u}, + {0x27134988u, 0x9144a04fu, 0x00000532u}, + {0x2b154a87u, 0xd164b057u, 0x00000002u}, + {0x2e970b67u, 0x0980be5eu, 0x00000003u}, + {0x3218cc48u, 0x499ccc65u, 0x000006a3u}, + {0x369b0d68u, 0x91c4e06eu, 0x00000733u}, + {0x3b9d4e88u, 0xd9e8f278u, 0x000007c3u}, + {0x401fcfa8u, 0x220d0481u, 0x00000864u}, + {0x44a210e8u, 0x7235168au, 0x000008f4u}, + {0x49245208u, 0xb2552893u, 0x00000974u}, + {0x4ca612f7u, 0xea71369au, 0x00000004u}, + {0x5027d3c6u, 0x028d44a1u, 0x00000000u}, + {0x53295486u, 0x02a550a7u, 0x00000000u}, + {0x562ad546u, 0x02bd5cadu, 0x00000000u}, + {0x592c5607u, 0xb2d568b3u, 0x00000005u}, + {0x5cae16e7u, 0xeaf176bau, 0x00000005u}, + {0x602fd7c8u, 0x230d84c1u, 0x00000c56u}, + {0x6431d8c7u, 0x632d94c9u, 0x00000006u}, + {0x67b399a7u, 0x9b49a2d0u, 0x00000006u}, + {0x01004006u, 0x00140803u, 0x00000000u}, + {0x0401c0c6u, 0x002c1409u, 0x00000000u}, + {0x07034186u, 0x0044200fu, 0x00000000u}, + {0x0a04c246u, 0x005c2c15u, 0x00000000u}, + {0x0c8602f6u, 0x0070361au, 0x00000000u}, + {0x0f8783a6u, 0x00884220u, 0x00000000u}, + {0x12890466u, 0x00a04e26u, 0x00000000u}, + {0x158a8526u, 0x00b85a2cu, 0x00000000u}, + {0x188c05e6u, 0x00d06632u, 0x00000000u}, + {0x1b0d4696u, 0x00e47037u, 0x00000000u}, + {0x1e0ec746u, 0x00fc7c3du, 0x00000000u}, + {0x21104807u, 0x31148843u, 0x00000002u}, + {0x249208e7u, 0x6930964au, 0x00000002u}, + {0x2813c9c7u, 0xa14ca451u, 0x00000002u}, + {0x2b958aa7u, 0xd968b258u, 0x00000002u}, + {0x2f174b86u, 0x0184c05fu, 0x00000000u}, + {0x3218cc48u, 0x419ccc65u, 0x00000693u}, + {0x361acd48u, 0x89c0de6eu, 0x00000723u}, + {0x3a9d0e68u, 0xc9e0ee76u, 0x000007b3u}, + {0x3f1f4f88u, 0x1205007fu, 0x00000834u}, + {0x43a15088u, 0x5a291288u, 0x000008c4u}, + {0x47a391a7u, 0xa2492290u, 0x00000004u}, + {0x4ba592a6u, 0x02693298u, 0x00000000u}, + {0x4ea71366u, 0x02813e9eu, 0x00000000u}, + {0x51a89426u, 0x02994aa4u, 0x00000000u}, + {0x5429d4d6u, 0x02ad54a9u, 0x00000000u}, + {0x572b5586u, 0x02c560afu, 0x00000000u}, + {0x5a2cd647u, 0xc2dd6cb5u, 0x00000005u}, + {0x5dae9727u, 0xfaf97abcu, 0x00000005u}, + {0x61305807u, 0x331588c3u, 0x00000006u}, + {0x6431d8d8u, 0x632d94c9u, 0x00000cd6u}, + {0x67b399b7u, 0x9b49a2d0u, 0x00000006u}, + {0x01004006u, 0x00140803u, 0x00000000u}, + {0x0401c0c5u, 0x00001409u, 0x00000000u}, + {0x06830166u, 0x00401e0eu, 0x00000000u}, + {0x09848225u, 0x00002a14u, 0x00000000u}, + {0x0c05c2c6u, 0x006c3419u, 0x00000000u}, + {0x0e870376u, 0x00803e1eu, 0x00000000u}, + {0x11888426u, 0x00984a24u, 0x00000000u}, + {0x1409c4d6u, 0x00ac5429u, 0x00000000u}, + {0x170b4586u, 0x00c4602fu, 0x00000000u}, + {0x198c8636u, 0x00d86a34u, 0x00000000u}, + {0x1c8e06e5u, 0x0000763au, 0x00000000u}, + {0x1f0f4786u, 0x0104803fu, 0x00000000u}, + {0x2210c847u, 0x411c8c45u, 0x00000002u}, + {0x25124917u, 0x7134984bu, 0x00000002u}, + {0x289409e7u, 0xa950a652u, 0x00000002u}, + {0x2b958ab7u, 0xd968b258u, 0x00000002u}, + {0x2f174b86u, 0x0184c05fu, 0x00000000u}, + {0x3218cc48u, 0x419ccc65u, 0x00000693u}, + {0x361acd48u, 0x81bcdc6du, 0x00000713u}, + {0x3a1cce48u, 0xc1dcec75u, 0x00000793u}, + {0x3e1ecf48u, 0x01fcfc7du, 0x00000814u}, + {0x4220d048u, 0x421d0c85u, 0x00000894u}, + {0x4622d148u, 0x8a411c8du, 0x00000924u}, + {0x4aa51266u, 0x02612e96u, 0x00000000u}, + {0x4d265316u, 0x0275389bu, 0x00000000u}, + {0x5027d3c6u, 0x028d44a1u, 0x00000000u}, + {0x52a91476u, 0x02a14ea6u, 0x00000000u}, + {0x55aa9525u, 0x00015aacu, 0x00000000u}, + {0x582bd5c6u, 0x02cd64b1u, 0x00000000u}, + {0x5b2d5686u, 0x02e570b7u, 0x00000000u}, + {0x5e2ed747u, 0x02fd7cbdu, 0x00000006u}, + {0x61b09826u, 0x03198ac4u, 0x00000000u}, + {0x64b218e7u, 0x6b3196cau, 0x00000006u}, + {0x6833d9c6u, 0x034da4d1u, 0x00000000u}, + {0x01004006u, 0x00140803u, 0x00000000u}, + {0x038180b6u, 0x00281208u, 0x00000000u}, + {0x0602c156u, 0x003c1c0du, 0x00000000u}, + {0x09044205u, 0x00002813u, 0x00000000u}, + {0x0b8582a5u, 0x00003218u, 0x00000000u}, + {0x0e06c345u, 0x00003c1du, 0x00000000u}, + {0x108803e6u, 0x00904622u, 0x00000000u}, + {0x13094496u, 0x00a45027u, 0x00000000u}, + {0x158a8536u, 0x00b85a2cu, 0x00000000u}, + {0x180bc5d6u, 0x00cc6431u, 0x00000000u}, + {0x1b0d4685u, 0x00007037u, 0x00000000u}, + {0x1d8e8725u, 0x00007a3cu, 0x00000000u}, + {0x200fc7c6u, 0x010c8441u, 0x00000000u}, + {0x23114886u, 0x01249047u, 0x00000000u}, + {0x2612c946u, 0x013c9c4du, 0x00000000u}, + {0x29144a06u, 0x0154a853u, 0x00000000u}, + {0x2c15cac7u, 0xe16cb459u, 0x00000002u}, + {0x2f174b96u, 0x0184c05fu, 0x00000000u}, + {0x3218cc47u, 0x419ccc65u, 0x00000003u}, + {0x359a8d28u, 0x79b8da6cu, 0x00000703u}, + {0x399c8e28u, 0xb9d8ea74u, 0x00000783u}, + {0x3d9e8f27u, 0xf9f8fa7cu, 0x00000003u}, + {0x41205008u, 0x32150883u, 0x00000874u}, + {0x45225108u, 0x7235188bu, 0x000008f4u}, + {0x49245207u, 0xb2552893u, 0x00000004u}, + {0x4c25d2d6u, 0x026d3499u, 0x00000000u}, + {0x4ea71376u, 0x02813e9eu, 0x00000000u}, + {0x51285416u, 0x029548a3u, 0x00000000u}, + {0x5429d4c5u, 0x000154a9u, 0x00000000u}, + {0x56ab1565u, 0x00015eaeu, 0x00000000u}, + {0x592c5606u, 0x02d568b3u, 0x00000000u}, + {0x5bad96b7u, 0xdae972b8u, 0x00000005u}, + {0x5f2f5786u, 0x030580bfu, 0x00000000u}, + {0x6230d846u, 0x031d8cc5u, 0x00000000u}, + {0x65325906u, 0x033598cbu, 0x00000000u}, + {0x6833d9c6u, 0x034da4d1u, 0x00000000u}, + {0x01004005u, 0x00000803u, 0x00000000u}, + {0x038180a5u, 0x00001208u, 0x00000000u}, + {0x0602c145u, 0x00001c0du, 0x00000000u}, + {0x088401e5u, 0x00002612u, 0x00000000u}, + {0x0b054285u, 0x00003017u, 0x00000000u}, + {0x0d064316u, 0x0074381bu, 0x00000000u}, + {0x0f8783b6u, 0x00884220u, 0x00000000u}, + {0x1208c456u, 0x009c4c25u, 0x00000000u}, + {0x148a04f5u, 0x0000562au, 0x00000000u}, + {0x170b4585u, 0x0000602fu, 0x00000000u}, + {0x198c8625u, 0x00006a34u, 0x00000000u}, + {0x1c0dc6c5u, 0x00007439u, 0x00000000u}, + {0x1e8f0765u, 0x00007e3eu, 0x00000000u}, + {0x21104805u, 0x00008843u, 0x00000000u}, + {0x239188a6u, 0x01289248u, 0x00000000u}, + {0x26930966u, 0x01409e4eu, 0x00000000u}, + {0x29948a26u, 0x0158aa54u, 0x00000000u}, + {0x2c960ae6u, 0x0170b65au, 0x00000000u}, + {0x2f978ba5u, 0x0000c260u, 0x00000000u}, + {0x3218cc47u, 0x419ccc65u, 0x00000003u}, + {0x359a8d27u, 0x79b8da6cu, 0x00000003u}, + {0x391c4e07u, 0xb1d4e873u, 0x00000003u}, + {0x3c9e0ee8u, 0xe9f0f67au, 0x000007e3u}, + {0x40a00fe7u, 0x2a110682u, 0x00000004u}, + {0x4421d0c7u, 0x622d1489u, 0x00000004u}, + {0x47a391a8u, 0x9a492290u, 0x00000944u}, + {0x4b255296u, 0x02653097u, 0x00000000u}, + {0x4da69335u, 0x00013a9cu, 0x00000000u}, + {0x5027d3c5u, 0x000144a1u, 0x00000000u}, + {0x52a91465u, 0x00014ea6u, 0x00000000u}, + {0x552a5505u, 0x000158abu, 0x00000000u}, + {0x57ab95a5u, 0x000162b0u, 0x00000000u}, + {0x5a2cd645u, 0x00016cb5u, 0x00000000u}, + {0x5cae16e6u, 0x02f176bau, 0x00000000u}, + {0x5faf97a6u, 0x030982c0u, 0x00000000u}, + {0x62b11866u, 0x03218ec6u, 0x00000000u}, + {0x65b29926u, 0x03399accu, 0x00000000u}, + {0x6833d9d6u, 0x034da4d1u, 0x00000000u}, + {0x01004005u, 0x00000803u, 0x00000000u}, + {0x038180a5u, 0x00001208u, 0x00000000u}, + {0x05828135u, 0x00001a0cu, 0x00000000u}, + {0x0803c1c5u, 0x00002411u, 0x00000000u}, + {0x0a850265u, 0x00002e16u, 0x00000000u}, + {0x0c8602f5u, 0x0000361au, 0x00000000u}, + {0x0f074385u, 0x0000401fu, 0x00000000u}, + {0x11888425u, 0x00004a24u, 0x00000000u}, + {0x138984b5u, 0x00005228u, 0x00000000u}, + {0x160ac545u, 0x00005c2du, 0x00000000u}, + {0x180bc5d5u, 0x00006431u, 0x00000000u}, + {0x1a8d0665u, 0x00006e36u, 0x00000000u}, + {0x1d0e4705u, 0x0000783bu, 0x00000000u}, + {0x1f0f4795u, 0x0000803fu, 0x00000000u}, + {0x21908826u, 0x01188a44u, 0x00000000u}, + {0x249208e5u, 0x0000964au, 0x00000000u}, + {0x27134986u, 0x0144a04fu, 0x00000000u}, + {0x2a14ca46u, 0x015cac55u, 0x00000000u}, + {0x2c960af6u, 0x0170b65au, 0x00000000u}, + {0x2f978ba5u, 0x0000c260u, 0x00000000u}, + {0x3218cc46u, 0x019ccc65u, 0x00000000u}, + {0x351a4d07u, 0x71b4d86bu, 0x00000003u}, + {0x389c0de7u, 0xa9d0e672u, 0x00000003u}, + {0x3c1dcec7u, 0xe1ecf479u, 0x00000003u}, + {0x3f9f8fa7u, 0x1a090280u, 0x00000004u}, + {0x43215087u, 0x52251087u, 0x00000004u}, + {0x46a31167u, 0x8a411e8eu, 0x00000004u}, + {0x4a24d246u, 0x025d2c95u, 0x00000000u}, + {0x4ca612f5u, 0x0001369au, 0x00000000u}, + {0x4f275385u, 0x0001409fu, 0x00000000u}, + {0x51285415u, 0x000148a3u, 0x00000000u}, + {0x53a994a5u, 0x000152a8u, 0x00000000u}, + {0x562ad545u, 0x00015cadu, 0x00000000u}, + {0x582bd5d5u, 0x000164b1u, 0x00000000u}, + {0x5aad1666u, 0x02e16eb6u, 0x00000000u}, + {0x5d2e5716u, 0x02f578bbu, 0x00000000u}, + {0x602fd7c6u, 0x030d84c1u, 0x00000000u}, + {0x63315885u, 0x000190c7u, 0x00000000u}, + {0x65b29926u, 0x03399accu, 0x00000000u}, + {0x68b419e5u, 0x0001a6d2u, 0x00000000u}, + {0x01004005u, 0x00000803u, 0x00000000u}, + {0x03014095u, 0x00001007u, 0x00000000u}, + {0x05828125u, 0x00001a0cu, 0x00000000u}, + {0x078381b5u, 0x00002210u, 0x00000000u}, + {0x0a04c245u, 0x00002c15u, 0x00000000u}, + {0x0c05c2d5u, 0x00003419u, 0x00000000u}, + {0x0e870364u, 0x0000001eu, 0x00000000u}, + {0x108803e5u, 0x00004622u, 0x00000000u}, + {0x12890475u, 0x00004e26u, 0x00000000u}, + {0x150a4505u, 0x0000582bu, 0x00000000u}, + {0x170b4595u, 0x0000602fu, 0x00000000u}, + {0x198c8624u, 0x00000034u, 0x00000000u}, + {0x1b8d86a5u, 0x00007238u, 0x00000000u}, + {0x1d8e8735u, 0x00007a3cu, 0x00000000u}, + {0x200fc7c5u, 0x00008441u, 0x00000000u}, + {0x22910865u, 0x00008e46u, 0x00000000u}, + {0x25124906u, 0x0134984bu, 0x00000000u}, + {0x279389b6u, 0x0148a250u, 0x00000000u}, + {0x2a14ca56u, 0x015cac55u, 0x00000000u}, + {0x2d164b05u, 0x0000b85bu, 0x00000000u}, + {0x2f978ba5u, 0x0000c260u, 0x00000000u}, + {0x3218cc46u, 0x019ccc65u, 0x00000000u}, + {0x351a4d07u, 0x71b4d86bu, 0x00000003u}, + {0x389c0de6u, 0x01d0e672u, 0x00000000u}, + {0x3b9d8ea7u, 0xd9e8f278u, 0x00000003u}, + {0x3f1f4f87u, 0x1205007fu, 0x00000004u}, + {0x4220d057u, 0x421d0c85u, 0x00000004u}, + {0x45a29127u, 0x7a391a8cu, 0x00000004u}, + {0x49245206u, 0x02552893u, 0x00000000u}, + {0x4ba592b5u, 0x00013298u, 0x00000000u}, + {0x4e26d345u, 0x00013c9du, 0x00000000u}, + {0x5027d3d5u, 0x000144a1u, 0x00000000u}, + {0x52a91464u, 0x000000a6u, 0x00000000u}, + {0x54aa14e5u, 0x000156aau, 0x00000000u}, + {0x56ab1575u, 0x00015eaeu, 0x00000000u}, + {0x592c5605u, 0x000168b3u, 0x00000000u}, + {0x5b2d5696u, 0x02e570b7u, 0x00000000u}, + {0x5e2ed745u, 0x00017cbdu, 0x00000000u}, + {0x60b017e6u, 0x031186c2u, 0x00000000u}, + {0x63315896u, 0x032590c7u, 0x00000000u}, + {0x6632d945u, 0x00019ccdu, 0x00000000u}, + {0x68b419e5u, 0x0001a6d2u, 0x00000000u}, + {0x01004005u, 0x00000803u, 0x00000000u}, + {0x03014095u, 0x00001007u, 0x00000000u}, + {0x05024115u, 0x0000180bu, 0x00000000u}, + {0x078381a4u, 0x00000010u, 0x00000000u}, + {0x09848225u, 0x00002a14u, 0x00000000u}, + {0x0b8582b5u, 0x00003218u, 0x00000000u}, + {0x0d868335u, 0x00003a1cu, 0x00000000u}, + {0x1007c3c4u, 0x00000021u, 0x00000000u}, + {0x1208c444u, 0x00000025u, 0x00000000u}, + {0x1409c4c5u, 0x00005429u, 0x00000000u}, + {0x160ac555u, 0x00005c2du, 0x00000000u}, + {0x180bc5d5u, 0x00006431u, 0x00000000u}, + {0x1a8d0664u, 0x00000036u, 0x00000000u}, + {0x1c8e06e4u, 0x0000003au, 0x00000000u}, + {0x1e8f0765u, 0x00007e3eu, 0x00000000u}, + {0x209007f5u, 0x00008642u, 0x00000000u}, + {0x23114885u, 0x00009047u, 0x00000000u}, + {0x25928925u, 0x00009a4cu, 0x00000000u}, + {0x2813c9c6u, 0x014ca451u, 0x00000000u}, + {0x2a950a76u, 0x0160ae56u, 0x00000000u}, + {0x2d164b16u, 0x0174b85bu, 0x00000000u}, + {0x2f978bb5u, 0x0000c260u, 0x00000000u}, + {0x3218cc46u, 0x019ccc65u, 0x00000000u}, + {0x351a4d06u, 0x01b4d86bu, 0x00000000u}, + {0x381bcdc6u, 0x01cce471u, 0x00000000u}, + {0x3b1d4e87u, 0xd1e4f077u, 0x00000003u}, + {0x3e9f0f66u, 0x0200fe7eu, 0x00000000u}, + {0x41a09026u, 0x02190a84u, 0x00000000u}, + {0x44a210e7u, 0x6a31168au, 0x00000004u}, + {0x4823d1c6u, 0x024d2491u, 0x00000000u}, + {0x4b255284u, 0x00000097u, 0x00000000u}, + {0x4d265305u, 0x0001389bu, 0x00000000u}, + {0x4f275395u, 0x0001409fu, 0x00000000u}, + {0x51285415u, 0x000148a3u, 0x00000000u}, + {0x53a994a4u, 0x000000a8u, 0x00000000u}, + {0x55aa9524u, 0x000000acu, 0x00000000u}, + {0x57ab95a5u, 0x000162b0u, 0x00000000u}, + {0x59ac9635u, 0x00016ab4u, 0x00000000u}, + {0x5c2dd6c5u, 0x000174b9u, 0x00000000u}, + {0x5eaf1765u, 0x00017ebeu, 0x00000000u}, + {0x61305805u, 0x000188c3u, 0x00000000u}, + {0x63b198a5u, 0x000192c8u, 0x00000000u}, + {0x6632d945u, 0x00019ccdu, 0x00000000u}, + {0x68b419e5u, 0x0001a6d2u, 0x00000000u}, + {0x01004005u, 0x00000803u, 0x00000000u}, + {0x03014095u, 0x00001007u, 0x00000000u}, + {0x05024115u, 0x0000180bu, 0x00000000u}, + {0x07034195u, 0x0000200fu, 0x00000000u}, + {0x09044215u, 0x00002813u, 0x00000000u}, + {0x0b054295u, 0x00003017u, 0x00000000u}, + {0x0d064315u, 0x0000381bu, 0x00000000u}, + {0x0f074395u, 0x0000401fu, 0x00000000u}, + {0x11084415u, 0x00004823u, 0x00000000u}, + {0x13094495u, 0x00005027u, 0x00000000u}, + {0x150a4515u, 0x0000582bu, 0x00000000u}, + {0x170b4595u, 0x0000602fu, 0x00000000u}, + {0x190c4615u, 0x00006833u, 0x00000000u}, + {0x1b0d4695u, 0x00007037u, 0x00000000u}, + {0x1d0e4715u, 0x0000783bu, 0x00000000u}, + {0x1f0f4795u, 0x0000803fu, 0x00000000u}, + {0x21104816u, 0x01148843u, 0x00000000u}, + {0x239188b6u, 0x01289248u, 0x00000000u}, + {0x2612c955u, 0x00009c4du, 0x00000000u}, + {0x289409e5u, 0x0000a652u, 0x00000000u}, + {0x2b154a85u, 0x0000b057u, 0x00000000u}, + {0x2d968b25u, 0x0000ba5cu, 0x00000000u}, + {0x2f978bb5u, 0x0000c260u, 0x00000000u}, + {0x3218cc46u, 0x019ccc65u, 0x00000000u}, + {0x349a0cf7u, 0x69b0d66au, 0x00000003u}, + {0x379b8db7u, 0x99c8e270u, 0x00000003u}, + {0x3a9d0e77u, 0xc9e0ee76u, 0x00000003u}, + {0x3d9e8f37u, 0xf9f8fa7cu, 0x00000003u}, + {0x40a00ff7u, 0x2a110682u, 0x00000004u}, + {0x4421d0c6u, 0x022d1489u, 0x00000000u}, + {0x47235186u, 0x0245208fu, 0x00000000u}, + {0x4a24d245u, 0x00012c95u, 0x00000000u}, + {0x4c25d2d5u, 0x00013499u, 0x00000000u}, + {0x4e26d355u, 0x00013c9du, 0x00000000u}, + {0x5027d3d5u, 0x000144a1u, 0x00000000u}, + {0x5228d455u, 0x00014ca5u, 0x00000000u}, + {0x5429d4d5u, 0x000154a9u, 0x00000000u}, + {0x562ad555u, 0x00015cadu, 0x00000000u}, + {0x582bd5d5u, 0x000164b1u, 0x00000000u}, + {0x5a2cd655u, 0x00016cb5u, 0x00000000u}, + {0x5cae16e5u, 0x000176bau, 0x00000000u}, + {0x5f2f5785u, 0x000180bfu, 0x00000000u}, + {0x61b09825u, 0x00018ac4u, 0x00000000u}, + {0x6431d8c5u, 0x000194c9u, 0x00000000u}, + {0x6632d956u, 0x033d9ccdu, 0x00000000u}, + {0x68b419f5u, 0x0001a6d2u, 0x00000000u}, + {0x01004004u, 0x00000003u, 0x00000000u}, + {0x03014084u, 0x00000007u, 0x00000000u}, + {0x05024104u, 0x0000000bu, 0x00000000u}, + {0x07034184u, 0x0000000fu, 0x00000000u}, + {0x088401f5u, 0x00002612u, 0x00000000u}, + {0x0a850275u, 0x00002e16u, 0x00000000u}, + {0x0c8602f5u, 0x0000361au, 0x00000000u}, + {0x0e870374u, 0x0000001eu, 0x00000000u}, + {0x108803e4u, 0x00000022u, 0x00000000u}, + {0x12890464u, 0x00000026u, 0x00000000u}, + {0x148a04e4u, 0x0000002au, 0x00000000u}, + {0x160ac555u, 0x00005c2du, 0x00000000u}, + {0x180bc5d5u, 0x00006431u, 0x00000000u}, + {0x1a0cc655u, 0x00006c35u, 0x00000000u}, + {0x1c0dc6d4u, 0x00000039u, 0x00000000u}, + {0x1e0ec744u, 0x0000003du, 0x00000000u}, + {0x200fc7c4u, 0x00000041u, 0x00000000u}, + {0x2210c845u, 0x00008c45u, 0x00000000u}, + {0x2411c8d6u, 0x012c9449u, 0x00000000u}, + {0x26930975u, 0x00009e4eu, 0x00000000u}, + {0x29144a05u, 0x0000a853u, 0x00000000u}, + {0x2b154a95u, 0x0000b057u, 0x00000000u}, + {0x2d968b25u, 0x0000ba5cu, 0x00000000u}, + {0x3017cbc4u, 0x00000061u, 0x00000000u}, + {0x3218cc45u, 0x0000cc65u, 0x00000000u}, + {0x349a0ce6u, 0x01b0d66au, 0x00000000u}, + {0x379b8da6u, 0x01c8e270u, 0x00000000u}, + {0x3a9d0e66u, 0x01e0ee76u, 0x00000000u}, + {0x3d1e4f16u, 0x01f4f87bu, 0x00000000u}, + {0x401fcfc6u, 0x020d0481u, 0x00000000u}, + {0x43215086u, 0x02251087u, 0x00000000u}, + {0x4622d146u, 0x023d1c8du, 0x00000000u}, + {0x49245205u, 0x00012893u, 0x00000000u}, + {0x4ba592a4u, 0x00000098u, 0x00000000u}, + {0x4da69324u, 0x0000009cu, 0x00000000u}, + {0x4f275395u, 0x0001409fu, 0x00000000u}, + {0x51285415u, 0x000148a3u, 0x00000000u}, + {0x53295495u, 0x000150a7u, 0x00000000u}, + {0x552a5514u, 0x000000abu, 0x00000000u}, + {0x572b5584u, 0x000000afu, 0x00000000u}, + {0x592c5604u, 0x000000b3u, 0x00000000u}, + {0x5b2d5685u, 0x000170b7u, 0x00000000u}, + {0x5d2e5715u, 0x000178bbu, 0x00000000u}, + {0x5faf97a5u, 0x000182c0u, 0x00000000u}, + {0x6230d845u, 0x00018cc5u, 0x00000000u}, + {0x6431d8d5u, 0x000194c9u, 0x00000000u}, + {0x66b31965u, 0x00019eceu, 0x00000000u}, + {0x68b419f5u, 0x0001a6d2u, 0x00000000u}, + {0x01004004u, 0x00000003u, 0x00000000u}, + {0x03014084u, 0x00000007u, 0x00000000u}, + {0x048200f5u, 0x0000160au, 0x00000000u}, + {0x06830174u, 0x0000000eu, 0x00000000u}, + {0x088401e4u, 0x00000012u, 0x00000000u}, + {0x0a04c255u, 0x00002c15u, 0x00000000u}, + {0x0c05c2d4u, 0x00000019u, 0x00000000u}, + {0x0e06c344u, 0x0000001du, 0x00000000u}, + {0x1007c3c4u, 0x00000021u, 0x00000000u}, + {0x11888435u, 0x00004a24u, 0x00000000u}, + {0x138984b4u, 0x00000028u, 0x00000000u}, + {0x158a8524u, 0x0000002cu, 0x00000000u}, + {0x170b4595u, 0x0000602fu, 0x00000000u}, + {0x190c4614u, 0x00000033u, 0x00000000u}, + {0x1b0d4684u, 0x00000037u, 0x00000000u}, + {0x1d0e4704u, 0x0000003bu, 0x00000000u}, + {0x1e8f0775u, 0x00007e3eu, 0x00000000u}, + {0x209007f5u, 0x00008642u, 0x00000000u}, + {0x22910875u, 0x00008e46u, 0x00000000u}, + {0x25124904u, 0x0000004bu, 0x00000000u}, + {0x27134985u, 0x0000a04fu, 0x00000000u}, + {0x29144a15u, 0x0000a853u, 0x00000000u}, + {0x2b958aa5u, 0x0000b258u, 0x00000000u}, + {0x2d968b35u, 0x0000ba5cu, 0x00000000u}, + {0x3017cbc4u, 0x00000061u, 0x00000000u}, + {0x3218cc45u, 0x0000cc65u, 0x00000000u}, + {0x349a0ce6u, 0x01b0d66au, 0x00000000u}, + {0x371b4d96u, 0x01c4e06fu, 0x00000000u}, + {0x3a1cce46u, 0x01dcec75u, 0x00000000u}, + {0x3c9e0ef6u, 0x01f0f67au, 0x00000000u}, + {0x3f9f8fa6u, 0x02090280u, 0x00000000u}, + {0x42a11065u, 0x00010e86u, 0x00000000u}, + {0x45225106u, 0x0235188bu, 0x00000000u}, + {0x4823d1c6u, 0x024d2491u, 0x00000000u}, + {0x4aa51275u, 0x00012e96u, 0x00000000u}, + {0x4ca612f4u, 0x0000009au, 0x00000000u}, + {0x4ea71364u, 0x0000009eu, 0x00000000u}, + {0x5027d3d5u, 0x000144a1u, 0x00000000u}, + {0x5228d454u, 0x000000a5u, 0x00000000u}, + {0x5429d4c4u, 0x000000a9u, 0x00000000u}, + {0x562ad544u, 0x000000adu, 0x00000000u}, + {0x57ab95b5u, 0x000162b0u, 0x00000000u}, + {0x59ac9634u, 0x000000b4u, 0x00000000u}, + {0x5bad96a5u, 0x000172b8u, 0x00000000u}, + {0x5dae9735u, 0x00017abcu, 0x00000000u}, + {0x602fd7c5u, 0x000184c1u, 0x00000000u}, + {0x6230d855u, 0x00018cc5u, 0x00000000u}, + {0x64b218e5u, 0x000196cau, 0x00000000u}, + {0x66b31975u, 0x00019eceu, 0x00000000u}, + {0x69345a04u, 0x000000d3u, 0x00000000u}, + {0x01004004u, 0x00000003u, 0x00000000u}, + {0x03014084u, 0x00000007u, 0x00000000u}, + {0x048200f4u, 0x0000000au, 0x00000000u}, + {0x06830164u, 0x0000000eu, 0x00000000u}, + {0x0803c1d4u, 0x00000011u, 0x00000000u}, + {0x0a04c244u, 0x00000015u, 0x00000000u}, + {0x0b8582b4u, 0x00000018u, 0x00000000u}, + {0x0d868324u, 0x0000001cu, 0x00000000u}, + {0x0f074395u, 0x0000401fu, 0x00000000u}, + {0x11084414u, 0x00000023u, 0x00000000u}, + {0x13094484u, 0x00000027u, 0x00000000u}, + {0x148a04f4u, 0x0000002au, 0x00000000u}, + {0x168b0564u, 0x0000002eu, 0x00000000u}, + {0x180bc5d4u, 0x00000031u, 0x00000000u}, + {0x1a0cc644u, 0x00000035u, 0x00000000u}, + {0x1b8d86b5u, 0x00007238u, 0x00000000u}, + {0x1d8e8734u, 0x0000003cu, 0x00000000u}, + {0x1f8f87a4u, 0x00000040u, 0x00000000u}, + {0x21104815u, 0x00008843u, 0x00000000u}, + {0x23114895u, 0x00009047u, 0x00000000u}, + {0x25928924u, 0x0000004cu, 0x00000000u}, + {0x279389a5u, 0x0000a250u, 0x00000000u}, + {0x29948a35u, 0x0000aa54u, 0x00000000u}, + {0x2b958ab5u, 0x0000b258u, 0x00000000u}, + {0x2e16cb44u, 0x0000005du, 0x00000000u}, + {0x3017cbc4u, 0x00000061u, 0x00000000u}, + {0x3218cc45u, 0x0000cc65u, 0x00000000u}, + {0x349a0ce5u, 0x0000d66au, 0x00000000u}, + {0x371b4d86u, 0x01c4e06fu, 0x00000000u}, + {0x399c8e36u, 0x01d8ea74u, 0x00000000u}, + {0x3c9e0ee5u, 0x0000f67au, 0x00000000u}, + {0x3f1f4f86u, 0x0205007fu, 0x00000000u}, + {0x41a09036u, 0x02190a84u, 0x00000000u}, + {0x44a210e5u, 0x0001168au, 0x00000000u}, + {0x47235186u, 0x0245208fu, 0x00000000u}, + {0x49a49235u, 0x00012a94u, 0x00000000u}, + {0x4c25d2c4u, 0x00000099u, 0x00000000u}, + {0x4da69334u, 0x0000009cu, 0x00000000u}, + {0x4fa793a4u, 0x000000a0u, 0x00000000u}, + {0x51285414u, 0x000000a3u, 0x00000000u}, + {0x53295484u, 0x000000a7u, 0x00000000u}, + {0x54aa14f5u, 0x000156aau, 0x00000000u}, + {0x56ab1574u, 0x000000aeu, 0x00000000u}, + {0x58ac15e4u, 0x000000b2u, 0x00000000u}, + {0x5a2cd655u, 0x00016cb5u, 0x00000000u}, + {0x5c2dd6d5u, 0x000174b9u, 0x00000000u}, + {0x5e2ed755u, 0x00017cbdu, 0x00000000u}, + {0x60b017e4u, 0x000000c2u, 0x00000000u}, + {0x62b11865u, 0x00018ec6u, 0x00000000u}, + {0x64b218f5u, 0x000196cau, 0x00000000u}, + {0x66b31975u, 0x00019eceu, 0x00000000u}, + {0x69345a04u, 0x000000d3u, 0x00000000u}, + {0x01004004u, 0x00000003u, 0x00000000u}, + {0x02810074u, 0x00000006u, 0x00000000u}, + {0x048200e4u, 0x0000000au, 0x00000000u}, + {0x0602c154u, 0x0000000du, 0x00000000u}, + {0x078381b4u, 0x00000010u, 0x00000000u}, + {0x09044214u, 0x00000013u, 0x00000000u}, + {0x0b054283u, 0x00000000u, 0x00000000u}, + {0x0c8602e4u, 0x0000001au, 0x00000000u}, + {0x0e06c354u, 0x0000001du, 0x00000000u}, + {0x0f8783b4u, 0x00000020u, 0x00000000u}, + {0x11084414u, 0x00000023u, 0x00000000u}, + {0x13094483u, 0x00000000u, 0x00000000u}, + {0x148a04e4u, 0x0000002au, 0x00000000u}, + {0x160ac554u, 0x0000002du, 0x00000000u}, + {0x178b85b4u, 0x00000030u, 0x00000000u}, + {0x190c4614u, 0x00000033u, 0x00000000u}, + {0x1b0d4684u, 0x00000037u, 0x00000000u}, + {0x1c8e06f4u, 0x0000003au, 0x00000000u}, + {0x1e0ec754u, 0x0000003du, 0x00000000u}, + {0x200fc7c3u, 0x00000000u, 0x00000000u}, + {0x21908825u, 0x00008a44u, 0x00000000u}, + {0x239188b5u, 0x00009248u, 0x00000000u}, + {0x25928935u, 0x00009a4cu, 0x00000000u}, + {0x279389b5u, 0x0000a250u, 0x00000000u}, + {0x2a14ca44u, 0x00000055u, 0x00000000u}, + {0x2c15cac4u, 0x00000059u, 0x00000000u}, + {0x2e16cb45u, 0x0000bc5du, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x3218cc45u, 0x0000cc65u, 0x00000000u}, + {0x349a0ce6u, 0x01b4d66au, 0x00000000u}, + {0x379b8db7u, 0x99c8e270u, 0x00000003u}, + {0x3a9d0e77u, 0xc9e0ee76u, 0x00000003u}, + {0x3d9e8f37u, 0xf9f8fa7cu, 0x00000003u}, + {0x40a00ff7u, 0x2a110682u, 0x00000004u}, + {0x43a190b7u, 0x5a291288u, 0x00000004u}, + {0x46a31177u, 0x8a411e8eu, 0x00000004u}, + {0x49a49235u, 0x00012a94u, 0x00000000u}, + {0x4c25d2c3u, 0x00000000u, 0x00000000u}, + {0x4da69324u, 0x0000009cu, 0x00000000u}, + {0x4f275394u, 0x0000009fu, 0x00000000u}, + {0x50a813f4u, 0x000000a2u, 0x00000000u}, + {0x5228d455u, 0x00014ca5u, 0x00000000u}, + {0x5429d4d4u, 0x000000a9u, 0x00000000u}, + {0x55aa9534u, 0x000000acu, 0x00000000u}, + {0x572b5594u, 0x000000afu, 0x00000000u}, + {0x592c5603u, 0x00000000u, 0x00000000u}, + {0x5aad1664u, 0x000000b6u, 0x00000000u}, + {0x5cae16e4u, 0x000000bau, 0x00000000u}, + {0x5eaf1765u, 0x00017ebeu, 0x00000000u}, + {0x60b017f5u, 0x000186c2u, 0x00000000u}, + {0x62b11875u, 0x00018ec6u, 0x00000000u}, + {0x64b218f5u, 0x000196cau, 0x00000000u}, + {0x67335984u, 0x000000cfu, 0x00000000u}, + {0x69345a04u, 0x000000d3u, 0x00000000u}, + {0x01004004u, 0x00000003u, 0x00000000u}, + {0x02810074u, 0x00000006u, 0x00000000u}, + {0x0401c0d4u, 0x00000009u, 0x00000000u}, + {0x05828134u, 0x0000000cu, 0x00000000u}, + {0x07034194u, 0x0000000fu, 0x00000000u}, + {0x088401f4u, 0x00000012u, 0x00000000u}, + {0x0a850264u, 0x00000016u, 0x00000000u}, + {0x0c05c2d4u, 0x00000019u, 0x00000000u}, + {0x0d868334u, 0x0000001cu, 0x00000000u}, + {0x0f074394u, 0x0000001fu, 0x00000000u}, + {0x108803f4u, 0x00000022u, 0x00000000u}, + {0x1208c454u, 0x00000025u, 0x00000000u}, + {0x1409c4c3u, 0x00000000u, 0x00000000u}, + {0x158a8523u, 0x00000000u, 0x00000000u}, + {0x170b4583u, 0x00000000u, 0x00000000u}, + {0x188c05e3u, 0x00000000u, 0x00000000u}, + {0x1a0cc643u, 0x00000000u, 0x00000000u}, + {0x1b8d86a3u, 0x00000000u, 0x00000000u}, + {0x1d0e4704u, 0x0000003bu, 0x00000000u}, + {0x1e8f0774u, 0x0000003eu, 0x00000000u}, + {0x200fc7d4u, 0x00000041u, 0x00000000u}, + {0x2210c844u, 0x00000045u, 0x00000000u}, + {0x2411c8c4u, 0x00000049u, 0x00000000u}, + {0x2612c945u, 0x00009c4du, 0x00000000u}, + {0x2813c9d4u, 0x00000051u, 0x00000000u}, + {0x2a14ca44u, 0x00000055u, 0x00000000u}, + {0x2c15cac5u, 0x0000b459u, 0x00000000u}, + {0x2e16cb55u, 0x0000bc5du, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x3218cc44u, 0x00000065u, 0x00000000u}, + {0x3419ccc6u, 0x01acd469u, 0x00000000u}, + {0x371b4d85u, 0x0000e26fu, 0x00000000u}, + {0x3a1cce46u, 0x01dcec75u, 0x00000000u}, + {0x3d1e4f06u, 0x01f4f87bu, 0x00000000u}, + {0x401f8fb6u, 0x020d0481u, 0x00000000u}, + {0x43215086u, 0x02251087u, 0x00000000u}, + {0x45a29137u, 0x7a391a8cu, 0x00000004u}, + {0x49245205u, 0x00012893u, 0x00000000u}, + {0x4ba592a3u, 0x00000000u, 0x00000000u}, + {0x4d265303u, 0x00000000u, 0x00000000u}, + {0x4ea71363u, 0x00000000u, 0x00000000u}, + {0x5027d3c3u, 0x00000000u, 0x00000000u}, + {0x51a89423u, 0x00000000u, 0x00000000u}, + {0x53295484u, 0x000000a7u, 0x00000000u}, + {0x54aa14f4u, 0x000000aau, 0x00000000u}, + {0x562ad554u, 0x000000adu, 0x00000000u}, + {0x57ab95b4u, 0x000000b0u, 0x00000000u}, + {0x592c5614u, 0x000000b3u, 0x00000000u}, + {0x5aad1675u, 0x00016eb6u, 0x00000000u}, + {0x5d2e5704u, 0x000000bbu, 0x00000000u}, + {0x5f2f5784u, 0x000000bfu, 0x00000000u}, + {0x61305805u, 0x000188c3u, 0x00000000u}, + {0x63315895u, 0x000190c7u, 0x00000000u}, + {0x65325915u, 0x000198cbu, 0x00000000u}, + {0x67335995u, 0x0001a0cfu, 0x00000000u}, + {0x69345a14u, 0x000000d3u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x02810063u, 0x00000000u, 0x00000000u}, + {0x0401c0c4u, 0x00000009u, 0x00000000u}, + {0x05828133u, 0x00000000u, 0x00000000u}, + {0x07034184u, 0x0000000fu, 0x00000000u}, + {0x088401f4u, 0x00000012u, 0x00000000u}, + {0x0a04c254u, 0x00000015u, 0x00000000u}, + {0x0b8582b4u, 0x00000018u, 0x00000000u}, + {0x0d064314u, 0x0000001bu, 0x00000000u}, + {0x0e870374u, 0x0000001eu, 0x00000000u}, + {0x1007c3d4u, 0x00000021u, 0x00000000u}, + {0x11888434u, 0x00000024u, 0x00000000u}, + {0x13094494u, 0x00000027u, 0x00000000u}, + {0x148a04f4u, 0x0000002au, 0x00000000u}, + {0x160ac554u, 0x0000002du, 0x00000000u}, + {0x178b85b4u, 0x00000030u, 0x00000000u}, + {0x190c4614u, 0x00000033u, 0x00000000u}, + {0x1a8d0674u, 0x00000036u, 0x00000000u}, + {0x1c0dc6d3u, 0x00000000u, 0x00000000u}, + {0x1d8e8724u, 0x0000003cu, 0x00000000u}, + {0x1f0f4794u, 0x0000003fu, 0x00000000u}, + {0x209007f4u, 0x00000042u, 0x00000000u}, + {0x22910864u, 0x00000046u, 0x00000000u}, + {0x249208e4u, 0x0000004au, 0x00000000u}, + {0x2612c955u, 0x00009c4du, 0x00000000u}, + {0x2813c9d5u, 0x0000a451u, 0x00000000u}, + {0x2a14ca55u, 0x0000ac55u, 0x00000000u}, + {0x2c960ae4u, 0x0000005au, 0x00000000u}, + {0x2e16cb55u, 0x0000bc5du, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x31988c34u, 0x00000064u, 0x00000000u}, + {0x3419ccc5u, 0x0000d469u, 0x00000000u}, + {0x369b0d66u, 0x01c0de6eu, 0x00000000u}, + {0x3a1c8e25u, 0x0000ec75u, 0x00000000u}, + {0x3c9e0ee6u, 0x01f0f67au, 0x00000000u}, + {0x3f1f4f96u, 0x0205007fu, 0x00000000u}, + {0x4220d046u, 0x021d0c85u, 0x00000000u}, + {0x45a29105u, 0x00011a8cu, 0x00000000u}, + {0x4823d1c5u, 0x00012491u, 0x00000000u}, + {0x4aa51264u, 0x00000096u, 0x00000000u}, + {0x4c25d2d4u, 0x00000099u, 0x00000000u}, + {0x4da69334u, 0x0000009cu, 0x00000000u}, + {0x4f275394u, 0x0000009fu, 0x00000000u}, + {0x51285403u, 0x00000000u, 0x00000000u}, + {0x5228d454u, 0x000000a5u, 0x00000000u}, + {0x53a994b4u, 0x000000a8u, 0x00000000u}, + {0x552a5514u, 0x000000abu, 0x00000000u}, + {0x56ab1574u, 0x000000aeu, 0x00000000u}, + {0x582bd5d4u, 0x000000b1u, 0x00000000u}, + {0x59ac9634u, 0x000000b4u, 0x00000000u}, + {0x5bad96a4u, 0x000000b8u, 0x00000000u}, + {0x5dae9724u, 0x000000bcu, 0x00000000u}, + {0x5f2f5795u, 0x000180bfu, 0x00000000u}, + {0x61305815u, 0x000188c3u, 0x00000000u}, + {0x63315895u, 0x000190c7u, 0x00000000u}, + {0x65325915u, 0x000198cbu, 0x00000000u}, + {0x67335995u, 0x0001a0cfu, 0x00000000u}, + {0x69345a14u, 0x000000d3u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x02810063u, 0x00000000u, 0x00000000u}, + {0x038180b4u, 0x00000008u, 0x00000000u}, + {0x05828123u, 0x00000000u, 0x00000000u}, + {0x06830174u, 0x0000000eu, 0x00000000u}, + {0x0803c1d4u, 0x00000011u, 0x00000000u}, + {0x09848234u, 0x00000014u, 0x00000000u}, + {0x0b054294u, 0x00000017u, 0x00000000u}, + {0x0c8602f3u, 0x00000000u, 0x00000000u}, + {0x0e06c344u, 0x0000001du, 0x00000000u}, + {0x0f8783b3u, 0x00000000u, 0x00000000u}, + {0x11084404u, 0x00000023u, 0x00000000u}, + {0x12890473u, 0x00000000u, 0x00000000u}, + {0x1409c4c3u, 0x00000000u, 0x00000000u}, + {0x150a4514u, 0x0000002bu, 0x00000000u}, + {0x170b4583u, 0x00000000u, 0x00000000u}, + {0x180bc5d4u, 0x00000031u, 0x00000000u}, + {0x198c8634u, 0x00000034u, 0x00000000u}, + {0x1b0d4694u, 0x00000037u, 0x00000000u}, + {0x1c8e06f4u, 0x0000003au, 0x00000000u}, + {0x1e0ec754u, 0x0000003du, 0x00000000u}, + {0x1f8f87b4u, 0x00000040u, 0x00000000u}, + {0x21104814u, 0x00000043u, 0x00000000u}, + {0x22910875u, 0x00008e46u, 0x00000000u}, + {0x249208f5u, 0x0000964au, 0x00000000u}, + {0x26930975u, 0x00009e4eu, 0x00000000u}, + {0x289409f4u, 0x00000052u, 0x00000000u}, + {0x2a950a64u, 0x00000056u, 0x00000000u}, + {0x2c960ae4u, 0x0000005au, 0x00000000u}, + {0x2e970b64u, 0x0000005eu, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x3218cc44u, 0x00000065u, 0x00000000u}, + {0x3419ccc5u, 0x0000d469u, 0x00000000u}, + {0x369b0d65u, 0x0000de6eu, 0x00000000u}, + {0x391c4e06u, 0x01d4e873u, 0x00000000u}, + {0x3b9d8eb6u, 0x01e8f278u, 0x00000000u}, + {0x3e9f0f65u, 0x0001007fu, 0x00000000u}, + {0x41a09025u, 0x00010c84u, 0x00000000u}, + {0x4421d0d6u, 0x022d1489u, 0x00000000u}, + {0x47235175u, 0x0001208fu, 0x00000000u}, + {0x49a49225u, 0x00012a94u, 0x00000000u}, + {0x4ba592b4u, 0x00000098u, 0x00000000u}, + {0x4d265314u, 0x0000009bu, 0x00000000u}, + {0x4ea71374u, 0x0000009eu, 0x00000000u}, + {0x5027d3d3u, 0x00000000u, 0x00000000u}, + {0x51a89423u, 0x00000000u, 0x00000000u}, + {0x53295483u, 0x00000000u, 0x00000000u}, + {0x54aa14e3u, 0x00000000u, 0x00000000u}, + {0x562ad543u, 0x00000000u, 0x00000000u}, + {0x572b5594u, 0x000000afu, 0x00000000u}, + {0x592c5603u, 0x00000000u, 0x00000000u}, + {0x5a2cd654u, 0x000000b5u, 0x00000000u}, + {0x5c2dd6c4u, 0x000000b9u, 0x00000000u}, + {0x5e2ed744u, 0x000000bdu, 0x00000000u}, + {0x602fd7c4u, 0x000000c1u, 0x00000000u}, + {0x61b09835u, 0x00018ac4u, 0x00000000u}, + {0x63b198b4u, 0x000000c8u, 0x00000000u}, + {0x65b29924u, 0x000000ccu, 0x00000000u}, + {0x67b399a4u, 0x000000d0u, 0x00000000u}, + {0x69345a14u, 0x000000d3u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x02810063u, 0x00000000u, 0x00000000u}, + {0x0401c0c3u, 0x00000000u, 0x00000000u}, + {0x05024114u, 0x0000000bu, 0x00000000u}, + {0x06830174u, 0x0000000eu, 0x00000000u}, + {0x0803c1d3u, 0x00000000u, 0x00000000u}, + {0x09848223u, 0x00000000u, 0x00000000u}, + {0x0b054283u, 0x00000000u, 0x00000000u}, + {0x0c05c2d4u, 0x00000019u, 0x00000000u}, + {0x0d868334u, 0x0000001cu, 0x00000000u}, + {0x0f074393u, 0x00000000u, 0x00000000u}, + {0x108803e3u, 0x00000000u, 0x00000000u}, + {0x1208c443u, 0x00000000u, 0x00000000u}, + {0x13094494u, 0x00000027u, 0x00000000u}, + {0x148a04f4u, 0x0000002au, 0x00000000u}, + {0x160ac554u, 0x0000002du, 0x00000000u}, + {0x178b85b4u, 0x00000030u, 0x00000000u}, + {0x190c4613u, 0x00000000u, 0x00000000u}, + {0x1a8d0663u, 0x00000000u, 0x00000000u}, + {0x1c0dc6c3u, 0x00000000u, 0x00000000u}, + {0x1d0e4714u, 0x0000003bu, 0x00000000u}, + {0x1e8f0774u, 0x0000003eu, 0x00000000u}, + {0x200fc7d3u, 0x00000000u, 0x00000000u}, + {0x21908824u, 0x00000044u, 0x00000000u}, + {0x23114894u, 0x00000047u, 0x00000000u}, + {0x25124904u, 0x0000004bu, 0x00000000u}, + {0x27134984u, 0x0000004fu, 0x00000000u}, + {0x289409f4u, 0x00000052u, 0x00000000u}, + {0x2a950a64u, 0x00000056u, 0x00000000u}, + {0x2c960ae4u, 0x0000005au, 0x00000000u}, + {0x2e970b64u, 0x0000005eu, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x31988c34u, 0x00000064u, 0x00000000u}, + {0x33998ca5u, 0x0000d268u, 0x00000000u}, + {0x361acd46u, 0x01bcdc6du, 0x00000000u}, + {0x389c0df5u, 0x0000e872u, 0x00000000u}, + {0x3b9d8ea5u, 0x0000f278u, 0x00000000u}, + {0x3e1ecf46u, 0x01fcfc7du, 0x00000000u}, + {0x41205005u, 0x00010883u, 0x00000000u}, + {0x43a190a6u, 0x02291288u, 0x00000000u}, + {0x4622d156u, 0x023d1c8du, 0x00000000u}, + {0x49245205u, 0x00012893u, 0x00000000u}, + {0x4b255294u, 0x00000097u, 0x00000000u}, + {0x4ca612f3u, 0x00000000u, 0x00000000u}, + {0x4e26d343u, 0x00000000u, 0x00000000u}, + {0x4fa793a3u, 0x00000000u, 0x00000000u}, + {0x50a813f4u, 0x000000a2u, 0x00000000u}, + {0x5228d454u, 0x000000a5u, 0x00000000u}, + {0x53a994b4u, 0x000000a8u, 0x00000000u}, + {0x552a5514u, 0x000000abu, 0x00000000u}, + {0x56ab1573u, 0x00000000u, 0x00000000u}, + {0x582bd5c3u, 0x00000000u, 0x00000000u}, + {0x59ac9623u, 0x00000000u, 0x00000000u}, + {0x5aad1674u, 0x000000b6u, 0x00000000u}, + {0x5cae16e4u, 0x000000bau, 0x00000000u}, + {0x5e2ed754u, 0x000000bdu, 0x00000000u}, + {0x602fd7c4u, 0x000000c1u, 0x00000000u}, + {0x6230d844u, 0x000000c5u, 0x00000000u}, + {0x63b198b4u, 0x000000c8u, 0x00000000u}, + {0x65b29924u, 0x000000ccu, 0x00000000u}, + {0x67b399a4u, 0x000000d0u, 0x00000000u}, + {0x69345a14u, 0x000000d3u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x02810063u, 0x00000000u, 0x00000000u}, + {0x0401c0c3u, 0x00000000u, 0x00000000u}, + {0x05024113u, 0x00000000u, 0x00000000u}, + {0x06830163u, 0x00000000u, 0x00000000u}, + {0x078381b4u, 0x00000010u, 0x00000000u}, + {0x09044214u, 0x00000013u, 0x00000000u}, + {0x0a850273u, 0x00000000u, 0x00000000u}, + {0x0c05c2c3u, 0x00000000u, 0x00000000u}, + {0x0d064314u, 0x0000001bu, 0x00000000u}, + {0x0e870373u, 0x00000000u, 0x00000000u}, + {0x1007c3c3u, 0x00000000u, 0x00000000u}, + {0x11888423u, 0x00000000u, 0x00000000u}, + {0x12890474u, 0x00000026u, 0x00000000u}, + {0x1409c4d4u, 0x00000029u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x168b0574u, 0x0000002eu, 0x00000000u}, + {0x180bc5d3u, 0x00000000u, 0x00000000u}, + {0x198c8623u, 0x00000000u, 0x00000000u}, + {0x1a8d0674u, 0x00000036u, 0x00000000u}, + {0x1c0dc6d4u, 0x00000039u, 0x00000000u}, + {0x1d8e8734u, 0x0000003cu, 0x00000000u}, + {0x1f0f4793u, 0x00000000u, 0x00000000u}, + {0x200fc7d4u, 0x00000041u, 0x00000000u}, + {0x2210c844u, 0x00000045u, 0x00000000u}, + {0x239188b5u, 0x00009248u, 0x00000000u}, + {0x25928934u, 0x0000004cu, 0x00000000u}, + {0x27134994u, 0x0000004fu, 0x00000000u}, + {0x29144a04u, 0x00000053u, 0x00000000u}, + {0x2a950a74u, 0x00000056u, 0x00000000u}, + {0x2c960ae4u, 0x0000005au, 0x00000000u}, + {0x2e16cb55u, 0x0000bc5du, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x31988c34u, 0x00000064u, 0x00000000u}, + {0x33194c95u, 0x0000d067u, 0x00000000u}, + {0x361acd45u, 0x0000dc6du, 0x00000000u}, + {0x389c0de5u, 0x0000e672u, 0x00000000u}, + {0x3b9d4e84u, 0x00000078u, 0x00000000u}, + {0x3d9e8f26u, 0x01f8fa7cu, 0x00000000u}, + {0x40a00fe5u, 0x00010682u, 0x00000000u}, + {0x43215085u, 0x00011087u, 0x00000000u}, + {0x45225116u, 0x0235188bu, 0x00000000u}, + {0x4823d1c5u, 0x00012491u, 0x00000000u}, + {0x4aa51263u, 0x00000000u, 0x00000000u}, + {0x4c25d2c3u, 0x00000000u, 0x00000000u}, + {0x4da69323u, 0x00000000u, 0x00000000u}, + {0x4ea71373u, 0x00000000u, 0x00000000u}, + {0x5027d3c3u, 0x00000000u, 0x00000000u}, + {0x51a89423u, 0x00000000u, 0x00000000u}, + {0x52a91474u, 0x000000a6u, 0x00000000u}, + {0x5429d4d4u, 0x000000a9u, 0x00000000u}, + {0x55aa9533u, 0x00000000u, 0x00000000u}, + {0x572b5583u, 0x00000000u, 0x00000000u}, + {0x58ac15e3u, 0x00000000u, 0x00000000u}, + {0x59ac9634u, 0x000000b4u, 0x00000000u}, + {0x5b2d5694u, 0x000000b7u, 0x00000000u}, + {0x5cae16f5u, 0x000176bau, 0x00000000u}, + {0x5eaf1774u, 0x000000beu, 0x00000000u}, + {0x602fd7d5u, 0x000184c1u, 0x00000000u}, + {0x6230d855u, 0x00018cc5u, 0x00000000u}, + {0x6431d8d4u, 0x000000c9u, 0x00000000u}, + {0x65b29935u, 0x00019accu, 0x00000000u}, + {0x67b399b4u, 0x000000d0u, 0x00000000u}, + {0x69345a14u, 0x000000d3u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x02810063u, 0x00000000u, 0x00000000u}, + {0x038180b3u, 0x00000000u, 0x00000000u}, + {0x05024103u, 0x00000000u, 0x00000000u}, + {0x0602c154u, 0x0000000du, 0x00000000u}, + {0x078381b3u, 0x00000000u, 0x00000000u}, + {0x09044203u, 0x00000000u, 0x00000000u}, + {0x0a04c254u, 0x00000015u, 0x00000000u}, + {0x0b8582b3u, 0x00000000u, 0x00000000u}, + {0x0d064303u, 0x00000000u, 0x00000000u}, + {0x0e06c354u, 0x0000001du, 0x00000000u}, + {0x0f8783b3u, 0x00000000u, 0x00000000u}, + {0x11084403u, 0x00000000u, 0x00000000u}, + {0x1208c453u, 0x00000000u, 0x00000000u}, + {0x138984a3u, 0x00000000u, 0x00000000u}, + {0x150a4503u, 0x00000000u, 0x00000000u}, + {0x160ac554u, 0x0000002du, 0x00000000u}, + {0x178b85b3u, 0x00000000u, 0x00000000u}, + {0x188c05f4u, 0x00000032u, 0x00000000u}, + {0x1a0cc653u, 0x00000000u, 0x00000000u}, + {0x1b8d86a3u, 0x00000000u, 0x00000000u}, + {0x1c8e06f4u, 0x0000003au, 0x00000000u}, + {0x1e0ec753u, 0x00000000u, 0x00000000u}, + {0x1f8f87a3u, 0x00000000u, 0x00000000u}, + {0x209007f4u, 0x00000042u, 0x00000000u}, + {0x22910864u, 0x00000046u, 0x00000000u}, + {0x2411c8d4u, 0x00000049u, 0x00000000u}, + {0x25928934u, 0x0000004cu, 0x00000000u}, + {0x279389a4u, 0x00000050u, 0x00000000u}, + {0x29144a14u, 0x00000053u, 0x00000000u}, + {0x2b154a84u, 0x00000057u, 0x00000000u}, + {0x2c960af4u, 0x0000005au, 0x00000000u}, + {0x2e970b63u, 0x00000000u, 0x00000000u}, + {0x3017cbc4u, 0x00000061u, 0x00000000u}, + {0x31988c34u, 0x00000064u, 0x00000000u}, + {0x33194c96u, 0x01a4d067u, 0x00000000u}, + {0x359a8d35u, 0x0000da6cu, 0x00000000u}, + {0x381bcdc6u, 0x01cce471u, 0x00000000u}, + {0x3a9d0e75u, 0x0000ee76u, 0x00000000u}, + {0x3d1e4f06u, 0x01f4f87bu, 0x00000000u}, + {0x3f9f8fb5u, 0x00010280u, 0x00000000u}, + {0x42a11065u, 0x00010e86u, 0x00000000u}, + {0x44a210f5u, 0x0001168au, 0x00000000u}, + {0x47235185u, 0x00012290u, 0x00000000u}, + {0x49a49235u, 0x00012a94u, 0x00000000u}, + {0x4ba592b3u, 0x00000000u, 0x00000000u}, + {0x4ca612f4u, 0x0000009au, 0x00000000u}, + {0x4e26d354u, 0x0000009du, 0x00000000u}, + {0x4fa793b3u, 0x00000000u, 0x00000000u}, + {0x51285403u, 0x00000000u, 0x00000000u}, + {0x5228d453u, 0x00000000u, 0x00000000u}, + {0x53a994a3u, 0x00000000u, 0x00000000u}, + {0x54aa14f4u, 0x000000aau, 0x00000000u}, + {0x562ad553u, 0x00000000u, 0x00000000u}, + {0x57ab95a3u, 0x00000000u, 0x00000000u}, + {0x592c5603u, 0x00000000u, 0x00000000u}, + {0x5a2cd654u, 0x000000b5u, 0x00000000u}, + {0x5bad96b4u, 0x000000b8u, 0x00000000u}, + {0x5d2e5714u, 0x000000bbu, 0x00000000u}, + {0x5f2f5784u, 0x000000bfu, 0x00000000u}, + {0x61305803u, 0x00000000u, 0x00000000u}, + {0x62b11863u, 0x00000000u, 0x00000000u}, + {0x6431d8c4u, 0x000000c9u, 0x00000000u}, + {0x6632d944u, 0x000000cdu, 0x00000000u}, + {0x67b399b4u, 0x000000d0u, 0x00000000u}, + {0x69345a14u, 0x000000d3u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c053u, 0x00000000u, 0x00000000u}, + {0x038180a3u, 0x00000000u, 0x00000000u}, + {0x048200f4u, 0x0000000au, 0x00000000u}, + {0x0602c153u, 0x00000000u, 0x00000000u}, + {0x078381a3u, 0x00000000u, 0x00000000u}, + {0x088401f4u, 0x00000012u, 0x00000000u}, + {0x0a04c253u, 0x00000000u, 0x00000000u}, + {0x0b8582a3u, 0x00000000u, 0x00000000u}, + {0x0c8602f3u, 0x00000000u, 0x00000000u}, + {0x0e06c343u, 0x00000000u, 0x00000000u}, + {0x0f074393u, 0x00000000u, 0x00000000u}, + {0x108803e3u, 0x00000000u, 0x00000000u}, + {0x11888433u, 0x00000000u, 0x00000000u}, + {0x13094483u, 0x00000000u, 0x00000000u}, + {0x1409c4d4u, 0x00000029u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x170b4583u, 0x00000000u, 0x00000000u}, + {0x180bc5d4u, 0x00000031u, 0x00000000u}, + {0x198c8633u, 0x00000000u, 0x00000000u}, + {0x1a8d0673u, 0x00000000u, 0x00000000u}, + {0x1c0dc6c3u, 0x00000000u, 0x00000000u}, + {0x1d0e4714u, 0x0000003bu, 0x00000000u}, + {0x1e8f0773u, 0x00000000u, 0x00000000u}, + {0x200fc7c3u, 0x00000000u, 0x00000000u}, + {0x21104814u, 0x00000043u, 0x00000000u}, + {0x22910874u, 0x00000046u, 0x00000000u}, + {0x249208e4u, 0x0000004au, 0x00000000u}, + {0x2612c954u, 0x0000004du, 0x00000000u}, + {0x279389b4u, 0x00000050u, 0x00000000u}, + {0x29948a24u, 0x00000054u, 0x00000000u}, + {0x2b154a94u, 0x00000057u, 0x00000000u}, + {0x2c960af4u, 0x0000005au, 0x00000000u}, + {0x2e970b64u, 0x0000005eu, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x31988c34u, 0x00000064u, 0x00000000u}, + {0x33194c95u, 0x0000d067u, 0x00000000u}, + {0x351a4d16u, 0x01b4d86bu, 0x00000000u}, + {0x379b8db6u, 0x01c8e270u, 0x00000000u}, + {0x3a9d0e65u, 0x0000ee76u, 0x00000000u}, + {0x3c9e0ef6u, 0x01f0f67au, 0x00000000u}, + {0x3f1f4f96u, 0x0205007fu, 0x00000000u}, + {0x4220d044u, 0x00000085u, 0x00000000u}, + {0x4421d0c5u, 0x00011489u, 0x00000000u}, + {0x4622d156u, 0x023d1c8du, 0x00000000u}, + {0x49245205u, 0x00012893u, 0x00000000u}, + {0x4b255293u, 0x00000000u, 0x00000000u}, + {0x4ca612e3u, 0x00000000u, 0x00000000u}, + {0x4da69334u, 0x0000009cu, 0x00000000u}, + {0x4f275393u, 0x00000000u, 0x00000000u}, + {0x5027d3d3u, 0x00000000u, 0x00000000u}, + {0x51a89423u, 0x00000000u, 0x00000000u}, + {0x52a91474u, 0x000000a6u, 0x00000000u}, + {0x5429d4d3u, 0x00000000u, 0x00000000u}, + {0x552a5514u, 0x000000abu, 0x00000000u}, + {0x56ab1573u, 0x00000000u, 0x00000000u}, + {0x582bd5c3u, 0x00000000u, 0x00000000u}, + {0x592c5613u, 0x00000000u, 0x00000000u}, + {0x5aad1664u, 0x000000b6u, 0x00000000u}, + {0x5c2dd6d4u, 0x000000b9u, 0x00000000u}, + {0x5dae9734u, 0x000000bcu, 0x00000000u}, + {0x5f2f5794u, 0x000000bfu, 0x00000000u}, + {0x61305804u, 0x000000c3u, 0x00000000u}, + {0x62b11874u, 0x000000c6u, 0x00000000u}, + {0x64b218e4u, 0x000000cau, 0x00000000u}, + {0x6632d954u, 0x000000cdu, 0x00000000u}, + {0x6833d9c3u, 0x00000000u, 0x00000000u}, + {0x69b49a23u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c053u, 0x00000000u, 0x00000000u}, + {0x038180a3u, 0x00000000u, 0x00000000u}, + {0x048200f3u, 0x00000000u, 0x00000000u}, + {0x0602c143u, 0x00000000u, 0x00000000u}, + {0x07034194u, 0x0000000fu, 0x00000000u}, + {0x088401f3u, 0x00000000u, 0x00000000u}, + {0x09848233u, 0x00000000u, 0x00000000u}, + {0x0b054283u, 0x00000000u, 0x00000000u}, + {0x0c05c2d3u, 0x00000000u, 0x00000000u}, + {0x0d868323u, 0x00000000u, 0x00000000u}, + {0x0e870374u, 0x0000001eu, 0x00000000u}, + {0x1007c3d3u, 0x00000000u, 0x00000000u}, + {0x11084413u, 0x00000000u, 0x00000000u}, + {0x12890463u, 0x00000000u, 0x00000000u}, + {0x138984b3u, 0x00000000u, 0x00000000u}, + {0x150a4503u, 0x00000000u, 0x00000000u}, + {0x160ac554u, 0x0000002du, 0x00000000u}, + {0x178b85b3u, 0x00000000u, 0x00000000u}, + {0x188c05f3u, 0x00000000u, 0x00000000u}, + {0x1a0cc643u, 0x00000000u, 0x00000000u}, + {0x1b0d4693u, 0x00000000u, 0x00000000u}, + {0x1c8e06e3u, 0x00000000u, 0x00000000u}, + {0x1d8e8734u, 0x0000003cu, 0x00000000u}, + {0x1f0f4793u, 0x00000000u, 0x00000000u}, + {0x200fc7d3u, 0x00000000u, 0x00000000u}, + {0x21908824u, 0x00000044u, 0x00000000u}, + {0x23114894u, 0x00000047u, 0x00000000u}, + {0x249208f4u, 0x0000004au, 0x00000000u}, + {0x26930964u, 0x0000004eu, 0x00000000u}, + {0x2813c9d4u, 0x00000051u, 0x00000000u}, + {0x29948a34u, 0x00000054u, 0x00000000u}, + {0x2b154a94u, 0x00000057u, 0x00000000u}, + {0x2d164b04u, 0x0000005bu, 0x00000000u}, + {0x2e970b74u, 0x0000005eu, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x31988c33u, 0x00000000u, 0x00000000u}, + {0x33194c85u, 0x0000d067u, 0x00000000u}, + {0x351a4d15u, 0x0000d86bu, 0x00000000u}, + {0x379b8da5u, 0x0000e270u, 0x00000000u}, + {0x399c8e36u, 0x01d8ea74u, 0x00000000u}, + {0x3c1dced5u, 0x0000f479u, 0x00000000u}, + {0x3e9f0f65u, 0x0000fe7eu, 0x00000000u}, + {0x41205005u, 0x00010883u, 0x00000000u}, + {0x43a190a5u, 0x00011288u, 0x00000000u}, + {0x45a29136u, 0x02391a8cu, 0x00000000u}, + {0x4823d1d5u, 0x00012491u, 0x00000000u}, + {0x4a24d254u, 0x00000095u, 0x00000000u}, + {0x4ba592b4u, 0x00000098u, 0x00000000u}, + {0x4d265313u, 0x00000000u, 0x00000000u}, + {0x4ea71363u, 0x00000000u, 0x00000000u}, + {0x4fa793b3u, 0x00000000u, 0x00000000u}, + {0x51285403u, 0x00000000u, 0x00000000u}, + {0x5228d453u, 0x00000000u, 0x00000000u}, + {0x53295494u, 0x000000a7u, 0x00000000u}, + {0x54aa14f3u, 0x00000000u, 0x00000000u}, + {0x562ad543u, 0x00000000u, 0x00000000u}, + {0x572b5593u, 0x00000000u, 0x00000000u}, + {0x58ac15e3u, 0x00000000u, 0x00000000u}, + {0x59ac9633u, 0x00000000u, 0x00000000u}, + {0x5aad1674u, 0x000000b6u, 0x00000000u}, + {0x5cae16e4u, 0x000000bau, 0x00000000u}, + {0x5e2ed754u, 0x000000bdu, 0x00000000u}, + {0x602fd7c3u, 0x00000000u, 0x00000000u}, + {0x61b09824u, 0x000000c4u, 0x00000000u}, + {0x63315894u, 0x000000c7u, 0x00000000u}, + {0x64b218f4u, 0x000000cau, 0x00000000u}, + {0x6632d954u, 0x000000cdu, 0x00000000u}, + {0x6833d9c3u, 0x00000000u, 0x00000000u}, + {0x69b49a23u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c054u, 0x00000005u, 0x00000000u}, + {0x038180b3u, 0x00000000u, 0x00000000u}, + {0x048200f3u, 0x00000000u, 0x00000000u}, + {0x0602c143u, 0x00000000u, 0x00000000u}, + {0x07034193u, 0x00000000u, 0x00000000u}, + {0x088401e3u, 0x00000000u, 0x00000000u}, + {0x09848233u, 0x00000000u, 0x00000000u}, + {0x0a850273u, 0x00000000u, 0x00000000u}, + {0x0c05c2c3u, 0x00000000u, 0x00000000u}, + {0x0d064313u, 0x00000000u, 0x00000000u}, + {0x0e870363u, 0x00000000u, 0x00000000u}, + {0x0f8783b3u, 0x00000000u, 0x00000000u}, + {0x108803f3u, 0x00000000u, 0x00000000u}, + {0x1208c443u, 0x00000000u, 0x00000000u}, + {0x13094493u, 0x00000000u, 0x00000000u}, + {0x148a04e3u, 0x00000000u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x168b0573u, 0x00000000u, 0x00000000u}, + {0x180bc5c3u, 0x00000000u, 0x00000000u}, + {0x190c4613u, 0x00000000u, 0x00000000u}, + {0x1a8d0663u, 0x00000000u, 0x00000000u}, + {0x1b8d86b3u, 0x00000000u, 0x00000000u}, + {0x1c8e06f4u, 0x0000003au, 0x00000000u}, + {0x1e0ec753u, 0x00000000u, 0x00000000u}, + {0x1f8f87a3u, 0x00000000u, 0x00000000u}, + {0x209007f3u, 0x00000000u, 0x00000000u}, + {0x2210c843u, 0x00000000u, 0x00000000u}, + {0x239188a3u, 0x00000000u, 0x00000000u}, + {0x25124904u, 0x0000004bu, 0x00000000u}, + {0x26930974u, 0x0000004eu, 0x00000000u}, + {0x2813c9d4u, 0x00000051u, 0x00000000u}, + {0x29948a34u, 0x00000054u, 0x00000000u}, + {0x2b958aa3u, 0x00000000u, 0x00000000u}, + {0x2d164b03u, 0x00000000u, 0x00000000u}, + {0x2e970b64u, 0x0000005eu, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x31988c33u, 0x00000000u, 0x00000000u}, + {0x32990c75u, 0x0000ce66u, 0x00000000u}, + {0x351a4d05u, 0x0000d86bu, 0x00000000u}, + {0x371b4d96u, 0x01c4e06fu, 0x00000000u}, + {0x3a1cce44u, 0x00000075u, 0x00000000u}, + {0x3c1dcec5u, 0x0000f479u, 0x00000000u}, + {0x3e9f0f64u, 0x0000007eu, 0x00000000u}, + {0x40a00fe5u, 0x00010682u, 0x00000000u}, + {0x43215084u, 0x00000087u, 0x00000000u}, + {0x45225105u, 0x0001188bu, 0x00000000u}, + {0x47a391a4u, 0x00000090u, 0x00000000u}, + {0x49a49224u, 0x00000094u, 0x00000000u}, + {0x4b255294u, 0x00000097u, 0x00000000u}, + {0x4ca612f3u, 0x00000000u, 0x00000000u}, + {0x4da69334u, 0x0000009cu, 0x00000000u}, + {0x4f275393u, 0x00000000u, 0x00000000u}, + {0x5027d3d3u, 0x00000000u, 0x00000000u}, + {0x51a89423u, 0x00000000u, 0x00000000u}, + {0x52a91473u, 0x00000000u, 0x00000000u}, + {0x5429d4c3u, 0x00000000u, 0x00000000u}, + {0x552a5513u, 0x00000000u, 0x00000000u}, + {0x56ab1563u, 0x00000000u, 0x00000000u}, + {0x57ab95b3u, 0x00000000u, 0x00000000u}, + {0x58ac15f3u, 0x00000000u, 0x00000000u}, + {0x5a2cd643u, 0x00000000u, 0x00000000u}, + {0x5b2d5694u, 0x000000b7u, 0x00000000u}, + {0x5cae16f4u, 0x000000bau, 0x00000000u}, + {0x5eaf1763u, 0x00000000u, 0x00000000u}, + {0x602fd7c3u, 0x00000000u, 0x00000000u}, + {0x61b09824u, 0x000000c4u, 0x00000000u}, + {0x63315894u, 0x000000c7u, 0x00000000u}, + {0x64b218f4u, 0x000000cau, 0x00000000u}, + {0x6632d954u, 0x000000cdu, 0x00000000u}, + {0x6833d9c3u, 0x00000000u, 0x00000000u}, + {0x69b49a23u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c054u, 0x00000005u, 0x00000000u}, + {0x038180b3u, 0x00000000u, 0x00000000u}, + {0x048200f3u, 0x00000000u, 0x00000000u}, + {0x05828133u, 0x00000000u, 0x00000000u}, + {0x07034183u, 0x00000000u, 0x00000000u}, + {0x0803c1d3u, 0x00000000u, 0x00000000u}, + {0x09044213u, 0x00000000u, 0x00000000u}, + {0x0a850263u, 0x00000000u, 0x00000000u}, + {0x0b8582b3u, 0x00000000u, 0x00000000u}, + {0x0d064303u, 0x00000000u, 0x00000000u}, + {0x0e06c353u, 0x00000000u, 0x00000000u}, + {0x0f074393u, 0x00000000u, 0x00000000u}, + {0x000803e2u, 0x00000000u, 0x00000000u}, + {0x11888423u, 0x00000000u, 0x00000000u}, + {0x12890473u, 0x00000000u, 0x00000000u}, + {0x138984b4u, 0x00000028u, 0x00000000u}, + {0x150a4513u, 0x00000000u, 0x00000000u}, + {0x160ac554u, 0x0000002du, 0x00000000u}, + {0x178b85b3u, 0x00000000u, 0x00000000u}, + {0x188c05f3u, 0x00000000u, 0x00000000u}, + {0x198c8633u, 0x00000000u, 0x00000000u}, + {0x1b0d4683u, 0x00000000u, 0x00000000u}, + {0x1c0dc6d3u, 0x00000000u, 0x00000000u}, + {0x1d0e4713u, 0x00000000u, 0x00000000u}, + {0x1e8f0763u, 0x00000000u, 0x00000000u}, + {0x1f8f87b3u, 0x00000000u, 0x00000000u}, + {0x209007f4u, 0x00000042u, 0x00000000u}, + {0x2210c854u, 0x00000045u, 0x00000000u}, + {0x239188b4u, 0x00000048u, 0x00000000u}, + {0x25124914u, 0x0000004bu, 0x00000000u}, + {0x26930974u, 0x0000004eu, 0x00000000u}, + {0x2813c9d4u, 0x00000051u, 0x00000000u}, + {0x2a14ca44u, 0x00000055u, 0x00000000u}, + {0x2b958ab4u, 0x00000058u, 0x00000000u}, + {0x2d164b14u, 0x0000005bu, 0x00000000u}, + {0x2e970b74u, 0x0000005eu, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x31988c33u, 0x00000000u, 0x00000000u}, + {0x32990c75u, 0x0000ce66u, 0x00000000u}, + {0x349a0cf6u, 0x01b0d66au, 0x00000000u}, + {0x371b4d95u, 0x0000e06fu, 0x00000000u}, + {0x391c4e16u, 0x01d4e873u, 0x00000000u}, + {0x3b9d8eb5u, 0x0000f278u, 0x00000000u}, + {0x3d9e8f35u, 0x0000fa7cu, 0x00000000u}, + {0x401fcfc5u, 0x00010481u, 0x00000000u}, + {0x42a11064u, 0x00000086u, 0x00000000u}, + {0x44a210e4u, 0x0000008au, 0x00000000u}, + {0x46a31165u, 0x00011e8eu, 0x00000000u}, + {0x48a411f6u, 0x02512692u, 0x00000000u}, + {0x4b255293u, 0x00000000u, 0x00000000u}, + {0x4c25d2d3u, 0x00000000u, 0x00000000u}, + {0x4da69323u, 0x00000000u, 0x00000000u}, + {0x4ea71373u, 0x00000000u, 0x00000000u}, + {0x4fa793b3u, 0x00000000u, 0x00000000u}, + {0x51285403u, 0x00000000u, 0x00000000u}, + {0x5228d453u, 0x00000000u, 0x00000000u}, + {0x53295493u, 0x00000000u, 0x00000000u}, + {0x54aa14e3u, 0x00000000u, 0x00000000u}, + {0x55aa9533u, 0x00000000u, 0x00000000u}, + {0x572b5583u, 0x00000000u, 0x00000000u}, + {0x582bd5d3u, 0x00000000u, 0x00000000u}, + {0x592c5613u, 0x00000000u, 0x00000000u}, + {0x5a2cd654u, 0x000000b5u, 0x00000000u}, + {0x5bad96b4u, 0x000000b8u, 0x00000000u}, + {0x5d2e5714u, 0x000000bbu, 0x00000000u}, + {0x5eaf1774u, 0x000000beu, 0x00000000u}, + {0x602fd7d4u, 0x000000c1u, 0x00000000u}, + {0x6230d843u, 0x00000000u, 0x00000000u}, + {0x63b198a3u, 0x00000000u, 0x00000000u}, + {0x65325903u, 0x00000000u, 0x00000000u}, + {0x66b31963u, 0x00000000u, 0x00000000u}, + {0x6833d9c3u, 0x00000000u, 0x00000000u}, + {0x69b49a23u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c053u, 0x00000000u, 0x00000000u}, + {0x000180a2u, 0x00000000u, 0x00000000u}, + {0x048200e3u, 0x00000000u, 0x00000000u}, + {0x05828133u, 0x00000000u, 0x00000000u}, + {0x06830173u, 0x00000000u, 0x00000000u}, + {0x0803c1c3u, 0x00000000u, 0x00000000u}, + {0x09044213u, 0x00000000u, 0x00000000u}, + {0x0a04c253u, 0x00000000u, 0x00000000u}, + {0x0b054293u, 0x00000000u, 0x00000000u}, + {0x0c8602e3u, 0x00000000u, 0x00000000u}, + {0x0d868333u, 0x00000000u, 0x00000000u}, + {0x0f074383u, 0x00000000u, 0x00000000u}, + {0x1007c3d3u, 0x00000000u, 0x00000000u}, + {0x11084413u, 0x00000000u, 0x00000000u}, + {0x1208c453u, 0x00000000u, 0x00000000u}, + {0x13094494u, 0x00000027u, 0x00000000u}, + {0x148a04f3u, 0x00000000u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x000b4582u, 0x00000000u, 0x00000000u}, + {0x180bc5c3u, 0x00000000u, 0x00000000u}, + {0x190c4613u, 0x00000000u, 0x00000000u}, + {0x1a0cc653u, 0x00000000u, 0x00000000u}, + {0x1b8d86a3u, 0x00000000u, 0x00000000u}, + {0x1c8e06f3u, 0x00000000u, 0x00000000u}, + {0x1d8e8733u, 0x00000000u, 0x00000000u}, + {0x1e8f0773u, 0x00000000u, 0x00000000u}, + {0x200fc7c3u, 0x00000000u, 0x00000000u}, + {0x21104814u, 0x00000043u, 0x00000000u}, + {0x22910874u, 0x00000046u, 0x00000000u}, + {0x2411c8d4u, 0x00000049u, 0x00000000u}, + {0x25928934u, 0x0000004cu, 0x00000000u}, + {0x27134993u, 0x00000000u, 0x00000000u}, + {0x289409e4u, 0x00000052u, 0x00000000u}, + {0x2a14ca54u, 0x00000055u, 0x00000000u}, + {0x2b958ab3u, 0x00000000u, 0x00000000u}, + {0x2d164b04u, 0x0000005bu, 0x00000000u}, + {0x2e970b74u, 0x0000005eu, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x31988c33u, 0x00000000u, 0x00000000u}, + {0x32990c75u, 0x0000ce66u, 0x00000000u}, + {0x349a0cf5u, 0x0000d66au, 0x00000000u}, + {0x369b0d75u, 0x0000de6eu, 0x00000000u}, + {0x391c4e04u, 0x00000073u, 0x00000000u}, + {0x3b1d4e85u, 0x0000f077u, 0x00000000u}, + {0x3d9e8f25u, 0x0000fa7cu, 0x00000000u}, + {0x3f9f8fb5u, 0x00010280u, 0x00000000u}, + {0x41a09035u, 0x00010a84u, 0x00000000u}, + {0x4421d0c4u, 0x00000089u, 0x00000000u}, + {0x4622d145u, 0x00011c8du, 0x00000000u}, + {0x4823d1d5u, 0x00012491u, 0x00000000u}, + {0x4a24d254u, 0x00000095u, 0x00000000u}, + {0x4ba592b3u, 0x00000000u, 0x00000000u}, + {0x4d265303u, 0x00000000u, 0x00000000u}, + {0x4e26d353u, 0x00000000u, 0x00000000u}, + {0x4f275393u, 0x00000000u, 0x00000000u}, + {0x5027d3d4u, 0x000000a1u, 0x00000000u}, + {0x51a89433u, 0x00000000u, 0x00000000u}, + {0x52a91473u, 0x00000000u, 0x00000000u}, + {0x53a994b3u, 0x00000000u, 0x00000000u}, + {0x552a5503u, 0x00000000u, 0x00000000u}, + {0x562ad553u, 0x00000000u, 0x00000000u}, + {0x572b5593u, 0x00000000u, 0x00000000u}, + {0x002c15e2u, 0x00000000u, 0x00000000u}, + {0x59ac9623u, 0x00000000u, 0x00000000u}, + {0x5aad1674u, 0x000000b6u, 0x00000000u}, + {0x5c2dd6d3u, 0x00000000u, 0x00000000u}, + {0x5dae9724u, 0x000000bcu, 0x00000000u}, + {0x5f2f5794u, 0x000000bfu, 0x00000000u}, + {0x60b017f3u, 0x00000000u, 0x00000000u}, + {0x6230d844u, 0x000000c5u, 0x00000000u}, + {0x63b198b4u, 0x000000c8u, 0x00000000u}, + {0x65325914u, 0x000000cbu, 0x00000000u}, + {0x66b31974u, 0x000000ceu, 0x00000000u}, + {0x6833d9d4u, 0x000000d1u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c053u, 0x00000000u, 0x00000000u}, + {0x03014093u, 0x00000000u, 0x00000000u}, + {0x048200e3u, 0x00000000u, 0x00000000u}, + {0x05828133u, 0x00000000u, 0x00000000u}, + {0x06830173u, 0x00000000u, 0x00000000u}, + {0x078381b3u, 0x00000000u, 0x00000000u}, + {0x09044203u, 0x00000000u, 0x00000000u}, + {0x0a04c253u, 0x00000000u, 0x00000000u}, + {0x0b054293u, 0x00000000u, 0x00000000u}, + {0x0c05c2d3u, 0x00000000u, 0x00000000u}, + {0x0d064313u, 0x00000000u, 0x00000000u}, + {0x0e870363u, 0x00000000u, 0x00000000u}, + {0x0f8783b3u, 0x00000000u, 0x00000000u}, + {0x108803f3u, 0x00000000u, 0x00000000u}, + {0x11888433u, 0x00000000u, 0x00000000u}, + {0x13094483u, 0x00000000u, 0x00000000u}, + {0x1409c4d3u, 0x00000000u, 0x00000000u}, + {0x150a4513u, 0x00000000u, 0x00000000u}, + {0x160ac553u, 0x00000000u, 0x00000000u}, + {0x178b85a3u, 0x00000000u, 0x00000000u}, + {0x188c05f3u, 0x00000000u, 0x00000000u}, + {0x198c8633u, 0x00000000u, 0x00000000u}, + {0x1a8d0673u, 0x00000000u, 0x00000000u}, + {0x1c0dc6c3u, 0x00000000u, 0x00000000u}, + {0x1d0e4713u, 0x00000000u, 0x00000000u}, + {0x1e0ec753u, 0x00000000u, 0x00000000u}, + {0x1f0f4793u, 0x00000000u, 0x00000000u}, + {0x200fc7d3u, 0x00000000u, 0x00000000u}, + {0x21908823u, 0x00000000u, 0x00000000u}, + {0x23114883u, 0x00000000u, 0x00000000u}, + {0x249208e3u, 0x00000000u, 0x00000000u}, + {0x25928934u, 0x0000004cu, 0x00000000u}, + {0x27134994u, 0x0000004fu, 0x00000000u}, + {0x289409f4u, 0x00000052u, 0x00000000u}, + {0x2a14ca54u, 0x00000055u, 0x00000000u}, + {0x2c15cac3u, 0x00000000u, 0x00000000u}, + {0x2d164b14u, 0x0000005bu, 0x00000000u}, + {0x2e970b74u, 0x0000005eu, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x31988c33u, 0x00000000u, 0x00000000u}, + {0x32990c74u, 0x00000066u, 0x00000000u}, + {0x349a0ce5u, 0x0000d66au, 0x00000000u}, + {0x369b0d75u, 0x0000de6eu, 0x00000000u}, + {0x389c0df5u, 0x0000e672u, 0x00000000u}, + {0x3b1d4e84u, 0x00000077u, 0x00000000u}, + {0x3d1e4f04u, 0x0000007bu, 0x00000000u}, + {0x3f1f4f84u, 0x0000007fu, 0x00000000u}, + {0x41205005u, 0x00010883u, 0x00000000u}, + {0x43215095u, 0x00011087u, 0x00000000u}, + {0x45a29124u, 0x0000008cu, 0x00000000u}, + {0x47a391a5u, 0x00012290u, 0x00000000u}, + {0x4a24d243u, 0x00000000u, 0x00000000u}, + {0x002592a2u, 0x00000000u, 0x00000000u}, + {0x4ca612e3u, 0x00000000u, 0x00000000u}, + {0x4da69333u, 0x00000000u, 0x00000000u}, + {0x4ea71373u, 0x00000000u, 0x00000000u}, + {0x4fa793b3u, 0x00000000u, 0x00000000u}, + {0x00285402u, 0x00000000u, 0x00000000u}, + {0x5228d443u, 0x00000000u, 0x00000000u}, + {0x53295493u, 0x00000000u, 0x00000000u}, + {0x5429d4d3u, 0x00000000u, 0x00000000u}, + {0x55aa9523u, 0x00000000u, 0x00000000u}, + {0x56ab1573u, 0x00000000u, 0x00000000u}, + {0x57ab95b3u, 0x00000000u, 0x00000000u}, + {0x58ac15f3u, 0x00000000u, 0x00000000u}, + {0x59ac9633u, 0x00000000u, 0x00000000u}, + {0x5b2d5684u, 0x000000b7u, 0x00000000u}, + {0x5cae16f3u, 0x00000000u, 0x00000000u}, + {0x5e2ed743u, 0x00000000u, 0x00000000u}, + {0x5f2f5794u, 0x000000bfu, 0x00000000u}, + {0x61305803u, 0x00000000u, 0x00000000u}, + {0x62b11863u, 0x00000000u, 0x00000000u}, + {0x63b198b4u, 0x000000c8u, 0x00000000u}, + {0x65325914u, 0x000000cbu, 0x00000000u}, + {0x66b31974u, 0x000000ceu, 0x00000000u}, + {0x6833d9d4u, 0x000000d1u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c053u, 0x00000000u, 0x00000000u}, + {0x03014093u, 0x00000000u, 0x00000000u}, + {0x0401c0d3u, 0x00000000u, 0x00000000u}, + {0x05828123u, 0x00000000u, 0x00000000u}, + {0x06830173u, 0x00000000u, 0x00000000u}, + {0x078381b3u, 0x00000000u, 0x00000000u}, + {0x088401f3u, 0x00000000u, 0x00000000u}, + {0x09848233u, 0x00000000u, 0x00000000u}, + {0x0a850273u, 0x00000000u, 0x00000000u}, + {0x0005c2c2u, 0x00000000u, 0x00000000u}, + {0x0d064303u, 0x00000000u, 0x00000000u}, + {0x0e06c353u, 0x00000000u, 0x00000000u}, + {0x0f074393u, 0x00000000u, 0x00000000u}, + {0x000803e2u, 0x00000000u, 0x00000000u}, + {0x00088422u, 0x00000000u, 0x00000000u}, + {0x12890463u, 0x00000000u, 0x00000000u}, + {0x138984b3u, 0x00000000u, 0x00000000u}, + {0x148a04f3u, 0x00000000u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x000b4582u, 0x00000000u, 0x00000000u}, + {0x180bc5c3u, 0x00000000u, 0x00000000u}, + {0x190c4613u, 0x00000000u, 0x00000000u}, + {0x1a0cc653u, 0x00000000u, 0x00000000u}, + {0x1b0d4693u, 0x00000000u, 0x00000000u}, + {0x1c0dc6d3u, 0x00000000u, 0x00000000u}, + {0x1d0e4713u, 0x00000000u, 0x00000000u}, + {0x1e8f0763u, 0x00000000u, 0x00000000u}, + {0x1f8f87b3u, 0x00000000u, 0x00000000u}, + {0x209007f3u, 0x00000000u, 0x00000000u}, + {0x21908834u, 0x00000044u, 0x00000000u}, + {0x239188a3u, 0x00000000u, 0x00000000u}, + {0x249208f4u, 0x0000004au, 0x00000000u}, + {0x2612c953u, 0x00000000u, 0x00000000u}, + {0x279389a3u, 0x00000000u, 0x00000000u}, + {0x29144a03u, 0x00000000u, 0x00000000u}, + {0x2a950a63u, 0x00000000u, 0x00000000u}, + {0x2b958ab4u, 0x00000058u, 0x00000000u}, + {0x2d164b14u, 0x0000005bu, 0x00000000u}, + {0x2e970b74u, 0x0000005eu, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x31988c33u, 0x00000000u, 0x00000000u}, + {0x32990c74u, 0x00000066u, 0x00000000u}, + {0x349a0ce4u, 0x0000006au, 0x00000000u}, + {0x361acd55u, 0x0000dc6du, 0x00000000u}, + {0x389c0de4u, 0x00000072u, 0x00000000u}, + {0x3a9d0e65u, 0x0000ee76u, 0x00000000u}, + {0x3c9e0ef5u, 0x0000f67au, 0x00000000u}, + {0x3e9f0f75u, 0x0000fe7eu, 0x00000000u}, + {0x40a00ff5u, 0x00010682u, 0x00000000u}, + {0x43215084u, 0x00000087u, 0x00000000u}, + {0x45225104u, 0x0000008bu, 0x00000000u}, + {0x47235184u, 0x0000008fu, 0x00000000u}, + {0x48a411f5u, 0x00012692u, 0x00000000u}, + {0x4b255283u, 0x00000000u, 0x00000000u}, + {0x4c25d2d3u, 0x00000000u, 0x00000000u}, + {0x4d265313u, 0x00000000u, 0x00000000u}, + {0x4e26d353u, 0x00000000u, 0x00000000u}, + {0x4f275393u, 0x00000000u, 0x00000000u}, + {0x5027d3d3u, 0x00000000u, 0x00000000u}, + {0x51a89423u, 0x00000000u, 0x00000000u}, + {0x52a91473u, 0x00000000u, 0x00000000u}, + {0x53a994b3u, 0x00000000u, 0x00000000u}, + {0x54aa14f3u, 0x00000000u, 0x00000000u}, + {0x55aa9533u, 0x00000000u, 0x00000000u}, + {0x572b5583u, 0x00000000u, 0x00000000u}, + {0x582bd5d3u, 0x00000000u, 0x00000000u}, + {0x592c5613u, 0x00000000u, 0x00000000u}, + {0x5a2cd653u, 0x00000000u, 0x00000000u}, + {0x5b2d5694u, 0x000000b7u, 0x00000000u}, + {0x5cae16f4u, 0x000000bau, 0x00000000u}, + {0x5e2ed754u, 0x000000bdu, 0x00000000u}, + {0x5faf97b4u, 0x000000c0u, 0x00000000u}, + {0x61305813u, 0x00000000u, 0x00000000u}, + {0x62b11863u, 0x00000000u, 0x00000000u}, + {0x6431d8c3u, 0x00000000u, 0x00000000u}, + {0x65b29923u, 0x00000000u, 0x00000000u}, + {0x66b31974u, 0x000000ceu, 0x00000000u}, + {0x6833d9d4u, 0x000000d1u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c053u, 0x00000000u, 0x00000000u}, + {0x03014093u, 0x00000000u, 0x00000000u}, + {0x0401c0d3u, 0x00000000u, 0x00000000u}, + {0x05024113u, 0x00000000u, 0x00000000u}, + {0x0602c153u, 0x00000000u, 0x00000000u}, + {0x07034194u, 0x0000000fu, 0x00000000u}, + {0x088401f3u, 0x00000000u, 0x00000000u}, + {0x09848233u, 0x00000000u, 0x00000000u}, + {0x0a850273u, 0x00000000u, 0x00000000u}, + {0x0b8582b3u, 0x00000000u, 0x00000000u}, + {0x0c8602f3u, 0x00000000u, 0x00000000u}, + {0x0d868333u, 0x00000000u, 0x00000000u}, + {0x00074382u, 0x00000000u, 0x00000000u}, + {0x0f8783b3u, 0x00000000u, 0x00000000u}, + {0x00084402u, 0x00000000u, 0x00000000u}, + {0x1208c443u, 0x00000000u, 0x00000000u}, + {0x13094493u, 0x00000000u, 0x00000000u}, + {0x1409c4d3u, 0x00000000u, 0x00000000u}, + {0x150a4513u, 0x00000000u, 0x00000000u}, + {0x160ac553u, 0x00000000u, 0x00000000u}, + {0x000b85a2u, 0x00000000u, 0x00000000u}, + {0x000c05e2u, 0x00000000u, 0x00000000u}, + {0x198c8623u, 0x00000000u, 0x00000000u}, + {0x1a8d0673u, 0x00000000u, 0x00000000u}, + {0x1b8d86b3u, 0x00000000u, 0x00000000u}, + {0x1c8e06f3u, 0x00000000u, 0x00000000u}, + {0x1d8e8733u, 0x00000000u, 0x00000000u}, + {0x1e8f0773u, 0x00000000u, 0x00000000u}, + {0x1f8f87b3u, 0x00000000u, 0x00000000u}, + {0x21104803u, 0x00000000u, 0x00000000u}, + {0x2210c854u, 0x00000045u, 0x00000000u}, + {0x239188b3u, 0x00000000u, 0x00000000u}, + {0x25124903u, 0x00000000u, 0x00000000u}, + {0x2612c954u, 0x0000004du, 0x00000000u}, + {0x279389b4u, 0x00000050u, 0x00000000u}, + {0x29144a14u, 0x00000053u, 0x00000000u}, + {0x2a950a73u, 0x00000000u, 0x00000000u}, + {0x2c15cac3u, 0x00000000u, 0x00000000u}, + {0x2d164b14u, 0x0000005bu, 0x00000000u}, + {0x2e970b73u, 0x00000000u, 0x00000000u}, + {0x3017cbc3u, 0x00000000u, 0x00000000u}, + {0x00188c22u, 0x00000000u, 0x00000000u}, + {0x32990c64u, 0x00000066u, 0x00000000u}, + {0x3419ccd5u, 0x0000d469u, 0x00000000u}, + {0x361acd55u, 0x0000dc6du, 0x00000000u}, + {0x381bcdd5u, 0x0000e471u, 0x00000000u}, + {0x3a9d0e64u, 0x00000076u, 0x00000000u}, + {0x3c9e0ee4u, 0x0000007au, 0x00000000u}, + {0x3e1ecf55u, 0x0000fc7du, 0x00000000u}, + {0x401fcfd5u, 0x00010481u, 0x00000000u}, + {0x4220d055u, 0x00010c85u, 0x00000000u}, + {0x4421d0d5u, 0x00011489u, 0x00000000u}, + {0x46a31164u, 0x0000008eu, 0x00000000u}, + {0x48a411e4u, 0x00000092u, 0x00000000u}, + {0x4aa51263u, 0x00000000u, 0x00000000u}, + {0x4ba592b3u, 0x00000000u, 0x00000000u}, + {0x4ca612f3u, 0x00000000u, 0x00000000u}, + {0x4da69333u, 0x00000000u, 0x00000000u}, + {0x4ea71373u, 0x00000000u, 0x00000000u}, + {0x4fa793b3u, 0x00000000u, 0x00000000u}, + {0x51285403u, 0x00000000u, 0x00000000u}, + {0x5228d453u, 0x00000000u, 0x00000000u}, + {0x53295493u, 0x00000000u, 0x00000000u}, + {0x5429d4d3u, 0x00000000u, 0x00000000u}, + {0x552a5513u, 0x00000000u, 0x00000000u}, + {0x562ad553u, 0x00000000u, 0x00000000u}, + {0x572b5594u, 0x000000afu, 0x00000000u}, + {0x58ac15f3u, 0x00000000u, 0x00000000u}, + {0x59ac9633u, 0x00000000u, 0x00000000u}, + {0x5aad1673u, 0x00000000u, 0x00000000u}, + {0x5bad96b4u, 0x000000b8u, 0x00000000u}, + {0x5d2e5713u, 0x00000000u, 0x00000000u}, + {0x5eaf1763u, 0x00000000u, 0x00000000u}, + {0x602fd7c3u, 0x00000000u, 0x00000000u}, + {0x61b09823u, 0x00000000u, 0x00000000u}, + {0x62b11874u, 0x000000c6u, 0x00000000u}, + {0x6431d8d4u, 0x000000c9u, 0x00000000u}, + {0x65b29933u, 0x00000000u, 0x00000000u}, + {0x67335983u, 0x00000000u, 0x00000000u}, + {0x68b419e3u, 0x00000000u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c053u, 0x00000000u, 0x00000000u}, + {0x03014093u, 0x00000000u, 0x00000000u}, + {0x0401c0d3u, 0x00000000u, 0x00000000u}, + {0x05024113u, 0x00000000u, 0x00000000u}, + {0x0602c153u, 0x00000000u, 0x00000000u}, + {0x07034193u, 0x00000000u, 0x00000000u}, + {0x0803c1d3u, 0x00000000u, 0x00000000u}, + {0x09044213u, 0x00000000u, 0x00000000u}, + {0x0a04c253u, 0x00000000u, 0x00000000u}, + {0x0b054293u, 0x00000000u, 0x00000000u}, + {0x0c05c2d4u, 0x00000019u, 0x00000000u}, + {0x0d868333u, 0x00000000u, 0x00000000u}, + {0x0e870373u, 0x00000000u, 0x00000000u}, + {0x0f8783b3u, 0x00000000u, 0x00000000u}, + {0x108803f3u, 0x00000000u, 0x00000000u}, + {0x11888433u, 0x00000000u, 0x00000000u}, + {0x12890473u, 0x00000000u, 0x00000000u}, + {0x138984b3u, 0x00000000u, 0x00000000u}, + {0x148a04f3u, 0x00000000u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x168b0573u, 0x00000000u, 0x00000000u}, + {0x178b85b3u, 0x00000000u, 0x00000000u}, + {0x188c05f4u, 0x00000032u, 0x00000000u}, + {0x1a0cc653u, 0x00000000u, 0x00000000u}, + {0x1b0d4693u, 0x00000000u, 0x00000000u}, + {0x1c0dc6d3u, 0x00000000u, 0x00000000u}, + {0x1d0e4713u, 0x00000000u, 0x00000000u}, + {0x1e0ec753u, 0x00000000u, 0x00000000u}, + {0x1f0f4793u, 0x00000000u, 0x00000000u}, + {0x200fc7d3u, 0x00000000u, 0x00000000u}, + {0x21104814u, 0x00000043u, 0x00000000u}, + {0x22910873u, 0x00000000u, 0x00000000u}, + {0x2411c8c3u, 0x00000000u, 0x00000000u}, + {0x25124914u, 0x0000004bu, 0x00000000u}, + {0x26930973u, 0x00000000u, 0x00000000u}, + {0x2813c9c3u, 0x00000000u, 0x00000000u}, + {0x29144a14u, 0x00000053u, 0x00000000u}, + {0x2a950a74u, 0x00000056u, 0x00000000u}, + {0x2c15cad3u, 0x00000000u, 0x00000000u}, + {0x2d968b23u, 0x00000000u, 0x00000000u}, + {0x2f174b83u, 0x00000000u, 0x00000000u}, + {0x3017cbd4u, 0x00000061u, 0x00000000u}, + {0x31988c33u, 0x00000000u, 0x00000000u}, + {0x32990c73u, 0x00000000u, 0x00000000u}, + {0x3419ccc4u, 0x00000069u, 0x00000000u}, + {0x361acd44u, 0x0000006du, 0x00000000u}, + {0x379b8db5u, 0x0000e270u, 0x00000000u}, + {0x399c8e35u, 0x0000ea74u, 0x00000000u}, + {0x3b9d8eb5u, 0x0000f278u, 0x00000000u}, + {0x3d9e8f35u, 0x0000fa7cu, 0x00000000u}, + {0x3f9f8fb4u, 0x00000080u, 0x00000000u}, + {0x41a09025u, 0x00010a84u, 0x00000000u}, + {0x43a190b4u, 0x00000088u, 0x00000000u}, + {0x45a29125u, 0x00011a8cu, 0x00000000u}, + {0x47a391b4u, 0x00000090u, 0x00000000u}, + {0x49a49224u, 0x00000094u, 0x00000000u}, + {0x4b255293u, 0x00000000u, 0x00000000u}, + {0x4c25d2d3u, 0x00000000u, 0x00000000u}, + {0x4d265313u, 0x00000000u, 0x00000000u}, + {0x00271362u, 0x00000000u, 0x00000000u}, + {0x002793a2u, 0x00000000u, 0x00000000u}, + {0x50a813e3u, 0x00000000u, 0x00000000u}, + {0x51a89433u, 0x00000000u, 0x00000000u}, + {0x52a91473u, 0x00000000u, 0x00000000u}, + {0x53a994b3u, 0x00000000u, 0x00000000u}, + {0x54aa14f3u, 0x00000000u, 0x00000000u}, + {0x55aa9533u, 0x00000000u, 0x00000000u}, + {0x56ab1573u, 0x00000000u, 0x00000000u}, + {0x57ab95b3u, 0x00000000u, 0x00000000u}, + {0x58ac15f3u, 0x00000000u, 0x00000000u}, + {0x59ac9633u, 0x00000000u, 0x00000000u}, + {0x5b2d5683u, 0x00000000u, 0x00000000u}, + {0x5c2dd6d4u, 0x000000b9u, 0x00000000u}, + {0x5dae9733u, 0x00000000u, 0x00000000u}, + {0x5f2f5783u, 0x00000000u, 0x00000000u}, + {0x60b017e3u, 0x00000000u, 0x00000000u}, + {0x61b09834u, 0x000000c4u, 0x00000000u}, + {0x63315893u, 0x00000000u, 0x00000000u}, + {0x6431d8d4u, 0x000000c9u, 0x00000000u}, + {0x65b29934u, 0x000000ccu, 0x00000000u}, + {0x67335993u, 0x00000000u, 0x00000000u}, + {0x68b419e3u, 0x00000000u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c053u, 0x00000000u, 0x00000000u}, + {0x03014093u, 0x00000000u, 0x00000000u}, + {0x0401c0d3u, 0x00000000u, 0x00000000u}, + {0x05024113u, 0x00000000u, 0x00000000u}, + {0x0602c153u, 0x00000000u, 0x00000000u}, + {0x07034193u, 0x00000000u, 0x00000000u}, + {0x0803c1d3u, 0x00000000u, 0x00000000u}, + {0x09044213u, 0x00000000u, 0x00000000u}, + {0x0a04c253u, 0x00000000u, 0x00000000u}, + {0x0b054293u, 0x00000000u, 0x00000000u}, + {0x0c05c2d3u, 0x00000000u, 0x00000000u}, + {0x0d064313u, 0x00000000u, 0x00000000u}, + {0x0e06c353u, 0x00000000u, 0x00000000u}, + {0x0f074393u, 0x00000000u, 0x00000000u}, + {0x1007c3d3u, 0x00000000u, 0x00000000u}, + {0x11084413u, 0x00000000u, 0x00000000u}, + {0x1208c453u, 0x00000000u, 0x00000000u}, + {0x13094493u, 0x00000000u, 0x00000000u}, + {0x1409c4d3u, 0x00000000u, 0x00000000u}, + {0x150a4513u, 0x00000000u, 0x00000000u}, + {0x160ac553u, 0x00000000u, 0x00000000u}, + {0x170b4593u, 0x00000000u, 0x00000000u}, + {0x180bc5d3u, 0x00000000u, 0x00000000u}, + {0x000c8622u, 0x00000000u, 0x00000000u}, + {0x000d0662u, 0x00000000u, 0x00000000u}, + {0x1b0d4693u, 0x00000000u, 0x00000000u}, + {0x1c0dc6d3u, 0x00000000u, 0x00000000u}, + {0x1d0e4713u, 0x00000000u, 0x00000000u}, + {0x000f0762u, 0x00000000u, 0x00000000u}, + {0x000f87a2u, 0x00000000u, 0x00000000u}, + {0x001007e2u, 0x00000000u, 0x00000000u}, + {0x21908823u, 0x00000000u, 0x00000000u}, + {0x22910874u, 0x00000046u, 0x00000000u}, + {0x2411c8d4u, 0x00000049u, 0x00000000u}, + {0x25928933u, 0x00000000u, 0x00000000u}, + {0x27134983u, 0x00000000u, 0x00000000u}, + {0x2813c9d3u, 0x00000000u, 0x00000000u}, + {0x29948a23u, 0x00000000u, 0x00000000u}, + {0x2a950a74u, 0x00000056u, 0x00000000u}, + {0x2c15cad3u, 0x00000000u, 0x00000000u}, + {0x2d968b23u, 0x00000000u, 0x00000000u}, + {0x2e970b74u, 0x0000005eu, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x00188c22u, 0x00000000u, 0x00000000u}, + {0x32990c63u, 0x00000000u, 0x00000000u}, + {0x3419ccc4u, 0x00000069u, 0x00000000u}, + {0x359a8d35u, 0x0000da6cu, 0x00000000u}, + {0x379b8db5u, 0x0000e270u, 0x00000000u}, + {0x399c8e34u, 0x00000074u, 0x00000000u}, + {0x3b9d8ea4u, 0x00000078u, 0x00000000u}, + {0x3d1e4f15u, 0x0000f87bu, 0x00000000u}, + {0x3f9f8fa4u, 0x00000080u, 0x00000000u}, + {0x41205015u, 0x00010883u, 0x00000000u}, + {0x43215094u, 0x00000087u, 0x00000000u}, + {0x45225105u, 0x0001188bu, 0x00000000u}, + {0x47235195u, 0x0001208fu, 0x00000000u}, + {0x49245214u, 0x00000093u, 0x00000000u}, + {0x00255282u, 0x00000000u, 0x00000000u}, + {0x4c25d2c3u, 0x00000000u, 0x00000000u}, + {0x4d265313u, 0x00000000u, 0x00000000u}, + {0x4e26d353u, 0x00000000u, 0x00000000u}, + {0x00275392u, 0x00000000u, 0x00000000u}, + {0x0027d3c2u, 0x00000000u, 0x00000000u}, + {0x51285403u, 0x00000000u, 0x00000000u}, + {0x5228d453u, 0x00000000u, 0x00000000u}, + {0x53295493u, 0x00000000u, 0x00000000u}, + {0x5429d4d3u, 0x00000000u, 0x00000000u}, + {0x552a5513u, 0x00000000u, 0x00000000u}, + {0x562ad553u, 0x00000000u, 0x00000000u}, + {0x572b5593u, 0x00000000u, 0x00000000u}, + {0x582bd5d3u, 0x00000000u, 0x00000000u}, + {0x592c5613u, 0x00000000u, 0x00000000u}, + {0x5a2cd653u, 0x00000000u, 0x00000000u}, + {0x5b2d5693u, 0x00000000u, 0x00000000u}, + {0x5cae16e3u, 0x00000000u, 0x00000000u}, + {0x5dae9734u, 0x000000bcu, 0x00000000u}, + {0x5f2f5793u, 0x00000000u, 0x00000000u}, + {0x60b017e3u, 0x00000000u, 0x00000000u}, + {0x61b09834u, 0x000000c4u, 0x00000000u}, + {0x63315893u, 0x00000000u, 0x00000000u}, + {0x64b218e3u, 0x00000000u, 0x00000000u}, + {0x65b29934u, 0x000000ccu, 0x00000000u}, + {0x67335993u, 0x00000000u, 0x00000000u}, + {0x68b419e3u, 0x00000000u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c053u, 0x00000000u, 0x00000000u}, + {0x03014093u, 0x00000000u, 0x00000000u}, + {0x0401c0d3u, 0x00000000u, 0x00000000u}, + {0x05024113u, 0x00000000u, 0x00000000u}, + {0x0002c152u, 0x00000000u, 0x00000000u}, + {0x07034183u, 0x00000000u, 0x00000000u}, + {0x0003c1d2u, 0x00000000u, 0x00000000u}, + {0x00044202u, 0x00000000u, 0x00000000u}, + {0x0004c242u, 0x00000000u, 0x00000000u}, + {0x00054282u, 0x00000000u, 0x00000000u}, + {0x0b8582b3u, 0x00000000u, 0x00000000u}, + {0x00064302u, 0x00000000u, 0x00000000u}, + {0x0e06c343u, 0x00000000u, 0x00000000u}, + {0x0f074393u, 0x00000000u, 0x00000000u}, + {0x0007c3d2u, 0x00000000u, 0x00000000u}, + {0x00084402u, 0x00000000u, 0x00000000u}, + {0x0008c442u, 0x00000000u, 0x00000000u}, + {0x12890473u, 0x00000000u, 0x00000000u}, + {0x0009c4c2u, 0x00000000u, 0x00000000u}, + {0x000a4502u, 0x00000000u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x168b0573u, 0x00000000u, 0x00000000u}, + {0x178b85b3u, 0x00000000u, 0x00000000u}, + {0x188c05f3u, 0x00000000u, 0x00000000u}, + {0x198c8633u, 0x00000000u, 0x00000000u}, + {0x1a8d0673u, 0x00000000u, 0x00000000u}, + {0x1b8d86b3u, 0x00000000u, 0x00000000u}, + {0x1c8e06f3u, 0x00000000u, 0x00000000u}, + {0x1d8e8733u, 0x00000000u, 0x00000000u}, + {0x1e8f0773u, 0x00000000u, 0x00000000u}, + {0x1f8f87b3u, 0x00000000u, 0x00000000u}, + {0x209007f3u, 0x00000000u, 0x00000000u}, + {0x21908834u, 0x00000044u, 0x00000000u}, + {0x23114893u, 0x00000000u, 0x00000000u}, + {0x249208e3u, 0x00000000u, 0x00000000u}, + {0x25928933u, 0x00000000u, 0x00000000u}, + {0x27134983u, 0x00000000u, 0x00000000u}, + {0x289409e3u, 0x00000000u, 0x00000000u}, + {0x29948a33u, 0x00000000u, 0x00000000u}, + {0x2b154a83u, 0x00000000u, 0x00000000u}, + {0x2c15cad3u, 0x00000000u, 0x00000000u}, + {0x2d968b23u, 0x00000000u, 0x00000000u}, + {0x2e970b73u, 0x00000000u, 0x00000000u}, + {0x3017cbc3u, 0x00000000u, 0x00000000u}, + {0x31184c13u, 0x00000000u, 0x00000000u}, + {0x3218cc54u, 0x00000065u, 0x00000000u}, + {0x33998cb4u, 0x00000068u, 0x00000000u}, + {0x359a8d25u, 0x0000da6cu, 0x00000000u}, + {0x379b8db4u, 0x00000070u, 0x00000000u}, + {0x399c8e24u, 0x00000074u, 0x00000000u}, + {0x3b9d8ea4u, 0x00000078u, 0x00000000u}, + {0x3d1e4f14u, 0x0000007bu, 0x00000000u}, + {0x3f1f4f84u, 0x0000007fu, 0x00000000u}, + {0x40a00ff5u, 0x00010682u, 0x00000000u}, + {0x43215084u, 0x00000087u, 0x00000000u}, + {0x44a210f5u, 0x0001168au, 0x00000000u}, + {0x46a31174u, 0x0000008eu, 0x00000000u}, + {0x48a411e4u, 0x00000092u, 0x00000000u}, + {0x4a24d254u, 0x00000095u, 0x00000000u}, + {0x4ba592b3u, 0x00000000u, 0x00000000u}, + {0x002612f2u, 0x00000000u, 0x00000000u}, + {0x00269322u, 0x00000000u, 0x00000000u}, + {0x00271362u, 0x00000000u, 0x00000000u}, + {0x4f275393u, 0x00000000u, 0x00000000u}, + {0x50a813e3u, 0x00000000u, 0x00000000u}, + {0x51a89433u, 0x00000000u, 0x00000000u}, + {0x52a91473u, 0x00000000u, 0x00000000u}, + {0x53a994b3u, 0x00000000u, 0x00000000u}, + {0x002a14f2u, 0x00000000u, 0x00000000u}, + {0x002a9522u, 0x00000000u, 0x00000000u}, + {0x002b1562u, 0x00000000u, 0x00000000u}, + {0x002b95a2u, 0x00000000u, 0x00000000u}, + {0x002c15e2u, 0x00000000u, 0x00000000u}, + {0x002c9622u, 0x00000000u, 0x00000000u}, + {0x5a2cd653u, 0x00000000u, 0x00000000u}, + {0x5bad96a3u, 0x00000000u, 0x00000000u}, + {0x5cae16f4u, 0x000000bau, 0x00000000u}, + {0x5e2ed754u, 0x000000bdu, 0x00000000u}, + {0x5faf97b3u, 0x00000000u, 0x00000000u}, + {0x60b017f4u, 0x000000c2u, 0x00000000u}, + {0x6230d853u, 0x00000000u, 0x00000000u}, + {0x63b198a3u, 0x00000000u, 0x00000000u}, + {0x64b218f3u, 0x00000000u, 0x00000000u}, + {0x6632d943u, 0x00000000u, 0x00000000u}, + {0x67335993u, 0x00000000u, 0x00000000u}, + {0x68b419e3u, 0x00000000u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c053u, 0x00000000u, 0x00000000u}, + {0x03014093u, 0x00000000u, 0x00000000u}, + {0x0001c0d2u, 0x00000000u, 0x00000000u}, + {0x00024102u, 0x00000000u, 0x00000000u}, + {0x05828133u, 0x00000000u, 0x00000000u}, + {0x06830173u, 0x00000000u, 0x00000000u}, + {0x078381b3u, 0x00000000u, 0x00000000u}, + {0x088401f3u, 0x00000000u, 0x00000000u}, + {0x0004c242u, 0x00000000u, 0x00000000u}, + {0x0a850273u, 0x00000000u, 0x00000000u}, + {0x0b8582b3u, 0x00000000u, 0x00000000u}, + {0x0c8602f3u, 0x00000000u, 0x00000000u}, + {0x0d868333u, 0x00000000u, 0x00000000u}, + {0x0e870373u, 0x00000000u, 0x00000000u}, + {0x0f8783b3u, 0x00000000u, 0x00000000u}, + {0x108803f3u, 0x00000000u, 0x00000000u}, + {0x11888433u, 0x00000000u, 0x00000000u}, + {0x12890473u, 0x00000000u, 0x00000000u}, + {0x000984b2u, 0x00000000u, 0x00000000u}, + {0x000a04e2u, 0x00000000u, 0x00000000u}, + {0x000a8522u, 0x00000000u, 0x00000000u}, + {0x160ac553u, 0x00000000u, 0x00000000u}, + {0x170b4593u, 0x00000000u, 0x00000000u}, + {0x180bc5d3u, 0x00000000u, 0x00000000u}, + {0x190c4613u, 0x00000000u, 0x00000000u}, + {0x1a0cc653u, 0x00000000u, 0x00000000u}, + {0x1b0d4693u, 0x00000000u, 0x00000000u}, + {0x1c0dc6d3u, 0x00000000u, 0x00000000u}, + {0x1d0e4713u, 0x00000000u, 0x00000000u}, + {0x1e0ec753u, 0x00000000u, 0x00000000u}, + {0x1f0f4793u, 0x00000000u, 0x00000000u}, + {0x200fc7d3u, 0x00000000u, 0x00000000u}, + {0x21104813u, 0x00000000u, 0x00000000u}, + {0x2210c853u, 0x00000000u, 0x00000000u}, + {0x239188a3u, 0x00000000u, 0x00000000u}, + {0x249208f3u, 0x00000000u, 0x00000000u}, + {0x2612c943u, 0x00000000u, 0x00000000u}, + {0x27134993u, 0x00000000u, 0x00000000u}, + {0x289409e3u, 0x00000000u, 0x00000000u}, + {0x29948a34u, 0x00000054u, 0x00000000u}, + {0x2b154a93u, 0x00000000u, 0x00000000u}, + {0x2c15cad4u, 0x00000059u, 0x00000000u}, + {0x2d968b33u, 0x00000000u, 0x00000000u}, + {0x2e970b74u, 0x0000005eu, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x00188c22u, 0x00000000u, 0x00000000u}, + {0x32990c63u, 0x00000000u, 0x00000000u}, + {0x33998cb4u, 0x00000068u, 0x00000000u}, + {0x359a8d24u, 0x0000006cu, 0x00000000u}, + {0x371b4d94u, 0x0000006fu, 0x00000000u}, + {0x391c4e05u, 0x0000e873u, 0x00000000u}, + {0x3b1d4e94u, 0x00000077u, 0x00000000u}, + {0x3c9e0ef4u, 0x0000007au, 0x00000000u}, + {0x3e9f0f64u, 0x0000007eu, 0x00000000u}, + {0x40a00fe4u, 0x00000082u, 0x00000000u}, + {0x42a11064u, 0x00000086u, 0x00000000u}, + {0x4421d0d4u, 0x00000089u, 0x00000000u}, + {0x4622d144u, 0x0000008du, 0x00000000u}, + {0x47a391b4u, 0x00000090u, 0x00000000u}, + {0x49a49224u, 0x00000094u, 0x00000000u}, + {0x4b255293u, 0x00000000u, 0x00000000u}, + {0x4c25d2d3u, 0x00000000u, 0x00000000u}, + {0x4d265313u, 0x00000000u, 0x00000000u}, + {0x4e26d353u, 0x00000000u, 0x00000000u}, + {0x00275392u, 0x00000000u, 0x00000000u}, + {0x0027d3c2u, 0x00000000u, 0x00000000u}, + {0x51285403u, 0x00000000u, 0x00000000u}, + {0x5228d453u, 0x00000000u, 0x00000000u}, + {0x00295492u, 0x00000000u, 0x00000000u}, + {0x0029d4c2u, 0x00000000u, 0x00000000u}, + {0x002a5502u, 0x00000000u, 0x00000000u}, + {0x55aa9533u, 0x00000000u, 0x00000000u}, + {0x56ab1573u, 0x00000000u, 0x00000000u}, + {0x57ab95b3u, 0x00000000u, 0x00000000u}, + {0x58ac15f3u, 0x00000000u, 0x00000000u}, + {0x59ac9633u, 0x00000000u, 0x00000000u}, + {0x5aad1673u, 0x00000000u, 0x00000000u}, + {0x5c2dd6c3u, 0x00000000u, 0x00000000u}, + {0x5d2e5713u, 0x00000000u, 0x00000000u}, + {0x5eaf1763u, 0x00000000u, 0x00000000u}, + {0x5faf97b3u, 0x00000000u, 0x00000000u}, + {0x61305803u, 0x00000000u, 0x00000000u}, + {0x6230d854u, 0x000000c5u, 0x00000000u}, + {0x63b198b3u, 0x00000000u, 0x00000000u}, + {0x64b218f4u, 0x000000cau, 0x00000000u}, + {0x6632d953u, 0x00000000u, 0x00000000u}, + {0x67335994u, 0x000000cfu, 0x00000000u}, + {0x68b419f3u, 0x00000000u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x01004003u, 0x00000000u, 0x00000000u}, + {0x0200c053u, 0x00000000u, 0x00000000u}, + {0x00014092u, 0x00000000u, 0x00000000u}, + {0x0001c0c2u, 0x00000000u, 0x00000000u}, + {0x048200f3u, 0x00000000u, 0x00000000u}, + {0x05828133u, 0x00000000u, 0x00000000u}, + {0x06830173u, 0x00000000u, 0x00000000u}, + {0x078381b3u, 0x00000000u, 0x00000000u}, + {0x088401f3u, 0x00000000u, 0x00000000u}, + {0x00048232u, 0x00000000u, 0x00000000u}, + {0x00050262u, 0x00000000u, 0x00000000u}, + {0x0b054293u, 0x00000000u, 0x00000000u}, + {0x0c05c2d3u, 0x00000000u, 0x00000000u}, + {0x00068322u, 0x00000000u, 0x00000000u}, + {0x00070362u, 0x00000000u, 0x00000000u}, + {0x0f074393u, 0x00000000u, 0x00000000u}, + {0x1007c3d3u, 0x00000000u, 0x00000000u}, + {0x11084413u, 0x00000000u, 0x00000000u}, + {0x1208c453u, 0x00000000u, 0x00000000u}, + {0x13094493u, 0x00000000u, 0x00000000u}, + {0x0009c4d2u, 0x00000000u, 0x00000000u}, + {0x000a4502u, 0x00000000u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x168b0573u, 0x00000000u, 0x00000000u}, + {0x178b85b3u, 0x00000000u, 0x00000000u}, + {0x188c05f3u, 0x00000000u, 0x00000000u}, + {0x198c8633u, 0x00000000u, 0x00000000u}, + {0x1a8d0673u, 0x00000000u, 0x00000000u}, + {0x1b8d86b3u, 0x00000000u, 0x00000000u}, + {0x1c8e06f3u, 0x00000000u, 0x00000000u}, + {0x1d8e8733u, 0x00000000u, 0x00000000u}, + {0x000f0772u, 0x00000000u, 0x00000000u}, + {0x000f87a2u, 0x00000000u, 0x00000000u}, + {0x200fc7d3u, 0x00000000u, 0x00000000u}, + {0x21104813u, 0x00000000u, 0x00000000u}, + {0x2210c854u, 0x00000045u, 0x00000000u}, + {0x239188b4u, 0x00000048u, 0x00000000u}, + {0x25124913u, 0x00000000u, 0x00000000u}, + {0x2612c953u, 0x00000000u, 0x00000000u}, + {0x279389a3u, 0x00000000u, 0x00000000u}, + {0x289409f4u, 0x00000052u, 0x00000000u}, + {0x2a14ca53u, 0x00000000u, 0x00000000u}, + {0x2b154a93u, 0x00000000u, 0x00000000u}, + {0x2c960ae3u, 0x00000000u, 0x00000000u}, + {0x2d968b33u, 0x00000000u, 0x00000000u}, + {0x2e970b74u, 0x0000005eu, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x31184c13u, 0x00000000u, 0x00000000u}, + {0x3218cc54u, 0x00000065u, 0x00000000u}, + {0x33998cb4u, 0x00000068u, 0x00000000u}, + {0x351a4d15u, 0x0000d86bu, 0x00000000u}, + {0x371b4d94u, 0x0000006fu, 0x00000000u}, + {0x391c4e04u, 0x00000073u, 0x00000000u}, + {0x3a9d0e75u, 0x0000ee76u, 0x00000000u}, + {0x3c9e0ef4u, 0x0000007au, 0x00000000u}, + {0x3e1ecf54u, 0x0000007du, 0x00000000u}, + {0x401fcfc4u, 0x00000081u, 0x00000000u}, + {0x41a09035u, 0x00010a84u, 0x00000000u}, + {0x43a190b4u, 0x00000088u, 0x00000000u}, + {0x45a29124u, 0x0000008cu, 0x00000000u}, + {0x47a391a4u, 0x00000090u, 0x00000000u}, + {0x49245214u, 0x00000093u, 0x00000000u}, + {0x4aa51273u, 0x00000000u, 0x00000000u}, + {0x4ba592b3u, 0x00000000u, 0x00000000u}, + {0x4ca612f3u, 0x00000000u, 0x00000000u}, + {0x4da69333u, 0x00000000u, 0x00000000u}, + {0x4ea71373u, 0x00000000u, 0x00000000u}, + {0x002793b2u, 0x00000000u, 0x00000000u}, + {0x50a813e3u, 0x00000000u, 0x00000000u}, + {0x51a89433u, 0x00000000u, 0x00000000u}, + {0x52a91473u, 0x00000000u, 0x00000000u}, + {0x002994b2u, 0x00000000u, 0x00000000u}, + {0x002a14e2u, 0x00000000u, 0x00000000u}, + {0x552a5513u, 0x00000000u, 0x00000000u}, + {0x562ad553u, 0x00000000u, 0x00000000u}, + {0x572b5593u, 0x00000000u, 0x00000000u}, + {0x582bd5d3u, 0x00000000u, 0x00000000u}, + {0x592c5613u, 0x00000000u, 0x00000000u}, + {0x002cd652u, 0x00000000u, 0x00000000u}, + {0x5b2d5683u, 0x00000000u, 0x00000000u}, + {0x5c2dd6d4u, 0x000000b9u, 0x00000000u}, + {0x5dae9733u, 0x00000000u, 0x00000000u}, + {0x5f2f5783u, 0x00000000u, 0x00000000u}, + {0x602fd7d3u, 0x00000000u, 0x00000000u}, + {0x61305813u, 0x00000000u, 0x00000000u}, + {0x62b11863u, 0x00000000u, 0x00000000u}, + {0x63b198b3u, 0x00000000u, 0x00000000u}, + {0x65325903u, 0x00000000u, 0x00000000u}, + {0x6632d953u, 0x00000000u, 0x00000000u}, + {0x67335994u, 0x000000cfu, 0x00000000u}, + {0x68b419f3u, 0x00000000u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x0000c042u, 0x00000000u, 0x00000000u}, + {0x00014082u, 0x00000000u, 0x00000000u}, + {0x038180b3u, 0x00000000u, 0x00000000u}, + {0x048200f3u, 0x00000000u, 0x00000000u}, + {0x05828133u, 0x00000000u, 0x00000000u}, + {0x06830173u, 0x00000000u, 0x00000000u}, + {0x000381b2u, 0x00000000u, 0x00000000u}, + {0x000401e2u, 0x00000000u, 0x00000000u}, + {0x09044213u, 0x00000000u, 0x00000000u}, + {0x0a04c253u, 0x00000000u, 0x00000000u}, + {0x0b054293u, 0x00000000u, 0x00000000u}, + {0x0c05c2d3u, 0x00000000u, 0x00000000u}, + {0x00064312u, 0x00000000u, 0x00000000u}, + {0x0006c342u, 0x00000000u, 0x00000000u}, + {0x00074382u, 0x00000000u, 0x00000000u}, + {0x0007c3c2u, 0x00000000u, 0x00000000u}, + {0x108803f3u, 0x00000000u, 0x00000000u}, + {0x11888433u, 0x00000000u, 0x00000000u}, + {0x12890473u, 0x00000000u, 0x00000000u}, + {0x138984b3u, 0x00000000u, 0x00000000u}, + {0x000a04f2u, 0x00000000u, 0x00000000u}, + {0x000a8522u, 0x00000000u, 0x00000000u}, + {0x160ac553u, 0x00000000u, 0x00000000u}, + {0x170b4593u, 0x00000000u, 0x00000000u}, + {0x180bc5d3u, 0x00000000u, 0x00000000u}, + {0x190c4613u, 0x00000000u, 0x00000000u}, + {0x000cc652u, 0x00000000u, 0x00000000u}, + {0x000d4682u, 0x00000000u, 0x00000000u}, + {0x000dc6c2u, 0x00000000u, 0x00000000u}, + {0x000e4702u, 0x00000000u, 0x00000000u}, + {0x1d8e8733u, 0x00000000u, 0x00000000u}, + {0x1e8f0773u, 0x00000000u, 0x00000000u}, + {0x1f8f87b3u, 0x00000000u, 0x00000000u}, + {0x209007f3u, 0x00000000u, 0x00000000u}, + {0x21908833u, 0x00000000u, 0x00000000u}, + {0x22910873u, 0x00000000u, 0x00000000u}, + {0x2411c8c3u, 0x00000000u, 0x00000000u}, + {0x25124913u, 0x00000000u, 0x00000000u}, + {0x26930963u, 0x00000000u, 0x00000000u}, + {0x279389b3u, 0x00000000u, 0x00000000u}, + {0x289409f4u, 0x00000052u, 0x00000000u}, + {0x2a14ca53u, 0x00000000u, 0x00000000u}, + {0x2b154a93u, 0x00000000u, 0x00000000u}, + {0x2c960ae3u, 0x00000000u, 0x00000000u}, + {0x2d968b33u, 0x00000000u, 0x00000000u}, + {0x2f174b83u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x31184c13u, 0x00000000u, 0x00000000u}, + {0x3218cc53u, 0x00000000u, 0x00000000u}, + {0x33998ca4u, 0x00000068u, 0x00000000u}, + {0x351a4d14u, 0x0000006bu, 0x00000000u}, + {0x371b4d84u, 0x0000006fu, 0x00000000u}, + {0x391c4e03u, 0x00000000u, 0x00000000u}, + {0x3a1cce55u, 0x0000ec75u, 0x00000000u}, + {0x3c1dced4u, 0x00000079u, 0x00000000u}, + {0x3e1ecf44u, 0x0000007du, 0x00000000u}, + {0x3f9f8fb5u, 0x00010280u, 0x00000000u}, + {0x41a09034u, 0x00000084u, 0x00000000u}, + {0x43215094u, 0x00000087u, 0x00000000u}, + {0x45225104u, 0x0000008bu, 0x00000000u}, + {0x46a31174u, 0x0000008eu, 0x00000000u}, + {0x48a411e4u, 0x00000092u, 0x00000000u}, + {0x00251262u, 0x00000000u, 0x00000000u}, + {0x002592a2u, 0x00000000u, 0x00000000u}, + {0x4c25d2d3u, 0x00000000u, 0x00000000u}, + {0x4d265313u, 0x00000000u, 0x00000000u}, + {0x4e26d353u, 0x00000000u, 0x00000000u}, + {0x4f275393u, 0x00000000u, 0x00000000u}, + {0x5027d3d3u, 0x00000000u, 0x00000000u}, + {0x00285412u, 0x00000000u, 0x00000000u}, + {0x0028d442u, 0x00000000u, 0x00000000u}, + {0x00295482u, 0x00000000u, 0x00000000u}, + {0x0029d4c2u, 0x00000000u, 0x00000000u}, + {0x54aa14f3u, 0x00000000u, 0x00000000u}, + {0x55aa9533u, 0x00000000u, 0x00000000u}, + {0x56ab1573u, 0x00000000u, 0x00000000u}, + {0x57ab95b3u, 0x00000000u, 0x00000000u}, + {0x002c15f2u, 0x00000000u, 0x00000000u}, + {0x002c9622u, 0x00000000u, 0x00000000u}, + {0x5a2cd653u, 0x00000000u, 0x00000000u}, + {0x5bad96a3u, 0x00000000u, 0x00000000u}, + {0x5cae16f3u, 0x00000000u, 0x00000000u}, + {0x5dae9733u, 0x00000000u, 0x00000000u}, + {0x5f2f5783u, 0x00000000u, 0x00000000u}, + {0x602fd7d4u, 0x000000c1u, 0x00000000u}, + {0x61b09833u, 0x00000000u, 0x00000000u}, + {0x62b11873u, 0x00000000u, 0x00000000u}, + {0x6431d8c3u, 0x00000000u, 0x00000000u}, + {0x65325913u, 0x00000000u, 0x00000000u}, + {0x6632d953u, 0x00000000u, 0x00000000u}, + {0x67b399a3u, 0x00000000u, 0x00000000u}, + {0x68b419f3u, 0x00000000u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x0000c042u, 0x00000000u, 0x00000000u}, + {0x00014082u, 0x00000000u, 0x00000000u}, + {0x038180b3u, 0x00000000u, 0x00000000u}, + {0x048200f3u, 0x00000000u, 0x00000000u}, + {0x00028132u, 0x00000000u, 0x00000000u}, + {0x00030162u, 0x00000000u, 0x00000000u}, + {0x000381a2u, 0x00000000u, 0x00000000u}, + {0x0803c1d3u, 0x00000000u, 0x00000000u}, + {0x09044213u, 0x00000000u, 0x00000000u}, + {0x0004c252u, 0x00000000u, 0x00000000u}, + {0x00054282u, 0x00000000u, 0x00000000u}, + {0x0005c2c2u, 0x00000000u, 0x00000000u}, + {0x0c8602f3u, 0x00000000u, 0x00000000u}, + {0x0d868333u, 0x00000000u, 0x00000000u}, + {0x0e870373u, 0x00000000u, 0x00000000u}, + {0x0f8783b3u, 0x00000000u, 0x00000000u}, + {0x000803f2u, 0x00000000u, 0x00000000u}, + {0x11084413u, 0x00000000u, 0x00000000u}, + {0x1208c453u, 0x00000000u, 0x00000000u}, + {0x13094493u, 0x00000000u, 0x00000000u}, + {0x1409c4d3u, 0x00000000u, 0x00000000u}, + {0x000a4512u, 0x00000000u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x168b0573u, 0x00000000u, 0x00000000u}, + {0x178b85b3u, 0x00000000u, 0x00000000u}, + {0x188c05f3u, 0x00000000u, 0x00000000u}, + {0x000c8632u, 0x00000000u, 0x00000000u}, + {0x1a0cc653u, 0x00000000u, 0x00000000u}, + {0x1b0d4693u, 0x00000000u, 0x00000000u}, + {0x1c0dc6d3u, 0x00000000u, 0x00000000u}, + {0x1d0e4713u, 0x00000000u, 0x00000000u}, + {0x1e0ec753u, 0x00000000u, 0x00000000u}, + {0x000f4792u, 0x00000000u, 0x00000000u}, + {0x000fc7c2u, 0x00000000u, 0x00000000u}, + {0x209007f3u, 0x00000000u, 0x00000000u}, + {0x21908834u, 0x00000044u, 0x00000000u}, + {0x23114893u, 0x00000000u, 0x00000000u}, + {0x2411c8d3u, 0x00000000u, 0x00000000u}, + {0x25124914u, 0x0000004bu, 0x00000000u}, + {0x26930973u, 0x00000000u, 0x00000000u}, + {0x279389b4u, 0x00000050u, 0x00000000u}, + {0x29144a13u, 0x00000000u, 0x00000000u}, + {0x2a14ca53u, 0x00000000u, 0x00000000u}, + {0x2b958aa3u, 0x00000000u, 0x00000000u}, + {0x2c960af3u, 0x00000000u, 0x00000000u}, + {0x2d968b34u, 0x0000005cu, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x31184c13u, 0x00000000u, 0x00000000u}, + {0x3218cc53u, 0x00000000u, 0x00000000u}, + {0x33998ca3u, 0x00000000u, 0x00000000u}, + {0x349a0cf5u, 0x0000d66au, 0x00000000u}, + {0x369b0d74u, 0x0000006eu, 0x00000000u}, + {0x389c0de4u, 0x00000072u, 0x00000000u}, + {0x3a1cce54u, 0x00000075u, 0x00000000u}, + {0x3b9d8eb5u, 0x0000f278u, 0x00000000u}, + {0x3d9e8f34u, 0x0000007cu, 0x00000000u}, + {0x3f1f4f94u, 0x0000007fu, 0x00000000u}, + {0x41205004u, 0x00000083u, 0x00000000u}, + {0x42a11075u, 0x00010e86u, 0x00000000u}, + {0x44a210f4u, 0x0000008au, 0x00000000u}, + {0x4622d154u, 0x0000008du, 0x00000000u}, + {0x4823d1c4u, 0x00000091u, 0x00000000u}, + {0x49a49234u, 0x00000094u, 0x00000000u}, + {0x4b255293u, 0x00000000u, 0x00000000u}, + {0x0025d2d2u, 0x00000000u, 0x00000000u}, + {0x00265302u, 0x00000000u, 0x00000000u}, + {0x0026d342u, 0x00000000u, 0x00000000u}, + {0x4ea71373u, 0x00000000u, 0x00000000u}, + {0x4fa793b3u, 0x00000000u, 0x00000000u}, + {0x002813f2u, 0x00000000u, 0x00000000u}, + {0x00289422u, 0x00000000u, 0x00000000u}, + {0x00291462u, 0x00000000u, 0x00000000u}, + {0x53295493u, 0x00000000u, 0x00000000u}, + {0x5429d4d3u, 0x00000000u, 0x00000000u}, + {0x002a5512u, 0x00000000u, 0x00000000u}, + {0x002ad542u, 0x00000000u, 0x00000000u}, + {0x002b5582u, 0x00000000u, 0x00000000u}, + {0x002bd5c2u, 0x00000000u, 0x00000000u}, + {0x58ac15f3u, 0x00000000u, 0x00000000u}, + {0x002c9632u, 0x00000000u, 0x00000000u}, + {0x5aad1663u, 0x00000000u, 0x00000000u}, + {0x5bad96b3u, 0x00000000u, 0x00000000u}, + {0x5d2e5703u, 0x00000000u, 0x00000000u}, + {0x5e2ed753u, 0x00000000u, 0x00000000u}, + {0x002f97a2u, 0x00000000u, 0x00000000u}, + {0x60b017e3u, 0x00000000u, 0x00000000u}, + {0x61b09833u, 0x00000000u, 0x00000000u}, + {0x62b11874u, 0x000000c6u, 0x00000000u}, + {0x6431d8d3u, 0x00000000u, 0x00000000u}, + {0x65325913u, 0x00000000u, 0x00000000u}, + {0x6632d954u, 0x000000cdu, 0x00000000u}, + {0x67b399b3u, 0x00000000u, 0x00000000u}, + {0x68b419f3u, 0x00000000u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x0000c042u, 0x00000000u, 0x00000000u}, + {0x02810073u, 0x00000000u, 0x00000000u}, + {0x038180b3u, 0x00000000u, 0x00000000u}, + {0x048200f3u, 0x00000000u, 0x00000000u}, + {0x05828133u, 0x00000000u, 0x00000000u}, + {0x00030172u, 0x00000000u, 0x00000000u}, + {0x07034193u, 0x00000000u, 0x00000000u}, + {0x0803c1d3u, 0x00000000u, 0x00000000u}, + {0x00044212u, 0x00000000u, 0x00000000u}, + {0x0004c242u, 0x00000000u, 0x00000000u}, + {0x00054282u, 0x00000000u, 0x00000000u}, + {0x0b8582b3u, 0x00000000u, 0x00000000u}, + {0x0c8602f3u, 0x00000000u, 0x00000000u}, + {0x00068332u, 0x00000000u, 0x00000000u}, + {0x00070362u, 0x00000000u, 0x00000000u}, + {0x0f074393u, 0x00000000u, 0x00000000u}, + {0x1007c3d3u, 0x00000000u, 0x00000000u}, + {0x11084413u, 0x00000000u, 0x00000000u}, + {0x0008c452u, 0x00000000u, 0x00000000u}, + {0x00094482u, 0x00000000u, 0x00000000u}, + {0x138984b3u, 0x00000000u, 0x00000000u}, + {0x000a04f2u, 0x00000000u, 0x00000000u}, + {0x000a8522u, 0x00000000u, 0x00000000u}, + {0x000b0562u, 0x00000000u, 0x00000000u}, + {0x170b4593u, 0x00000000u, 0x00000000u}, + {0x180bc5d3u, 0x00000000u, 0x00000000u}, + {0x190c4613u, 0x00000000u, 0x00000000u}, + {0x000cc652u, 0x00000000u, 0x00000000u}, + {0x000d4682u, 0x00000000u, 0x00000000u}, + {0x1b8d86b3u, 0x00000000u, 0x00000000u}, + {0x1c8e06f3u, 0x00000000u, 0x00000000u}, + {0x000e8732u, 0x00000000u, 0x00000000u}, + {0x000f0762u, 0x00000000u, 0x00000000u}, + {0x1f0f4793u, 0x00000000u, 0x00000000u}, + {0x200fc7d3u, 0x00000000u, 0x00000000u}, + {0x21104813u, 0x00000000u, 0x00000000u}, + {0x2210c853u, 0x00000000u, 0x00000000u}, + {0x23114893u, 0x00000000u, 0x00000000u}, + {0x249208e3u, 0x00000000u, 0x00000000u}, + {0x25928933u, 0x00000000u, 0x00000000u}, + {0x00134982u, 0x00000000u, 0x00000000u}, + {0x2813c9c3u, 0x00000000u, 0x00000000u}, + {0x29144a13u, 0x00000000u, 0x00000000u}, + {0x2a14ca53u, 0x00000000u, 0x00000000u}, + {0x2b958aa3u, 0x00000000u, 0x00000000u}, + {0x2c960af3u, 0x00000000u, 0x00000000u}, + {0x0016cb42u, 0x00000000u, 0x00000000u}, + {0x2f174b83u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x31184c13u, 0x00000000u, 0x00000000u}, + {0x3218cc53u, 0x00000000u, 0x00000000u}, + {0x33998ca3u, 0x00000000u, 0x00000000u}, + {0x351a4d03u, 0x00000000u, 0x00000000u}, + {0x369b0d64u, 0x0000006eu, 0x00000000u}, + {0x381bcdd5u, 0x0000e471u, 0x00000000u}, + {0x3a1cce54u, 0x00000075u, 0x00000000u}, + {0x3b9d8eb4u, 0x00000078u, 0x00000000u}, + {0x3d1e4f14u, 0x0000007bu, 0x00000000u}, + {0x3f1f4f84u, 0x0000007fu, 0x00000000u}, + {0x40a00ff4u, 0x00000082u, 0x00000000u}, + {0x42a11063u, 0x00000000u, 0x00000000u}, + {0x4421d0c4u, 0x00000089u, 0x00000000u}, + {0x45a29134u, 0x0000008cu, 0x00000000u}, + {0x47a391a4u, 0x00000090u, 0x00000000u}, + {0x49245214u, 0x00000093u, 0x00000000u}, + {0x4aa51273u, 0x00000000u, 0x00000000u}, + {0x4ba592b3u, 0x00000000u, 0x00000000u}, + {0x4ca612f3u, 0x00000000u, 0x00000000u}, + {0x00269332u, 0x00000000u, 0x00000000u}, + {0x00271362u, 0x00000000u, 0x00000000u}, + {0x4f275393u, 0x00000000u, 0x00000000u}, + {0x5027d3d3u, 0x00000000u, 0x00000000u}, + {0x00285412u, 0x00000000u, 0x00000000u}, + {0x0028d442u, 0x00000000u, 0x00000000u}, + {0x52a91473u, 0x00000000u, 0x00000000u}, + {0x53a994b3u, 0x00000000u, 0x00000000u}, + {0x54aa14f3u, 0x00000000u, 0x00000000u}, + {0x002a9532u, 0x00000000u, 0x00000000u}, + {0x562ad553u, 0x00000000u, 0x00000000u}, + {0x572b5593u, 0x00000000u, 0x00000000u}, + {0x582bd5d3u, 0x00000000u, 0x00000000u}, + {0x002c5612u, 0x00000000u, 0x00000000u}, + {0x002cd642u, 0x00000000u, 0x00000000u}, + {0x002d5682u, 0x00000000u, 0x00000000u}, + {0x5c2dd6c3u, 0x00000000u, 0x00000000u}, + {0x5d2e5713u, 0x00000000u, 0x00000000u}, + {0x5e2ed753u, 0x00000000u, 0x00000000u}, + {0x5faf97a3u, 0x00000000u, 0x00000000u}, + {0x60b017f3u, 0x00000000u, 0x00000000u}, + {0x0030d842u, 0x00000000u, 0x00000000u}, + {0x63315883u, 0x00000000u, 0x00000000u}, + {0x6431d8d3u, 0x00000000u, 0x00000000u}, + {0x65325913u, 0x00000000u, 0x00000000u}, + {0x66b31963u, 0x00000000u, 0x00000000u}, + {0x67b399b3u, 0x00000000u, 0x00000000u}, + {0x68b419f3u, 0x00000000u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x0000c042u, 0x00000000u, 0x00000000u}, + {0x02810073u, 0x00000000u, 0x00000000u}, + {0x038180b3u, 0x00000000u, 0x00000000u}, + {0x000200f2u, 0x00000000u, 0x00000000u}, + {0x00028122u, 0x00000000u, 0x00000000u}, + {0x0602c153u, 0x00000000u, 0x00000000u}, + {0x07034193u, 0x00000000u, 0x00000000u}, + {0x0803c1d3u, 0x00000000u, 0x00000000u}, + {0x00044212u, 0x00000000u, 0x00000000u}, + {0x09848233u, 0x00000000u, 0x00000000u}, + {0x0a850273u, 0x00000000u, 0x00000000u}, + {0x000582b2u, 0x00000000u, 0x00000000u}, + {0x000602e2u, 0x00000000u, 0x00000000u}, + {0x0d064313u, 0x00000000u, 0x00000000u}, + {0x0e06c353u, 0x00000000u, 0x00000000u}, + {0x00074392u, 0x00000000u, 0x00000000u}, + {0x0007c3c2u, 0x00000000u, 0x00000000u}, + {0x108803f3u, 0x00000000u, 0x00000000u}, + {0x11888433u, 0x00000000u, 0x00000000u}, + {0x00090472u, 0x00000000u, 0x00000000u}, + {0x13094493u, 0x00000000u, 0x00000000u}, + {0x1409c4d3u, 0x00000000u, 0x00000000u}, + {0x150a4513u, 0x00000000u, 0x00000000u}, + {0x000ac552u, 0x00000000u, 0x00000000u}, + {0x000b4582u, 0x00000000u, 0x00000000u}, + {0x178b85b3u, 0x00000000u, 0x00000000u}, + {0x188c05f3u, 0x00000000u, 0x00000000u}, + {0x000c8632u, 0x00000000u, 0x00000000u}, + {0x1a0cc653u, 0x00000000u, 0x00000000u}, + {0x1b0d4693u, 0x00000000u, 0x00000000u}, + {0x000dc6d2u, 0x00000000u, 0x00000000u}, + {0x000e4702u, 0x00000000u, 0x00000000u}, + {0x1d8e8733u, 0x00000000u, 0x00000000u}, + {0x1e8f0773u, 0x00000000u, 0x00000000u}, + {0x1f8f87b3u, 0x00000000u, 0x00000000u}, + {0x001007f2u, 0x00000000u, 0x00000000u}, + {0x21908823u, 0x00000000u, 0x00000000u}, + {0x22910873u, 0x00000000u, 0x00000000u}, + {0x239188b3u, 0x00000000u, 0x00000000u}, + {0x249208f3u, 0x00000000u, 0x00000000u}, + {0x25928933u, 0x00000000u, 0x00000000u}, + {0x27134983u, 0x00000000u, 0x00000000u}, + {0x2813c9d3u, 0x00000000u, 0x00000000u}, + {0x29144a14u, 0x00000053u, 0x00000000u}, + {0x2a950a73u, 0x00000000u, 0x00000000u}, + {0x2b958ab3u, 0x00000000u, 0x00000000u}, + {0x2c960af3u, 0x00000000u, 0x00000000u}, + {0x2d968b33u, 0x00000000u, 0x00000000u}, + {0x2f174b83u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x31184c13u, 0x00000000u, 0x00000000u}, + {0x3218cc53u, 0x00000000u, 0x00000000u}, + {0x33194c94u, 0x00000067u, 0x00000000u}, + {0x349a0cf4u, 0x0000006au, 0x00000000u}, + {0x361acd54u, 0x0000006du, 0x00000000u}, + {0x381bcdc4u, 0x00000071u, 0x00000000u}, + {0x399c8e34u, 0x00000074u, 0x00000000u}, + {0x3b9d8ea4u, 0x00000078u, 0x00000000u}, + {0x3d1e4f14u, 0x0000007bu, 0x00000000u}, + {0x3e9f0f74u, 0x0000007eu, 0x00000000u}, + {0x401fcfd4u, 0x00000081u, 0x00000000u}, + {0x4220d044u, 0x00000085u, 0x00000000u}, + {0x43a190b4u, 0x00000088u, 0x00000000u}, + {0x45a29123u, 0x00000000u, 0x00000000u}, + {0x47235183u, 0x00000000u, 0x00000000u}, + {0x48a411e4u, 0x00000092u, 0x00000000u}, + {0x4a24d253u, 0x00000000u, 0x00000000u}, + {0x002592a2u, 0x00000000u, 0x00000000u}, + {0x4c25d2d3u, 0x00000000u, 0x00000000u}, + {0x4d265313u, 0x00000000u, 0x00000000u}, + {0x4e26d353u, 0x00000000u, 0x00000000u}, + {0x00275392u, 0x00000000u, 0x00000000u}, + {0x4fa793b3u, 0x00000000u, 0x00000000u}, + {0x002813f2u, 0x00000000u, 0x00000000u}, + {0x00289422u, 0x00000000u, 0x00000000u}, + {0x5228d453u, 0x00000000u, 0x00000000u}, + {0x53295493u, 0x00000000u, 0x00000000u}, + {0x5429d4d3u, 0x00000000u, 0x00000000u}, + {0x002a5512u, 0x00000000u, 0x00000000u}, + {0x002ad542u, 0x00000000u, 0x00000000u}, + {0x56ab1573u, 0x00000000u, 0x00000000u}, + {0x57ab95b3u, 0x00000000u, 0x00000000u}, + {0x002c15f2u, 0x00000000u, 0x00000000u}, + {0x592c5613u, 0x00000000u, 0x00000000u}, + {0x5a2cd653u, 0x00000000u, 0x00000000u}, + {0x5b2d5693u, 0x00000000u, 0x00000000u}, + {0x5c2dd6d3u, 0x00000000u, 0x00000000u}, + {0x5dae9723u, 0x00000000u, 0x00000000u}, + {0x5eaf1773u, 0x00000000u, 0x00000000u}, + {0x5faf97b3u, 0x00000000u, 0x00000000u}, + {0x60b017f3u, 0x00000000u, 0x00000000u}, + {0x6230d843u, 0x00000000u, 0x00000000u}, + {0x63315893u, 0x00000000u, 0x00000000u}, + {0x6431d8d3u, 0x00000000u, 0x00000000u}, + {0x65b29923u, 0x00000000u, 0x00000000u}, + {0x66b31973u, 0x00000000u, 0x00000000u}, + {0x67b399b3u, 0x00000000u, 0x00000000u}, + {0x68b419f3u, 0x00000000u, 0x00000000u}, + {0x69b49a33u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x01808033u, 0x00000000u, 0x00000000u}, + {0x02810073u, 0x00000000u, 0x00000000u}, + {0x000180b2u, 0x00000000u, 0x00000000u}, + {0x000200e2u, 0x00000000u, 0x00000000u}, + {0x05024113u, 0x00000000u, 0x00000000u}, + {0x0602c153u, 0x00000000u, 0x00000000u}, + {0x00034192u, 0x00000000u, 0x00000000u}, + {0x0003c1c2u, 0x00000000u, 0x00000000u}, + {0x088401f3u, 0x00000000u, 0x00000000u}, + {0x09848233u, 0x00000000u, 0x00000000u}, + {0x00050272u, 0x00000000u, 0x00000000u}, + {0x000582a2u, 0x00000000u, 0x00000000u}, + {0x0c05c2d3u, 0x00000000u, 0x00000000u}, + {0x00064312u, 0x00000000u, 0x00000000u}, + {0x0006c342u, 0x00000000u, 0x00000000u}, + {0x0e870373u, 0x00000000u, 0x00000000u}, + {0x000783b2u, 0x00000000u, 0x00000000u}, + {0x000803e2u, 0x00000000u, 0x00000000u}, + {0x11084413u, 0x00000000u, 0x00000000u}, + {0x1208c453u, 0x00000000u, 0x00000000u}, + {0x00094492u, 0x00000000u, 0x00000000u}, + {0x0009c4c2u, 0x00000000u, 0x00000000u}, + {0x148a04f3u, 0x00000000u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x000b0572u, 0x00000000u, 0x00000000u}, + {0x170b4593u, 0x00000000u, 0x00000000u}, + {0x180bc5d3u, 0x00000000u, 0x00000000u}, + {0x000c4612u, 0x00000000u, 0x00000000u}, + {0x198c8633u, 0x00000000u, 0x00000000u}, + {0x000d0672u, 0x00000000u, 0x00000000u}, + {0x000d86a2u, 0x00000000u, 0x00000000u}, + {0x1c0dc6d3u, 0x00000000u, 0x00000000u}, + {0x1d0e4713u, 0x00000000u, 0x00000000u}, + {0x000ec752u, 0x00000000u, 0x00000000u}, + {0x000f4782u, 0x00000000u, 0x00000000u}, + {0x1f8f87b3u, 0x00000000u, 0x00000000u}, + {0x209007f3u, 0x00000000u, 0x00000000u}, + {0x21908833u, 0x00000000u, 0x00000000u}, + {0x22910873u, 0x00000000u, 0x00000000u}, + {0x2411c8c3u, 0x00000000u, 0x00000000u}, + {0x25124913u, 0x00000000u, 0x00000000u}, + {0x2612c953u, 0x00000000u, 0x00000000u}, + {0x27134993u, 0x00000000u, 0x00000000u}, + {0x2813c9d3u, 0x00000000u, 0x00000000u}, + {0x29144a13u, 0x00000000u, 0x00000000u}, + {0x2a950a63u, 0x00000000u, 0x00000000u}, + {0x2b958ab3u, 0x00000000u, 0x00000000u}, + {0x2c960af3u, 0x00000000u, 0x00000000u}, + {0x2e16cb43u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x31184c13u, 0x00000000u, 0x00000000u}, + {0x3218cc53u, 0x00000000u, 0x00000000u}, + {0x33194c94u, 0x00000067u, 0x00000000u}, + {0x349a0cf4u, 0x0000006au, 0x00000000u}, + {0x361acd54u, 0x0000006du, 0x00000000u}, + {0x379b8db4u, 0x00000070u, 0x00000000u}, + {0x399c8e24u, 0x00000074u, 0x00000000u}, + {0x3b1d4e94u, 0x00000077u, 0x00000000u}, + {0x3c9e0ef4u, 0x0000007au, 0x00000000u}, + {0x3e9f0f64u, 0x0000007eu, 0x00000000u}, + {0x401fcfd4u, 0x00000081u, 0x00000000u}, + {0x41a09033u, 0x00000000u, 0x00000000u}, + {0x43215084u, 0x00000087u, 0x00000000u}, + {0x44a210f4u, 0x0000008au, 0x00000000u}, + {0x46a31163u, 0x00000000u, 0x00000000u}, + {0x4823d1c4u, 0x00000091u, 0x00000000u}, + {0x49a49234u, 0x00000094u, 0x00000000u}, + {0x4b255293u, 0x00000000u, 0x00000000u}, + {0x0025d2d2u, 0x00000000u, 0x00000000u}, + {0x00265302u, 0x00000000u, 0x00000000u}, + {0x4da69333u, 0x00000000u, 0x00000000u}, + {0x4ea71373u, 0x00000000u, 0x00000000u}, + {0x002793b2u, 0x00000000u, 0x00000000u}, + {0x5027d3d3u, 0x00000000u, 0x00000000u}, + {0x00285412u, 0x00000000u, 0x00000000u}, + {0x51a89433u, 0x00000000u, 0x00000000u}, + {0x52a91473u, 0x00000000u, 0x00000000u}, + {0x002994b2u, 0x00000000u, 0x00000000u}, + {0x002a14e2u, 0x00000000u, 0x00000000u}, + {0x552a5513u, 0x00000000u, 0x00000000u}, + {0x562ad553u, 0x00000000u, 0x00000000u}, + {0x002b5592u, 0x00000000u, 0x00000000u}, + {0x002bd5c2u, 0x00000000u, 0x00000000u}, + {0x58ac15f3u, 0x00000000u, 0x00000000u}, + {0x59ac9633u, 0x00000000u, 0x00000000u}, + {0x5aad1673u, 0x00000000u, 0x00000000u}, + {0x5bad96b3u, 0x00000000u, 0x00000000u}, + {0x5cae16f3u, 0x00000000u, 0x00000000u}, + {0x5dae9733u, 0x00000000u, 0x00000000u}, + {0x5eaf1773u, 0x00000000u, 0x00000000u}, + {0x602fd7c3u, 0x00000000u, 0x00000000u}, + {0x61305813u, 0x00000000u, 0x00000000u}, + {0x6230d853u, 0x00000000u, 0x00000000u}, + {0x63315893u, 0x00000000u, 0x00000000u}, + {0x64b218e3u, 0x00000000u, 0x00000000u}, + {0x65b29933u, 0x00000000u, 0x00000000u}, + {0x66b31973u, 0x00000000u, 0x00000000u}, + {0x0033d9c2u, 0x00000000u, 0x00000000u}, + {0x00345a02u, 0x00000000u, 0x00000000u}, + {0x0034da42u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x01808033u, 0x00000000u, 0x00000000u}, + {0x02810073u, 0x00000000u, 0x00000000u}, + {0x000180b2u, 0x00000000u, 0x00000000u}, + {0x0401c0d3u, 0x00000000u, 0x00000000u}, + {0x05024113u, 0x00000000u, 0x00000000u}, + {0x0002c152u, 0x00000000u, 0x00000000u}, + {0x00034182u, 0x00000000u, 0x00000000u}, + {0x078381b3u, 0x00000000u, 0x00000000u}, + {0x000401f2u, 0x00000000u, 0x00000000u}, + {0x00048222u, 0x00000000u, 0x00000000u}, + {0x0a04c253u, 0x00000000u, 0x00000000u}, + {0x00054292u, 0x00000000u, 0x00000000u}, + {0x0005c2c2u, 0x00000000u, 0x00000000u}, + {0x0c8602f3u, 0x00000000u, 0x00000000u}, + {0x0d868333u, 0x00000000u, 0x00000000u}, + {0x00070372u, 0x00000000u, 0x00000000u}, + {0x0f074393u, 0x00000000u, 0x00000000u}, + {0x1007c3d3u, 0x00000000u, 0x00000000u}, + {0x00084412u, 0x00000000u, 0x00000000u}, + {0x0008c442u, 0x00000000u, 0x00000000u}, + {0x12890473u, 0x00000000u, 0x00000000u}, + {0x000984b2u, 0x00000000u, 0x00000000u}, + {0x000a04e2u, 0x00000000u, 0x00000000u}, + {0x150a4513u, 0x00000000u, 0x00000000u}, + {0x000ac552u, 0x00000000u, 0x00000000u}, + {0x000b4582u, 0x00000000u, 0x00000000u}, + {0x178b85b3u, 0x00000000u, 0x00000000u}, + {0x000c05f2u, 0x00000000u, 0x00000000u}, + {0x000c8622u, 0x00000000u, 0x00000000u}, + {0x000cc652u, 0x00000000u, 0x00000000u}, + {0x000d4682u, 0x00000000u, 0x00000000u}, + {0x1b8d86b3u, 0x00000000u, 0x00000000u}, + {0x1c8e06f3u, 0x00000000u, 0x00000000u}, + {0x000e8732u, 0x00000000u, 0x00000000u}, + {0x1e0ec753u, 0x00000000u, 0x00000000u}, + {0x1f0f4793u, 0x00000000u, 0x00000000u}, + {0x000fc7d2u, 0x00000000u, 0x00000000u}, + {0x00104802u, 0x00000000u, 0x00000000u}, + {0x21908833u, 0x00000000u, 0x00000000u}, + {0x23114883u, 0x00000000u, 0x00000000u}, + {0x2411c8d3u, 0x00000000u, 0x00000000u}, + {0x25124913u, 0x00000000u, 0x00000000u}, + {0x2612c953u, 0x00000000u, 0x00000000u}, + {0x27134993u, 0x00000000u, 0x00000000u}, + {0x289409e3u, 0x00000000u, 0x00000000u}, + {0x29948a33u, 0x00000000u, 0x00000000u}, + {0x2a950a73u, 0x00000000u, 0x00000000u}, + {0x2b958ab3u, 0x00000000u, 0x00000000u}, + {0x2d164b03u, 0x00000000u, 0x00000000u}, + {0x2e16cb53u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x31184c13u, 0x00000000u, 0x00000000u}, + {0x3218cc53u, 0x00000000u, 0x00000000u}, + {0x33194c94u, 0x00000067u, 0x00000000u}, + {0x349a0cf3u, 0x00000000u, 0x00000000u}, + {0x361acd44u, 0x0000006du, 0x00000000u}, + {0x379b8db4u, 0x00000070u, 0x00000000u}, + {0x391c4e14u, 0x00000073u, 0x00000000u}, + {0x3a9d0e74u, 0x00000076u, 0x00000000u}, + {0x3c9e0ee3u, 0x00000000u, 0x00000000u}, + {0x3e1ecf44u, 0x0000007du, 0x00000000u}, + {0x3f9f8fb4u, 0x00000080u, 0x00000000u}, + {0x41205014u, 0x00000083u, 0x00000000u}, + {0x42a11074u, 0x00000086u, 0x00000000u}, + {0x44a210e3u, 0x00000000u, 0x00000000u}, + {0x4622d144u, 0x0000008du, 0x00000000u}, + {0x47a391b4u, 0x00000090u, 0x00000000u}, + {0x49245214u, 0x00000093u, 0x00000000u}, + {0x00255282u, 0x00000000u, 0x00000000u}, + {0x4ba592b3u, 0x00000000u, 0x00000000u}, + {0x002612f2u, 0x00000000u, 0x00000000u}, + {0x00269322u, 0x00000000u, 0x00000000u}, + {0x4e26d353u, 0x00000000u, 0x00000000u}, + {0x00275392u, 0x00000000u, 0x00000000u}, + {0x4fa793b3u, 0x00000000u, 0x00000000u}, + {0x002813f2u, 0x00000000u, 0x00000000u}, + {0x00289422u, 0x00000000u, 0x00000000u}, + {0x5228d453u, 0x00000000u, 0x00000000u}, + {0x00295492u, 0x00000000u, 0x00000000u}, + {0x0029d4c2u, 0x00000000u, 0x00000000u}, + {0x54aa14f3u, 0x00000000u, 0x00000000u}, + {0x55aa9533u, 0x00000000u, 0x00000000u}, + {0x002b1572u, 0x00000000u, 0x00000000u}, + {0x572b5593u, 0x00000000u, 0x00000000u}, + {0x582bd5d3u, 0x00000000u, 0x00000000u}, + {0x002c5612u, 0x00000000u, 0x00000000u}, + {0x002cd642u, 0x00000000u, 0x00000000u}, + {0x5aad1673u, 0x00000000u, 0x00000000u}, + {0x5bad96b3u, 0x00000000u, 0x00000000u}, + {0x002e5702u, 0x00000000u, 0x00000000u}, + {0x5e2ed743u, 0x00000000u, 0x00000000u}, + {0x5f2f5793u, 0x00000000u, 0x00000000u}, + {0x602fd7d3u, 0x00000000u, 0x00000000u}, + {0x61305813u, 0x00000000u, 0x00000000u}, + {0x62b11863u, 0x00000000u, 0x00000000u}, + {0x63b198b3u, 0x00000000u, 0x00000000u}, + {0x64b218f3u, 0x00000000u, 0x00000000u}, + {0x65b29933u, 0x00000000u, 0x00000000u}, + {0x66b31973u, 0x00000000u, 0x00000000u}, + {0x0033d9c2u, 0x00000000u, 0x00000000u}, + {0x00345a02u, 0x00000000u, 0x00000000u}, + {0x0034da42u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x01808033u, 0x00000000u, 0x00000000u}, + {0x00010072u, 0x00000000u, 0x00000000u}, + {0x000180a2u, 0x00000000u, 0x00000000u}, + {0x0401c0d3u, 0x00000000u, 0x00000000u}, + {0x00024112u, 0x00000000u, 0x00000000u}, + {0x0002c142u, 0x00000000u, 0x00000000u}, + {0x06830173u, 0x00000000u, 0x00000000u}, + {0x000381b2u, 0x00000000u, 0x00000000u}, + {0x000401e2u, 0x00000000u, 0x00000000u}, + {0x09044213u, 0x00000000u, 0x00000000u}, + {0x0004c252u, 0x00000000u, 0x00000000u}, + {0x0a850273u, 0x00000000u, 0x00000000u}, + {0x0b8582b3u, 0x00000000u, 0x00000000u}, + {0x000602f2u, 0x00000000u, 0x00000000u}, + {0x0d064313u, 0x00000000u, 0x00000000u}, + {0x0006c352u, 0x00000000u, 0x00000000u}, + {0x00074382u, 0x00000000u, 0x00000000u}, + {0x0f8783b3u, 0x00000000u, 0x00000000u}, + {0x000803f2u, 0x00000000u, 0x00000000u}, + {0x00088422u, 0x00000000u, 0x00000000u}, + {0x1208c453u, 0x00000000u, 0x00000000u}, + {0x00094492u, 0x00000000u, 0x00000000u}, + {0x0009c4c2u, 0x00000000u, 0x00000000u}, + {0x148a04f3u, 0x00000000u, 0x00000000u}, + {0x000a8532u, 0x00000000u, 0x00000000u}, + {0x160ac553u, 0x00000000u, 0x00000000u}, + {0x170b4593u, 0x00000000u, 0x00000000u}, + {0x000bc5d2u, 0x00000000u, 0x00000000u}, + {0x188c05f3u, 0x00000000u, 0x00000000u}, + {0x000c8632u, 0x00000000u, 0x00000000u}, + {0x000d0662u, 0x00000000u, 0x00000000u}, + {0x1b0d4693u, 0x00000000u, 0x00000000u}, + {0x000dc6d2u, 0x00000000u, 0x00000000u}, + {0x000e4702u, 0x00000000u, 0x00000000u}, + {0x1d8e8733u, 0x00000000u, 0x00000000u}, + {0x000f0772u, 0x00000000u, 0x00000000u}, + {0x000f87a2u, 0x00000000u, 0x00000000u}, + {0x200fc7d3u, 0x00000000u, 0x00000000u}, + {0x21104813u, 0x00000000u, 0x00000000u}, + {0x2210c853u, 0x00000000u, 0x00000000u}, + {0x23114893u, 0x00000000u, 0x00000000u}, + {0x2411c8d3u, 0x00000000u, 0x00000000u}, + {0x25124913u, 0x00000000u, 0x00000000u}, + {0x2612c953u, 0x00000000u, 0x00000000u}, + {0x001389a2u, 0x00000000u, 0x00000000u}, + {0x289409e3u, 0x00000000u, 0x00000000u}, + {0x29948a33u, 0x00000000u, 0x00000000u}, + {0x2a950a73u, 0x00000000u, 0x00000000u}, + {0x2b958ab3u, 0x00000000u, 0x00000000u}, + {0x2c960af3u, 0x00000000u, 0x00000000u}, + {0x2d968b33u, 0x00000000u, 0x00000000u}, + {0x2f174b83u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x00184c12u, 0x00000000u, 0x00000000u}, + {0x0018cc42u, 0x00000000u, 0x00000000u}, + {0x32990c74u, 0x00000066u, 0x00000000u}, + {0x3419ccd4u, 0x00000069u, 0x00000000u}, + {0x359a8d34u, 0x0000006cu, 0x00000000u}, + {0x379b8da3u, 0x00000000u, 0x00000000u}, + {0x391c4e03u, 0x00000000u, 0x00000000u}, + {0x3a9d0e63u, 0x00000000u, 0x00000000u}, + {0x3c1dcec4u, 0x00000079u, 0x00000000u}, + {0x3d9e8f34u, 0x0000007cu, 0x00000000u}, + {0x3f1f4f94u, 0x0000007fu, 0x00000000u}, + {0x40a00ff4u, 0x00000082u, 0x00000000u}, + {0x4220d054u, 0x00000085u, 0x00000000u}, + {0x43a190b4u, 0x00000088u, 0x00000000u}, + {0x45225114u, 0x0000008bu, 0x00000000u}, + {0x47235183u, 0x00000000u, 0x00000000u}, + {0x48a411e3u, 0x00000000u, 0x00000000u}, + {0x4a24d243u, 0x00000000u, 0x00000000u}, + {0x4b255293u, 0x00000000u, 0x00000000u}, + {0x0025d2d2u, 0x00000000u, 0x00000000u}, + {0x00265302u, 0x00000000u, 0x00000000u}, + {0x4da69333u, 0x00000000u, 0x00000000u}, + {0x00271372u, 0x00000000u, 0x00000000u}, + {0x4f275393u, 0x00000000u, 0x00000000u}, + {0x5027d3d3u, 0x00000000u, 0x00000000u}, + {0x00285412u, 0x00000000u, 0x00000000u}, + {0x51a89433u, 0x00000000u, 0x00000000u}, + {0x00291472u, 0x00000000u, 0x00000000u}, + {0x002994a2u, 0x00000000u, 0x00000000u}, + {0x5429d4d3u, 0x00000000u, 0x00000000u}, + {0x002a5512u, 0x00000000u, 0x00000000u}, + {0x002ad542u, 0x00000000u, 0x00000000u}, + {0x56ab1573u, 0x00000000u, 0x00000000u}, + {0x002b95b2u, 0x00000000u, 0x00000000u}, + {0x002c15e2u, 0x00000000u, 0x00000000u}, + {0x592c5613u, 0x00000000u, 0x00000000u}, + {0x002cd652u, 0x00000000u, 0x00000000u}, + {0x002d5682u, 0x00000000u, 0x00000000u}, + {0x5c2dd6c3u, 0x00000000u, 0x00000000u}, + {0x5d2e5713u, 0x00000000u, 0x00000000u}, + {0x5e2ed753u, 0x00000000u, 0x00000000u}, + {0x5f2f5793u, 0x00000000u, 0x00000000u}, + {0x602fd7d3u, 0x00000000u, 0x00000000u}, + {0x61305813u, 0x00000000u, 0x00000000u}, + {0x00311862u, 0x00000000u, 0x00000000u}, + {0x63b198a3u, 0x00000000u, 0x00000000u}, + {0x64b218f3u, 0x00000000u, 0x00000000u}, + {0x65b29933u, 0x00000000u, 0x00000000u}, + {0x66b31973u, 0x00000000u, 0x00000000u}, + {0x67b399b3u, 0x00000000u, 0x00000000u}, + {0x68b419f3u, 0x00000000u, 0x00000000u}, + {0x0034da42u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x0000c042u, 0x00000000u, 0x00000000u}, + {0x02810073u, 0x00000000u, 0x00000000u}, + {0x000180b2u, 0x00000000u, 0x00000000u}, + {0x000200e2u, 0x00000000u, 0x00000000u}, + {0x05024113u, 0x00000000u, 0x00000000u}, + {0x0002c152u, 0x00000000u, 0x00000000u}, + {0x06830173u, 0x00000000u, 0x00000000u}, + {0x000381b2u, 0x00000000u, 0x00000000u}, + {0x000401e2u, 0x00000000u, 0x00000000u}, + {0x09044213u, 0x00000000u, 0x00000000u}, + {0x0004c252u, 0x00000000u, 0x00000000u}, + {0x0a850273u, 0x00000000u, 0x00000000u}, + {0x000582b2u, 0x00000000u, 0x00000000u}, + {0x000602e2u, 0x00000000u, 0x00000000u}, + {0x0d064313u, 0x00000000u, 0x00000000u}, + {0x0006c352u, 0x00000000u, 0x00000000u}, + {0x0e870373u, 0x00000000u, 0x00000000u}, + {0x000783b2u, 0x00000000u, 0x00000000u}, + {0x000803e2u, 0x00000000u, 0x00000000u}, + {0x11084413u, 0x00000000u, 0x00000000u}, + {0x0008c452u, 0x00000000u, 0x00000000u}, + {0x12890473u, 0x00000000u, 0x00000000u}, + {0x000984b2u, 0x00000000u, 0x00000000u}, + {0x000a04e2u, 0x00000000u, 0x00000000u}, + {0x150a4513u, 0x00000000u, 0x00000000u}, + {0x000ac552u, 0x00000000u, 0x00000000u}, + {0x168b0573u, 0x00000000u, 0x00000000u}, + {0x000b85b2u, 0x00000000u, 0x00000000u}, + {0x000c05e2u, 0x00000000u, 0x00000000u}, + {0x190c4613u, 0x00000000u, 0x00000000u}, + {0x000cc652u, 0x00000000u, 0x00000000u}, + {0x1a8d0673u, 0x00000000u, 0x00000000u}, + {0x1b8d86b3u, 0x00000000u, 0x00000000u}, + {0x000e06f2u, 0x00000000u, 0x00000000u}, + {0x000e8722u, 0x00000000u, 0x00000000u}, + {0x1e0ec753u, 0x00000000u, 0x00000000u}, + {0x000f4792u, 0x00000000u, 0x00000000u}, + {0x1f8f87b3u, 0x00000000u, 0x00000000u}, + {0x001007f2u, 0x00000000u, 0x00000000u}, + {0x00108822u, 0x00000000u, 0x00000000u}, + {0x22910863u, 0x00000000u, 0x00000000u}, + {0x239188b3u, 0x00000000u, 0x00000000u}, + {0x249208f3u, 0x00000000u, 0x00000000u}, + {0x25928933u, 0x00000000u, 0x00000000u}, + {0x26930973u, 0x00000000u, 0x00000000u}, + {0x279389b3u, 0x00000000u, 0x00000000u}, + {0x289409f3u, 0x00000000u, 0x00000000u}, + {0x29948a33u, 0x00000000u, 0x00000000u}, + {0x2a950a73u, 0x00000000u, 0x00000000u}, + {0x0015cac2u, 0x00000000u, 0x00000000u}, + {0x00164b02u, 0x00000000u, 0x00000000u}, + {0x2e16cb43u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x00184c12u, 0x00000000u, 0x00000000u}, + {0x0018cc42u, 0x00000000u, 0x00000000u}, + {0x32990c74u, 0x00000066u, 0x00000000u}, + {0x3419ccd4u, 0x00000069u, 0x00000000u}, + {0x359a8d35u, 0x0000da6cu, 0x00000000u}, + {0x379b8db4u, 0x00000070u, 0x00000000u}, + {0x391c4e14u, 0x00000073u, 0x00000000u}, + {0x3a9d0e74u, 0x00000076u, 0x00000000u}, + {0x3c1dced4u, 0x00000079u, 0x00000000u}, + {0x3d9e8f34u, 0x0000007cu, 0x00000000u}, + {0x3f1f4f94u, 0x0000007fu, 0x00000000u}, + {0x40a00ff4u, 0x00000082u, 0x00000000u}, + {0x4220d054u, 0x00000085u, 0x00000000u}, + {0x43a190b4u, 0x00000088u, 0x00000000u}, + {0x45225114u, 0x0000008bu, 0x00000000u}, + {0x46a31174u, 0x0000008eu, 0x00000000u}, + {0x4823d1d4u, 0x00000091u, 0x00000000u}, + {0x49a49234u, 0x00000094u, 0x00000000u}, + {0x00255292u, 0x00000000u, 0x00000000u}, + {0x4ba592b3u, 0x00000000u, 0x00000000u}, + {0x002612f2u, 0x00000000u, 0x00000000u}, + {0x00269322u, 0x00000000u, 0x00000000u}, + {0x4e26d353u, 0x00000000u, 0x00000000u}, + {0x00275392u, 0x00000000u, 0x00000000u}, + {0x4fa793b3u, 0x00000000u, 0x00000000u}, + {0x50a813f3u, 0x00000000u, 0x00000000u}, + {0x00289432u, 0x00000000u, 0x00000000u}, + {0x00291462u, 0x00000000u, 0x00000000u}, + {0x53295493u, 0x00000000u, 0x00000000u}, + {0x0029d4d2u, 0x00000000u, 0x00000000u}, + {0x54aa14f3u, 0x00000000u, 0x00000000u}, + {0x002a9532u, 0x00000000u, 0x00000000u}, + {0x002b1562u, 0x00000000u, 0x00000000u}, + {0x572b5593u, 0x00000000u, 0x00000000u}, + {0x002bd5d2u, 0x00000000u, 0x00000000u}, + {0x58ac15f3u, 0x00000000u, 0x00000000u}, + {0x002c9632u, 0x00000000u, 0x00000000u}, + {0x002d1662u, 0x00000000u, 0x00000000u}, + {0x5b2d5693u, 0x00000000u, 0x00000000u}, + {0x5c2dd6d3u, 0x00000000u, 0x00000000u}, + {0x002e9722u, 0x00000000u, 0x00000000u}, + {0x5eaf1763u, 0x00000000u, 0x00000000u}, + {0x5faf97b3u, 0x00000000u, 0x00000000u}, + {0x60b017f3u, 0x00000000u, 0x00000000u}, + {0x61b09833u, 0x00000000u, 0x00000000u}, + {0x62b11873u, 0x00000000u, 0x00000000u}, + {0x63b198b3u, 0x00000000u, 0x00000000u}, + {0x64b218f3u, 0x00000000u, 0x00000000u}, + {0x65b29933u, 0x00000000u, 0x00000000u}, + {0x66b31973u, 0x00000000u, 0x00000000u}, + {0x67b399b3u, 0x00000000u, 0x00000000u}, + {0x00345a02u, 0x00000000u, 0x00000000u}, + {0x0034da42u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x01808033u, 0x00000000u, 0x00000000u}, + {0x02810073u, 0x00000000u, 0x00000000u}, + {0x000180b2u, 0x00000000u, 0x00000000u}, + {0x0401c0d3u, 0x00000000u, 0x00000000u}, + {0x00024112u, 0x00000000u, 0x00000000u}, + {0x05828133u, 0x00000000u, 0x00000000u}, + {0x00030172u, 0x00000000u, 0x00000000u}, + {0x000381a2u, 0x00000000u, 0x00000000u}, + {0x0803c1d3u, 0x00000000u, 0x00000000u}, + {0x00044212u, 0x00000000u, 0x00000000u}, + {0x09848233u, 0x00000000u, 0x00000000u}, + {0x00050272u, 0x00000000u, 0x00000000u}, + {0x0b054293u, 0x00000000u, 0x00000000u}, + {0x0005c2d2u, 0x00000000u, 0x00000000u}, + {0x00064302u, 0x00000000u, 0x00000000u}, + {0x0d868333u, 0x00000000u, 0x00000000u}, + {0x00070372u, 0x00000000u, 0x00000000u}, + {0x000783a2u, 0x00000000u, 0x00000000u}, + {0x1007c3d3u, 0x00000000u, 0x00000000u}, + {0x00084412u, 0x00000000u, 0x00000000u}, + {0x11888433u, 0x00000000u, 0x00000000u}, + {0x00090472u, 0x00000000u, 0x00000000u}, + {0x13094493u, 0x00000000u, 0x00000000u}, + {0x0009c4d2u, 0x00000000u, 0x00000000u}, + {0x000a4502u, 0x00000000u, 0x00000000u}, + {0x000a8532u, 0x00000000u, 0x00000000u}, + {0x000b0562u, 0x00000000u, 0x00000000u}, + {0x170b4593u, 0x00000000u, 0x00000000u}, + {0x000bc5d2u, 0x00000000u, 0x00000000u}, + {0x188c05f3u, 0x00000000u, 0x00000000u}, + {0x000c8632u, 0x00000000u, 0x00000000u}, + {0x1a0cc653u, 0x00000000u, 0x00000000u}, + {0x1b0d4693u, 0x00000000u, 0x00000000u}, + {0x000dc6d2u, 0x00000000u, 0x00000000u}, + {0x000e4702u, 0x00000000u, 0x00000000u}, + {0x000e8732u, 0x00000000u, 0x00000000u}, + {0x000f0762u, 0x00000000u, 0x00000000u}, + {0x1f0f4793u, 0x00000000u, 0x00000000u}, + {0x000fc7d2u, 0x00000000u, 0x00000000u}, + {0x209007f3u, 0x00000000u, 0x00000000u}, + {0x21908833u, 0x00000000u, 0x00000000u}, + {0x22910873u, 0x00000000u, 0x00000000u}, + {0x239188b3u, 0x00000000u, 0x00000000u}, + {0x249208f3u, 0x00000000u, 0x00000000u}, + {0x25928933u, 0x00000000u, 0x00000000u}, + {0x26930973u, 0x00000000u, 0x00000000u}, + {0x279389b4u, 0x00000050u, 0x00000000u}, + {0x29144a13u, 0x00000000u, 0x00000000u}, + {0x2a14ca53u, 0x00000000u, 0x00000000u}, + {0x2b154a93u, 0x00000000u, 0x00000000u}, + {0x2c15cad3u, 0x00000000u, 0x00000000u}, + {0x2d164b13u, 0x00000000u, 0x00000000u}, + {0x2e16cb53u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x00184c12u, 0x00000000u, 0x00000000u}, + {0x0018cc42u, 0x00000000u, 0x00000000u}, + {0x32990c74u, 0x00000066u, 0x00000000u}, + {0x3419ccd4u, 0x00000069u, 0x00000000u}, + {0x359a8d34u, 0x0000006cu, 0x00000000u}, + {0x379b8da3u, 0x00000000u, 0x00000000u}, + {0x389c0df4u, 0x00000072u, 0x00000000u}, + {0x3a1cce54u, 0x00000075u, 0x00000000u}, + {0x3b9d8eb4u, 0x00000078u, 0x00000000u}, + {0x3d1e4f14u, 0x0000007bu, 0x00000000u}, + {0x3e9f0f74u, 0x0000007eu, 0x00000000u}, + {0x401fcfd4u, 0x00000081u, 0x00000000u}, + {0x41a09033u, 0x00000000u, 0x00000000u}, + {0x43215084u, 0x00000087u, 0x00000000u}, + {0x44a210f4u, 0x0000008au, 0x00000000u}, + {0x4622d154u, 0x0000008du, 0x00000000u}, + {0x47a391b4u, 0x00000090u, 0x00000000u}, + {0x49245214u, 0x00000093u, 0x00000000u}, + {0x4aa51273u, 0x00000000u, 0x00000000u}, + {0x002592b2u, 0x00000000u, 0x00000000u}, + {0x4c25d2d3u, 0x00000000u, 0x00000000u}, + {0x00265312u, 0x00000000u, 0x00000000u}, + {0x0026d342u, 0x00000000u, 0x00000000u}, + {0x00271372u, 0x00000000u, 0x00000000u}, + {0x002793a2u, 0x00000000u, 0x00000000u}, + {0x5027d3d3u, 0x00000000u, 0x00000000u}, + {0x51285413u, 0x00000000u, 0x00000000u}, + {0x0028d452u, 0x00000000u, 0x00000000u}, + {0x52a91473u, 0x00000000u, 0x00000000u}, + {0x002994b2u, 0x00000000u, 0x00000000u}, + {0x5429d4d3u, 0x00000000u, 0x00000000u}, + {0x002a5512u, 0x00000000u, 0x00000000u}, + {0x002ad542u, 0x00000000u, 0x00000000u}, + {0x002b1572u, 0x00000000u, 0x00000000u}, + {0x002b95a2u, 0x00000000u, 0x00000000u}, + {0x582bd5d3u, 0x00000000u, 0x00000000u}, + {0x002c5612u, 0x00000000u, 0x00000000u}, + {0x59ac9633u, 0x00000000u, 0x00000000u}, + {0x002d1672u, 0x00000000u, 0x00000000u}, + {0x5bad96a3u, 0x00000000u, 0x00000000u}, + {0x5cae16f3u, 0x00000000u, 0x00000000u}, + {0x5dae9733u, 0x00000000u, 0x00000000u}, + {0x5eaf1773u, 0x00000000u, 0x00000000u}, + {0x5faf97b3u, 0x00000000u, 0x00000000u}, + {0x60b017f3u, 0x00000000u, 0x00000000u}, + {0x61b09833u, 0x00000000u, 0x00000000u}, + {0x62b11873u, 0x00000000u, 0x00000000u}, + {0x0031d8c2u, 0x00000000u, 0x00000000u}, + {0x00325902u, 0x00000000u, 0x00000000u}, + {0x0032d942u, 0x00000000u, 0x00000000u}, + {0x00335982u, 0x00000000u, 0x00000000u}, + {0x0033d9c2u, 0x00000000u, 0x00000000u}, + {0x00345a02u, 0x00000000u, 0x00000000u}, + {0x0034da42u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x01808033u, 0x00000000u, 0x00000000u}, + {0x00010072u, 0x00000000u, 0x00000000u}, + {0x000180a2u, 0x00000000u, 0x00000000u}, + {0x0001c0d2u, 0x00000000u, 0x00000000u}, + {0x00024102u, 0x00000000u, 0x00000000u}, + {0x05828133u, 0x00000000u, 0x00000000u}, + {0x00030172u, 0x00000000u, 0x00000000u}, + {0x07034193u, 0x00000000u, 0x00000000u}, + {0x0003c1d2u, 0x00000000u, 0x00000000u}, + {0x088401f3u, 0x00000000u, 0x00000000u}, + {0x09848233u, 0x00000000u, 0x00000000u}, + {0x00050272u, 0x00000000u, 0x00000000u}, + {0x0b054293u, 0x00000000u, 0x00000000u}, + {0x0005c2d2u, 0x00000000u, 0x00000000u}, + {0x0c8602f3u, 0x00000000u, 0x00000000u}, + {0x00068332u, 0x00000000u, 0x00000000u}, + {0x0e06c353u, 0x00000000u, 0x00000000u}, + {0x00074392u, 0x00000000u, 0x00000000u}, + {0x0f8783b3u, 0x00000000u, 0x00000000u}, + {0x000803f2u, 0x00000000u, 0x00000000u}, + {0x11084413u, 0x00000000u, 0x00000000u}, + {0x0008c452u, 0x00000000u, 0x00000000u}, + {0x00094482u, 0x00000000u, 0x00000000u}, + {0x138984b3u, 0x00000000u, 0x00000000u}, + {0x000a04f2u, 0x00000000u, 0x00000000u}, + {0x150a4513u, 0x00000000u, 0x00000000u}, + {0x000ac552u, 0x00000000u, 0x00000000u}, + {0x168b0573u, 0x00000000u, 0x00000000u}, + {0x000b85b2u, 0x00000000u, 0x00000000u}, + {0x000c05e2u, 0x00000000u, 0x00000000u}, + {0x000c4612u, 0x00000000u, 0x00000000u}, + {0x000cc642u, 0x00000000u, 0x00000000u}, + {0x000d0672u, 0x00000000u, 0x00000000u}, + {0x000d86a2u, 0x00000000u, 0x00000000u}, + {0x1c0dc6d3u, 0x00000000u, 0x00000000u}, + {0x000e4712u, 0x00000000u, 0x00000000u}, + {0x000ec742u, 0x00000000u, 0x00000000u}, + {0x000f0772u, 0x00000000u, 0x00000000u}, + {0x000f87a2u, 0x00000000u, 0x00000000u}, + {0x000fc7d2u, 0x00000000u, 0x00000000u}, + {0x00104802u, 0x00000000u, 0x00000000u}, + {0x21908833u, 0x00000000u, 0x00000000u}, + {0x22910873u, 0x00000000u, 0x00000000u}, + {0x2411c8c3u, 0x00000000u, 0x00000000u}, + {0x25124913u, 0x00000000u, 0x00000000u}, + {0x2612c953u, 0x00000000u, 0x00000000u}, + {0x27134993u, 0x00000000u, 0x00000000u}, + {0x0013c9d2u, 0x00000000u, 0x00000000u}, + {0x00144a02u, 0x00000000u, 0x00000000u}, + {0x0014ca42u, 0x00000000u, 0x00000000u}, + {0x00154a82u, 0x00000000u, 0x00000000u}, + {0x0015cac2u, 0x00000000u, 0x00000000u}, + {0x2d164b03u, 0x00000000u, 0x00000000u}, + {0x2e16cb53u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x00184c12u, 0x00000000u, 0x00000000u}, + {0x0018cc42u, 0x00000000u, 0x00000000u}, + {0x32990c73u, 0x00000000u, 0x00000000u}, + {0x3419ccc3u, 0x00000000u, 0x00000000u}, + {0x359a8d23u, 0x00000000u, 0x00000000u}, + {0x371b4d84u, 0x0000006fu, 0x00000000u}, + {0x389c0df3u, 0x00000000u, 0x00000000u}, + {0x3a1cce43u, 0x00000000u, 0x00000000u}, + {0x3b9d8ea3u, 0x00000000u, 0x00000000u}, + {0x3c9e0ef4u, 0x0000007au, 0x00000000u}, + {0x3e9f0f63u, 0x00000000u, 0x00000000u}, + {0x401fcfc3u, 0x00000000u, 0x00000000u}, + {0x41a09023u, 0x00000000u, 0x00000000u}, + {0x42a11074u, 0x00000086u, 0x00000000u}, + {0x4421d0d4u, 0x00000089u, 0x00000000u}, + {0x45a29134u, 0x0000008cu, 0x00000000u}, + {0x47235194u, 0x0000008fu, 0x00000000u}, + {0x48a411f4u, 0x00000092u, 0x00000000u}, + {0x4a24d253u, 0x00000000u, 0x00000000u}, + {0x4b255293u, 0x00000000u, 0x00000000u}, + {0x0025d2d2u, 0x00000000u, 0x00000000u}, + {0x4ca612f3u, 0x00000000u, 0x00000000u}, + {0x00269332u, 0x00000000u, 0x00000000u}, + {0x4e26d353u, 0x00000000u, 0x00000000u}, + {0x00275392u, 0x00000000u, 0x00000000u}, + {0x4fa793b3u, 0x00000000u, 0x00000000u}, + {0x50a813f3u, 0x00000000u, 0x00000000u}, + {0x00289432u, 0x00000000u, 0x00000000u}, + {0x5228d453u, 0x00000000u, 0x00000000u}, + {0x00295492u, 0x00000000u, 0x00000000u}, + {0x53a994b3u, 0x00000000u, 0x00000000u}, + {0x002a14f2u, 0x00000000u, 0x00000000u}, + {0x002a9522u, 0x00000000u, 0x00000000u}, + {0x002ad552u, 0x00000000u, 0x00000000u}, + {0x002b5582u, 0x00000000u, 0x00000000u}, + {0x002b95b2u, 0x00000000u, 0x00000000u}, + {0x002c15e2u, 0x00000000u, 0x00000000u}, + {0x002c9622u, 0x00000000u, 0x00000000u}, + {0x002cd652u, 0x00000000u, 0x00000000u}, + {0x002d5682u, 0x00000000u, 0x00000000u}, + {0x002dd6c2u, 0x00000000u, 0x00000000u}, + {0x002e5702u, 0x00000000u, 0x00000000u}, + {0x002ed742u, 0x00000000u, 0x00000000u}, + {0x002f5782u, 0x00000000u, 0x00000000u}, + {0x5faf97b3u, 0x00000000u, 0x00000000u}, + {0x60b017f3u, 0x00000000u, 0x00000000u}, + {0x61b09834u, 0x000000c4u, 0x00000000u}, + {0x63315893u, 0x00000000u, 0x00000000u}, + {0x6431d8d3u, 0x00000000u, 0x00000000u}, + {0x65325913u, 0x00000000u, 0x00000000u}, + {0x6632d953u, 0x00000000u, 0x00000000u}, + {0x67335993u, 0x00000000u, 0x00000000u}, + {0x6833d9d3u, 0x00000000u, 0x00000000u}, + {0x69345a13u, 0x00000000u, 0x00000000u}, + {0x0034da52u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x01808033u, 0x00000000u, 0x00000000u}, + {0x00010072u, 0x00000000u, 0x00000000u}, + {0x000180a2u, 0x00000000u, 0x00000000u}, + {0x0001c0d2u, 0x00000000u, 0x00000000u}, + {0x048200f3u, 0x00000000u, 0x00000000u}, + {0x00028132u, 0x00000000u, 0x00000000u}, + {0x0602c153u, 0x00000000u, 0x00000000u}, + {0x07034193u, 0x00000000u, 0x00000000u}, + {0x0003c1d2u, 0x00000000u, 0x00000000u}, + {0x088401f3u, 0x00000000u, 0x00000000u}, + {0x00048232u, 0x00000000u, 0x00000000u}, + {0x0a04c253u, 0x00000000u, 0x00000000u}, + {0x00054292u, 0x00000000u, 0x00000000u}, + {0x0b8582b3u, 0x00000000u, 0x00000000u}, + {0x000602f2u, 0x00000000u, 0x00000000u}, + {0x0d064313u, 0x00000000u, 0x00000000u}, + {0x0006c352u, 0x00000000u, 0x00000000u}, + {0x0e870373u, 0x00000000u, 0x00000000u}, + {0x000783b2u, 0x00000000u, 0x00000000u}, + {0x000803e2u, 0x00000000u, 0x00000000u}, + {0x00084412u, 0x00000000u, 0x00000000u}, + {0x0008c442u, 0x00000000u, 0x00000000u}, + {0x00090472u, 0x00000000u, 0x00000000u}, + {0x13094493u, 0x00000000u, 0x00000000u}, + {0x1409c4d3u, 0x00000000u, 0x00000000u}, + {0x000a4512u, 0x00000000u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x000b0572u, 0x00000000u, 0x00000000u}, + {0x170b4593u, 0x00000000u, 0x00000000u}, + {0x000bc5d2u, 0x00000000u, 0x00000000u}, + {0x188c05f3u, 0x00000000u, 0x00000000u}, + {0x000c8632u, 0x00000000u, 0x00000000u}, + {0x1a0cc653u, 0x00000000u, 0x00000000u}, + {0x000d4692u, 0x00000000u, 0x00000000u}, + {0x1b8d86b3u, 0x00000000u, 0x00000000u}, + {0x000e06f2u, 0x00000000u, 0x00000000u}, + {0x000e8722u, 0x00000000u, 0x00000000u}, + {0x000ec752u, 0x00000000u, 0x00000000u}, + {0x000f4782u, 0x00000000u, 0x00000000u}, + {0x000f87b2u, 0x00000000u, 0x00000000u}, + {0x200fc7d3u, 0x00000000u, 0x00000000u}, + {0x21104813u, 0x00000000u, 0x00000000u}, + {0x2210c853u, 0x00000000u, 0x00000000u}, + {0x23114893u, 0x00000000u, 0x00000000u}, + {0x2411c8d3u, 0x00000000u, 0x00000000u}, + {0x25124913u, 0x00000000u, 0x00000000u}, + {0x2612c953u, 0x00000000u, 0x00000000u}, + {0x27134993u, 0x00000000u, 0x00000000u}, + {0x2813c9d3u, 0x00000000u, 0x00000000u}, + {0x29144a13u, 0x00000000u, 0x00000000u}, + {0x2a14ca53u, 0x00000000u, 0x00000000u}, + {0x2b154a93u, 0x00000000u, 0x00000000u}, + {0x2c15cad3u, 0x00000000u, 0x00000000u}, + {0x2d164b13u, 0x00000000u, 0x00000000u}, + {0x2e16cb53u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x00184c12u, 0x00000000u, 0x00000000u}, + {0x0018cc42u, 0x00000000u, 0x00000000u}, + {0x32990c73u, 0x00000000u, 0x00000000u}, + {0x3419ccc3u, 0x00000000u, 0x00000000u}, + {0x351a4d14u, 0x0000006bu, 0x00000000u}, + {0x371b4d83u, 0x00000000u, 0x00000000u}, + {0x389c0de3u, 0x00000000u, 0x00000000u}, + {0x399c8e34u, 0x00000074u, 0x00000000u}, + {0x3b1d4e93u, 0x00000000u, 0x00000000u}, + {0x3c9e0ee4u, 0x0000007au, 0x00000000u}, + {0x3e1ecf54u, 0x0000007du, 0x00000000u}, + {0x3f9f8fb3u, 0x00000000u, 0x00000000u}, + {0x41205003u, 0x00000000u, 0x00000000u}, + {0x4220d054u, 0x00000085u, 0x00000000u}, + {0x4421d0c3u, 0x00000000u, 0x00000000u}, + {0x45a29123u, 0x00000000u, 0x00000000u}, + {0x46a31174u, 0x0000008eu, 0x00000000u}, + {0x4823d1d3u, 0x00000000u, 0x00000000u}, + {0x49a49224u, 0x00000094u, 0x00000000u}, + {0x00255292u, 0x00000000u, 0x00000000u}, + {0x4ba592b3u, 0x00000000u, 0x00000000u}, + {0x002612f2u, 0x00000000u, 0x00000000u}, + {0x4d265313u, 0x00000000u, 0x00000000u}, + {0x0026d352u, 0x00000000u, 0x00000000u}, + {0x4ea71373u, 0x00000000u, 0x00000000u}, + {0x002793b2u, 0x00000000u, 0x00000000u}, + {0x002813e2u, 0x00000000u, 0x00000000u}, + {0x00285412u, 0x00000000u, 0x00000000u}, + {0x0028d442u, 0x00000000u, 0x00000000u}, + {0x00291472u, 0x00000000u, 0x00000000u}, + {0x002994a2u, 0x00000000u, 0x00000000u}, + {0x0029d4d2u, 0x00000000u, 0x00000000u}, + {0x002a5502u, 0x00000000u, 0x00000000u}, + {0x002a9532u, 0x00000000u, 0x00000000u}, + {0x002b1562u, 0x00000000u, 0x00000000u}, + {0x572b5593u, 0x00000000u, 0x00000000u}, + {0x002bd5d2u, 0x00000000u, 0x00000000u}, + {0x58ac15f3u, 0x00000000u, 0x00000000u}, + {0x002c9632u, 0x00000000u, 0x00000000u}, + {0x5a2cd653u, 0x00000000u, 0x00000000u}, + {0x5b2d5693u, 0x00000000u, 0x00000000u}, + {0x5c2dd6d3u, 0x00000000u, 0x00000000u}, + {0x5d2e5713u, 0x00000000u, 0x00000000u}, + {0x5e2ed753u, 0x00000000u, 0x00000000u}, + {0x5f2f5793u, 0x00000000u, 0x00000000u}, + {0x602fd7d3u, 0x00000000u, 0x00000000u}, + {0x61305813u, 0x00000000u, 0x00000000u}, + {0x6230d853u, 0x00000000u, 0x00000000u}, + {0x63315893u, 0x00000000u, 0x00000000u}, + {0x6431d8d3u, 0x00000000u, 0x00000000u}, + {0x65325913u, 0x00000000u, 0x00000000u}, + {0x6632d953u, 0x00000000u, 0x00000000u}, + {0x67335993u, 0x00000000u, 0x00000000u}, + {0x6833d9d3u, 0x00000000u, 0x00000000u}, + {0x69345a13u, 0x00000000u, 0x00000000u}, + {0x0034da52u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x01808033u, 0x00000000u, 0x00000000u}, + {0x00010072u, 0x00000000u, 0x00000000u}, + {0x03014093u, 0x00000000u, 0x00000000u}, + {0x0001c0d2u, 0x00000000u, 0x00000000u}, + {0x048200f3u, 0x00000000u, 0x00000000u}, + {0x00028132u, 0x00000000u, 0x00000000u}, + {0x00030162u, 0x00000000u, 0x00000000u}, + {0x00034192u, 0x00000000u, 0x00000000u}, + {0x078381b3u, 0x00000000u, 0x00000000u}, + {0x000401f2u, 0x00000000u, 0x00000000u}, + {0x09044213u, 0x00000000u, 0x00000000u}, + {0x0004c252u, 0x00000000u, 0x00000000u}, + {0x0a850273u, 0x00000000u, 0x00000000u}, + {0x000582b2u, 0x00000000u, 0x00000000u}, + {0x000602e2u, 0x00000000u, 0x00000000u}, + {0x00064312u, 0x00000000u, 0x00000000u}, + {0x0d868333u, 0x00000000u, 0x00000000u}, + {0x00070372u, 0x00000000u, 0x00000000u}, + {0x0f074393u, 0x00000000u, 0x00000000u}, + {0x0007c3d2u, 0x00000000u, 0x00000000u}, + {0x00084402u, 0x00000000u, 0x00000000u}, + {0x00088432u, 0x00000000u, 0x00000000u}, + {0x00090462u, 0x00000000u, 0x00000000u}, + {0x00094492u, 0x00000000u, 0x00000000u}, + {0x138984b3u, 0x00000000u, 0x00000000u}, + {0x000a04f2u, 0x00000000u, 0x00000000u}, + {0x150a4513u, 0x00000000u, 0x00000000u}, + {0x000ac552u, 0x00000000u, 0x00000000u}, + {0x000b4582u, 0x00000000u, 0x00000000u}, + {0x000b85b2u, 0x00000000u, 0x00000000u}, + {0x180bc5d3u, 0x00000000u, 0x00000000u}, + {0x000c4612u, 0x00000000u, 0x00000000u}, + {0x198c8633u, 0x00000000u, 0x00000000u}, + {0x000d0672u, 0x00000000u, 0x00000000u}, + {0x1b0d4693u, 0x00000000u, 0x00000000u}, + {0x000dc6d2u, 0x00000000u, 0x00000000u}, + {0x000e4702u, 0x00000000u, 0x00000000u}, + {0x000e8732u, 0x00000000u, 0x00000000u}, + {0x1e0ec753u, 0x00000000u, 0x00000000u}, + {0x000f4792u, 0x00000000u, 0x00000000u}, + {0x1f8f87b3u, 0x00000000u, 0x00000000u}, + {0x001007f2u, 0x00000000u, 0x00000000u}, + {0x00108822u, 0x00000000u, 0x00000000u}, + {0x00110862u, 0x00000000u, 0x00000000u}, + {0x23114893u, 0x00000000u, 0x00000000u}, + {0x2411c8d3u, 0x00000000u, 0x00000000u}, + {0x25124913u, 0x00000000u, 0x00000000u}, + {0x2612c953u, 0x00000000u, 0x00000000u}, + {0x27134993u, 0x00000000u, 0x00000000u}, + {0x2813c9d3u, 0x00000000u, 0x00000000u}, + {0x29144a13u, 0x00000000u, 0x00000000u}, + {0x2a14ca53u, 0x00000000u, 0x00000000u}, + {0x2b154a93u, 0x00000000u, 0x00000000u}, + {0x2c15cad3u, 0x00000000u, 0x00000000u}, + {0x2d164b13u, 0x00000000u, 0x00000000u}, + {0x2e16cb53u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x0017cbd2u, 0x00000000u, 0x00000000u}, + {0x00184c02u, 0x00000000u, 0x00000000u}, + {0x0018cc42u, 0x00000000u, 0x00000000u}, + {0x32990c73u, 0x00000000u, 0x00000000u}, + {0x33998cb4u, 0x00000068u, 0x00000000u}, + {0x351a4d13u, 0x00000000u, 0x00000000u}, + {0x369b0d64u, 0x0000006eu, 0x00000000u}, + {0x381bcdd3u, 0x00000000u, 0x00000000u}, + {0x399c8e24u, 0x00000074u, 0x00000000u}, + {0x3b1d4e93u, 0x00000000u, 0x00000000u}, + {0x3c9e0ee3u, 0x00000000u, 0x00000000u}, + {0x3d9e8f34u, 0x0000007cu, 0x00000000u}, + {0x3f1f4f94u, 0x0000007fu, 0x00000000u}, + {0x40a00ff4u, 0x00000082u, 0x00000000u}, + {0x4220d053u, 0x00000000u, 0x00000000u}, + {0x43a190a3u, 0x00000000u, 0x00000000u}, + {0x44a210f4u, 0x0000008au, 0x00000000u}, + {0x46a31163u, 0x00000000u, 0x00000000u}, + {0x47a391b4u, 0x00000090u, 0x00000000u}, + {0x49245213u, 0x00000000u, 0x00000000u}, + {0x4aa51263u, 0x00000000u, 0x00000000u}, + {0x002592b2u, 0x00000000u, 0x00000000u}, + {0x4c25d2d3u, 0x00000000u, 0x00000000u}, + {0x00265312u, 0x00000000u, 0x00000000u}, + {0x4da69333u, 0x00000000u, 0x00000000u}, + {0x00271372u, 0x00000000u, 0x00000000u}, + {0x00275392u, 0x00000000u, 0x00000000u}, + {0x0027d3c2u, 0x00000000u, 0x00000000u}, + {0x50a813f3u, 0x00000000u, 0x00000000u}, + {0x00289432u, 0x00000000u, 0x00000000u}, + {0x5228d453u, 0x00000000u, 0x00000000u}, + {0x00295492u, 0x00000000u, 0x00000000u}, + {0x002994b2u, 0x00000000u, 0x00000000u}, + {0x002a14e2u, 0x00000000u, 0x00000000u}, + {0x552a5513u, 0x00000000u, 0x00000000u}, + {0x002ad552u, 0x00000000u, 0x00000000u}, + {0x56ab1573u, 0x00000000u, 0x00000000u}, + {0x002b95b2u, 0x00000000u, 0x00000000u}, + {0x582bd5d3u, 0x00000000u, 0x00000000u}, + {0x002c5612u, 0x00000000u, 0x00000000u}, + {0x002c9632u, 0x00000000u, 0x00000000u}, + {0x002d1662u, 0x00000000u, 0x00000000u}, + {0x002d96a2u, 0x00000000u, 0x00000000u}, + {0x002e16e2u, 0x00000000u, 0x00000000u}, + {0x5d2e5713u, 0x00000000u, 0x00000000u}, + {0x5e2ed753u, 0x00000000u, 0x00000000u}, + {0x5f2f5793u, 0x00000000u, 0x00000000u}, + {0x602fd7d3u, 0x00000000u, 0x00000000u}, + {0x61305813u, 0x00000000u, 0x00000000u}, + {0x6230d853u, 0x00000000u, 0x00000000u}, + {0x63315893u, 0x00000000u, 0x00000000u}, + {0x6431d8d3u, 0x00000000u, 0x00000000u}, + {0x65325913u, 0x00000000u, 0x00000000u}, + {0x6632d953u, 0x00000000u, 0x00000000u}, + {0x67335993u, 0x00000000u, 0x00000000u}, + {0x6833d9d3u, 0x00000000u, 0x00000000u}, + {0x69345a13u, 0x00000000u, 0x00000000u}, + {0x0034da52u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x01808033u, 0x00000000u, 0x00000000u}, + {0x00010072u, 0x00000000u, 0x00000000u}, + {0x03014093u, 0x00000000u, 0x00000000u}, + {0x0001c0d2u, 0x00000000u, 0x00000000u}, + {0x048200f3u, 0x00000000u, 0x00000000u}, + {0x00028132u, 0x00000000u, 0x00000000u}, + {0x0602c153u, 0x00000000u, 0x00000000u}, + {0x00034192u, 0x00000000u, 0x00000000u}, + {0x000381b2u, 0x00000000u, 0x00000000u}, + {0x000401e2u, 0x00000000u, 0x00000000u}, + {0x09044213u, 0x00000000u, 0x00000000u}, + {0x0004c252u, 0x00000000u, 0x00000000u}, + {0x00050272u, 0x00000000u, 0x00000000u}, + {0x000582a2u, 0x00000000u, 0x00000000u}, + {0x0005c2d2u, 0x00000000u, 0x00000000u}, + {0x0c8602f3u, 0x00000000u, 0x00000000u}, + {0x00068332u, 0x00000000u, 0x00000000u}, + {0x0e06c353u, 0x00000000u, 0x00000000u}, + {0x00074392u, 0x00000000u, 0x00000000u}, + {0x0f8783b3u, 0x00000000u, 0x00000000u}, + {0x000803f2u, 0x00000000u, 0x00000000u}, + {0x11084413u, 0x00000000u, 0x00000000u}, + {0x0008c452u, 0x00000000u, 0x00000000u}, + {0x12890473u, 0x00000000u, 0x00000000u}, + {0x000984b2u, 0x00000000u, 0x00000000u}, + {0x1409c4d3u, 0x00000000u, 0x00000000u}, + {0x000a4512u, 0x00000000u, 0x00000000u}, + {0x000a8532u, 0x00000000u, 0x00000000u}, + {0x000b0562u, 0x00000000u, 0x00000000u}, + {0x170b4593u, 0x00000000u, 0x00000000u}, + {0x000bc5d2u, 0x00000000u, 0x00000000u}, + {0x000c05f2u, 0x00000000u, 0x00000000u}, + {0x000c8622u, 0x00000000u, 0x00000000u}, + {0x000cc652u, 0x00000000u, 0x00000000u}, + {0x000d4682u, 0x00000000u, 0x00000000u}, + {0x000d86b2u, 0x00000000u, 0x00000000u}, + {0x000e06e2u, 0x00000000u, 0x00000000u}, + {0x000e4712u, 0x00000000u, 0x00000000u}, + {0x1d8e8733u, 0x00000000u, 0x00000000u}, + {0x000f0772u, 0x00000000u, 0x00000000u}, + {0x1f0f4793u, 0x00000000u, 0x00000000u}, + {0x000fc7d2u, 0x00000000u, 0x00000000u}, + {0x00104802u, 0x00000000u, 0x00000000u}, + {0x21908833u, 0x00000000u, 0x00000000u}, + {0x00110872u, 0x00000000u, 0x00000000u}, + {0x239188a3u, 0x00000000u, 0x00000000u}, + {0x249208f3u, 0x00000000u, 0x00000000u}, + {0x00128932u, 0x00000000u, 0x00000000u}, + {0x00130962u, 0x00000000u, 0x00000000u}, + {0x27134993u, 0x00000000u, 0x00000000u}, + {0x2813c9d3u, 0x00000000u, 0x00000000u}, + {0x29144a13u, 0x00000000u, 0x00000000u}, + {0x2a14ca53u, 0x00000000u, 0x00000000u}, + {0x2b154a93u, 0x00000000u, 0x00000000u}, + {0x2c15cad3u, 0x00000000u, 0x00000000u}, + {0x2d164b13u, 0x00000000u, 0x00000000u}, + {0x2e16cb53u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x00184c12u, 0x00000000u, 0x00000000u}, + {0x31988c33u, 0x00000000u, 0x00000000u}, + {0x32990c73u, 0x00000000u, 0x00000000u}, + {0x33998cb4u, 0x00000068u, 0x00000000u}, + {0x351a4d14u, 0x0000006bu, 0x00000000u}, + {0x369b0d73u, 0x00000000u, 0x00000000u}, + {0x381bcdc3u, 0x00000000u, 0x00000000u}, + {0x399c8e23u, 0x00000000u, 0x00000000u}, + {0x3b1d4e83u, 0x00000000u, 0x00000000u}, + {0x3c1dced3u, 0x00000000u, 0x00000000u}, + {0x3d9e8f23u, 0x00000000u, 0x00000000u}, + {0x3f1f4f83u, 0x00000000u, 0x00000000u}, + {0x40a00fe3u, 0x00000000u, 0x00000000u}, + {0x41a09034u, 0x00000084u, 0x00000000u}, + {0x43215094u, 0x00000087u, 0x00000000u}, + {0x44a210f3u, 0x00000000u, 0x00000000u}, + {0x4622d143u, 0x00000000u, 0x00000000u}, + {0x47235194u, 0x0000008fu, 0x00000000u}, + {0x48a411f4u, 0x00000092u, 0x00000000u}, + {0x4a24d253u, 0x00000000u, 0x00000000u}, + {0x4b255293u, 0x00000000u, 0x00000000u}, + {0x0025d2d2u, 0x00000000u, 0x00000000u}, + {0x4ca612f3u, 0x00000000u, 0x00000000u}, + {0x00269332u, 0x00000000u, 0x00000000u}, + {0x0026d352u, 0x00000000u, 0x00000000u}, + {0x00275382u, 0x00000000u, 0x00000000u}, + {0x002793b2u, 0x00000000u, 0x00000000u}, + {0x5027d3d3u, 0x00000000u, 0x00000000u}, + {0x00285412u, 0x00000000u, 0x00000000u}, + {0x0028d442u, 0x00000000u, 0x00000000u}, + {0x00291472u, 0x00000000u, 0x00000000u}, + {0x53295493u, 0x00000000u, 0x00000000u}, + {0x0029d4d2u, 0x00000000u, 0x00000000u}, + {0x002a5502u, 0x00000000u, 0x00000000u}, + {0x002a9532u, 0x00000000u, 0x00000000u}, + {0x562ad553u, 0x00000000u, 0x00000000u}, + {0x002b5592u, 0x00000000u, 0x00000000u}, + {0x002b95b2u, 0x00000000u, 0x00000000u}, + {0x002c15e2u, 0x00000000u, 0x00000000u}, + {0x592c5613u, 0x00000000u, 0x00000000u}, + {0x002cd652u, 0x00000000u, 0x00000000u}, + {0x5aad1673u, 0x00000000u, 0x00000000u}, + {0x5bad96b3u, 0x00000000u, 0x00000000u}, + {0x5cae16f3u, 0x00000000u, 0x00000000u}, + {0x5dae9733u, 0x00000000u, 0x00000000u}, + {0x5eaf1773u, 0x00000000u, 0x00000000u}, + {0x002f97b2u, 0x00000000u, 0x00000000u}, + {0x003017e2u, 0x00000000u, 0x00000000u}, + {0x61305813u, 0x00000000u, 0x00000000u}, + {0x6230d853u, 0x00000000u, 0x00000000u}, + {0x003198a2u, 0x00000000u, 0x00000000u}, + {0x6431d8d3u, 0x00000000u, 0x00000000u}, + {0x65325913u, 0x00000000u, 0x00000000u}, + {0x6632d953u, 0x00000000u, 0x00000000u}, + {0x67335993u, 0x00000000u, 0x00000000u}, + {0x6833d9d3u, 0x00000000u, 0x00000000u}, + {0x69345a13u, 0x00000000u, 0x00000000u}, + {0x0034da52u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x01808033u, 0x00000000u, 0x00000000u}, + {0x00010072u, 0x00000000u, 0x00000000u}, + {0x00014092u, 0x00000000u, 0x00000000u}, + {0x0001c0c2u, 0x00000000u, 0x00000000u}, + {0x000200f2u, 0x00000000u, 0x00000000u}, + {0x00028122u, 0x00000000u, 0x00000000u}, + {0x0002c152u, 0x00000000u, 0x00000000u}, + {0x06830173u, 0x00000000u, 0x00000000u}, + {0x000381b2u, 0x00000000u, 0x00000000u}, + {0x0803c1d3u, 0x00000000u, 0x00000000u}, + {0x00044212u, 0x00000000u, 0x00000000u}, + {0x09848233u, 0x00000000u, 0x00000000u}, + {0x00050272u, 0x00000000u, 0x00000000u}, + {0x00054292u, 0x00000000u, 0x00000000u}, + {0x0005c2c2u, 0x00000000u, 0x00000000u}, + {0x000602f2u, 0x00000000u, 0x00000000u}, + {0x00068322u, 0x00000000u, 0x00000000u}, + {0x0006c352u, 0x00000000u, 0x00000000u}, + {0x0e870373u, 0x00000000u, 0x00000000u}, + {0x000783b2u, 0x00000000u, 0x00000000u}, + {0x000803e2u, 0x00000000u, 0x00000000u}, + {0x00084412u, 0x00000000u, 0x00000000u}, + {0x11888433u, 0x00000000u, 0x00000000u}, + {0x00090472u, 0x00000000u, 0x00000000u}, + {0x13094493u, 0x00000000u, 0x00000000u}, + {0x0009c4d2u, 0x00000000u, 0x00000000u}, + {0x000a04f2u, 0x00000000u, 0x00000000u}, + {0x000a8522u, 0x00000000u, 0x00000000u}, + {0x000ac552u, 0x00000000u, 0x00000000u}, + {0x000b4582u, 0x00000000u, 0x00000000u}, + {0x000b85b2u, 0x00000000u, 0x00000000u}, + {0x180bc5d3u, 0x00000000u, 0x00000000u}, + {0x000c4612u, 0x00000000u, 0x00000000u}, + {0x000c8632u, 0x00000000u, 0x00000000u}, + {0x000d0662u, 0x00000000u, 0x00000000u}, + {0x000d4692u, 0x00000000u, 0x00000000u}, + {0x1b8d86b3u, 0x00000000u, 0x00000000u}, + {0x000e06f2u, 0x00000000u, 0x00000000u}, + {0x000e8722u, 0x00000000u, 0x00000000u}, + {0x000ec752u, 0x00000000u, 0x00000000u}, + {0x1e8f0773u, 0x00000000u, 0x00000000u}, + {0x000f87b2u, 0x00000000u, 0x00000000u}, + {0x200fc7d3u, 0x00000000u, 0x00000000u}, + {0x00104812u, 0x00000000u, 0x00000000u}, + {0x21908833u, 0x00000000u, 0x00000000u}, + {0x22910873u, 0x00000000u, 0x00000000u}, + {0x239188b3u, 0x00000000u, 0x00000000u}, + {0x249208f3u, 0x00000000u, 0x00000000u}, + {0x25928933u, 0x00000000u, 0x00000000u}, + {0x26930973u, 0x00000000u, 0x00000000u}, + {0x279389b3u, 0x00000000u, 0x00000000u}, + {0x001409f2u, 0x00000000u, 0x00000000u}, + {0x00148a22u, 0x00000000u, 0x00000000u}, + {0x00150a62u, 0x00000000u, 0x00000000u}, + {0x00158aa2u, 0x00000000u, 0x00000000u}, + {0x2c15cad3u, 0x00000000u, 0x00000000u}, + {0x2d164b13u, 0x00000000u, 0x00000000u}, + {0x2e16cb53u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x00184c12u, 0x00000000u, 0x00000000u}, + {0x0018cc42u, 0x00000000u, 0x00000000u}, + {0x32990c73u, 0x00000000u, 0x00000000u}, + {0x33998cb3u, 0x00000000u, 0x00000000u}, + {0x351a4d03u, 0x00000000u, 0x00000000u}, + {0x361acd54u, 0x0000006du, 0x00000000u}, + {0x379b8db4u, 0x00000070u, 0x00000000u}, + {0x391c4e13u, 0x00000000u, 0x00000000u}, + {0x3a9d0e63u, 0x00000000u, 0x00000000u}, + {0x3c1dcec3u, 0x00000000u, 0x00000000u}, + {0x3d1e4f14u, 0x0000007bu, 0x00000000u}, + {0x3e9f0f73u, 0x00000000u, 0x00000000u}, + {0x401fcfc3u, 0x00000000u, 0x00000000u}, + {0x41a09023u, 0x00000000u, 0x00000000u}, + {0x42a11074u, 0x00000086u, 0x00000000u}, + {0x4421d0d4u, 0x00000089u, 0x00000000u}, + {0x45a29133u, 0x00000000u, 0x00000000u}, + {0x47235183u, 0x00000000u, 0x00000000u}, + {0x4823d1d4u, 0x00000091u, 0x00000000u}, + {0x49a49233u, 0x00000000u, 0x00000000u}, + {0x00255282u, 0x00000000u, 0x00000000u}, + {0x4ba592b3u, 0x00000000u, 0x00000000u}, + {0x002612f2u, 0x00000000u, 0x00000000u}, + {0x00265312u, 0x00000000u, 0x00000000u}, + {0x0026d342u, 0x00000000u, 0x00000000u}, + {0x00271372u, 0x00000000u, 0x00000000u}, + {0x4f275393u, 0x00000000u, 0x00000000u}, + {0x0027d3d2u, 0x00000000u, 0x00000000u}, + {0x002813f2u, 0x00000000u, 0x00000000u}, + {0x00289422u, 0x00000000u, 0x00000000u}, + {0x0028d452u, 0x00000000u, 0x00000000u}, + {0x00295482u, 0x00000000u, 0x00000000u}, + {0x002994b2u, 0x00000000u, 0x00000000u}, + {0x5429d4d3u, 0x00000000u, 0x00000000u}, + {0x002a5512u, 0x00000000u, 0x00000000u}, + {0x55aa9533u, 0x00000000u, 0x00000000u}, + {0x002b1572u, 0x00000000u, 0x00000000u}, + {0x002b5592u, 0x00000000u, 0x00000000u}, + {0x002bd5c2u, 0x00000000u, 0x00000000u}, + {0x58ac15f3u, 0x00000000u, 0x00000000u}, + {0x002c9632u, 0x00000000u, 0x00000000u}, + {0x002cd652u, 0x00000000u, 0x00000000u}, + {0x002d5682u, 0x00000000u, 0x00000000u}, + {0x002dd6c2u, 0x00000000u, 0x00000000u}, + {0x002e5702u, 0x00000000u, 0x00000000u}, + {0x5dae9733u, 0x00000000u, 0x00000000u}, + {0x5eaf1773u, 0x00000000u, 0x00000000u}, + {0x5faf97b3u, 0x00000000u, 0x00000000u}, + {0x60b017f3u, 0x00000000u, 0x00000000u}, + {0x61b09833u, 0x00000000u, 0x00000000u}, + {0x62b11873u, 0x00000000u, 0x00000000u}, + {0x63b198b3u, 0x00000000u, 0x00000000u}, + {0x003218f2u, 0x00000000u, 0x00000000u}, + {0x00329922u, 0x00000000u, 0x00000000u}, + {0x00331962u, 0x00000000u, 0x00000000u}, + {0x003399a2u, 0x00000000u, 0x00000000u}, + {0x6833d9d3u, 0x00000000u, 0x00000000u}, + {0x69345a13u, 0x00000000u, 0x00000000u}, + {0x0034da52u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x01808033u, 0x00000000u, 0x00000000u}, + {0x00010072u, 0x00000000u, 0x00000000u}, + {0x03014093u, 0x00000000u, 0x00000000u}, + {0x0001c0d2u, 0x00000000u, 0x00000000u}, + {0x000200f2u, 0x00000000u, 0x00000000u}, + {0x05024113u, 0x00000000u, 0x00000000u}, + {0x0002c152u, 0x00000000u, 0x00000000u}, + {0x06830173u, 0x00000000u, 0x00000000u}, + {0x000381b2u, 0x00000000u, 0x00000000u}, + {0x0003c1d2u, 0x00000000u, 0x00000000u}, + {0x00044202u, 0x00000000u, 0x00000000u}, + {0x00048232u, 0x00000000u, 0x00000000u}, + {0x0a04c253u, 0x00000000u, 0x00000000u}, + {0x00054292u, 0x00000000u, 0x00000000u}, + {0x0b8582b3u, 0x00000000u, 0x00000000u}, + {0x000602f2u, 0x00000000u, 0x00000000u}, + {0x00064312u, 0x00000000u, 0x00000000u}, + {0x0006c342u, 0x00000000u, 0x00000000u}, + {0x00070372u, 0x00000000u, 0x00000000u}, + {0x0f074393u, 0x00000000u, 0x00000000u}, + {0x0007c3d2u, 0x00000000u, 0x00000000u}, + {0x108803f3u, 0x00000000u, 0x00000000u}, + {0x00088432u, 0x00000000u, 0x00000000u}, + {0x0008c452u, 0x00000000u, 0x00000000u}, + {0x00094482u, 0x00000000u, 0x00000000u}, + {0x000984b2u, 0x00000000u, 0x00000000u}, + {0x1409c4d3u, 0x00000000u, 0x00000000u}, + {0x000a4512u, 0x00000000u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x000b0572u, 0x00000000u, 0x00000000u}, + {0x000b4592u, 0x00000000u, 0x00000000u}, + {0x178b85b3u, 0x00000000u, 0x00000000u}, + {0x000c05f2u, 0x00000000u, 0x00000000u}, + {0x190c4613u, 0x00000000u, 0x00000000u}, + {0x000cc652u, 0x00000000u, 0x00000000u}, + {0x000d0672u, 0x00000000u, 0x00000000u}, + {0x000d86a2u, 0x00000000u, 0x00000000u}, + {0x000dc6d2u, 0x00000000u, 0x00000000u}, + {0x000e4702u, 0x00000000u, 0x00000000u}, + {0x000e8732u, 0x00000000u, 0x00000000u}, + {0x1e0ec753u, 0x00000000u, 0x00000000u}, + {0x000f4792u, 0x00000000u, 0x00000000u}, + {0x1f8f87b3u, 0x00000000u, 0x00000000u}, + {0x001007f2u, 0x00000000u, 0x00000000u}, + {0x00104812u, 0x00000000u, 0x00000000u}, + {0x0010c842u, 0x00000000u, 0x00000000u}, + {0x00114882u, 0x00000000u, 0x00000000u}, + {0x2411c8c3u, 0x00000000u, 0x00000000u}, + {0x00124912u, 0x00000000u, 0x00000000u}, + {0x0012c942u, 0x00000000u, 0x00000000u}, + {0x26930973u, 0x00000000u, 0x00000000u}, + {0x279389b3u, 0x00000000u, 0x00000000u}, + {0x289409f3u, 0x00000000u, 0x00000000u}, + {0x29948a33u, 0x00000000u, 0x00000000u}, + {0x00150a72u, 0x00000000u, 0x00000000u}, + {0x00158aa2u, 0x00000000u, 0x00000000u}, + {0x00160ae2u, 0x00000000u, 0x00000000u}, + {0x2d164b13u, 0x00000000u, 0x00000000u}, + {0x2e16cb53u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x00184c12u, 0x00000000u, 0x00000000u}, + {0x31988c33u, 0x00000000u, 0x00000000u}, + {0x32990c73u, 0x00000000u, 0x00000000u}, + {0x33998cb3u, 0x00000000u, 0x00000000u}, + {0x351a4d03u, 0x00000000u, 0x00000000u}, + {0x369b0d63u, 0x00000000u, 0x00000000u}, + {0x379b8db3u, 0x00000000u, 0x00000000u}, + {0x391c4e03u, 0x00000000u, 0x00000000u}, + {0x3a9d0e63u, 0x00000000u, 0x00000000u}, + {0x3b9d8eb3u, 0x00000000u, 0x00000000u}, + {0x3d1e4f03u, 0x00000000u, 0x00000000u}, + {0x3e9f0f63u, 0x00000000u, 0x00000000u}, + {0x3f9f8fb4u, 0x00000080u, 0x00000000u}, + {0x41205013u, 0x00000000u, 0x00000000u}, + {0x4220d054u, 0x00000085u, 0x00000000u}, + {0x43a190b4u, 0x00000088u, 0x00000000u}, + {0x45225113u, 0x00000000u, 0x00000000u}, + {0x4622d154u, 0x0000008du, 0x00000000u}, + {0x4823d1c3u, 0x00000000u, 0x00000000u}, + {0x49a49223u, 0x00000000u, 0x00000000u}, + {0x00251272u, 0x00000000u, 0x00000000u}, + {0x002592a2u, 0x00000000u, 0x00000000u}, + {0x0025d2d2u, 0x00000000u, 0x00000000u}, + {0x00265302u, 0x00000000u, 0x00000000u}, + {0x00269332u, 0x00000000u, 0x00000000u}, + {0x4e26d353u, 0x00000000u, 0x00000000u}, + {0x00275392u, 0x00000000u, 0x00000000u}, + {0x4fa793b3u, 0x00000000u, 0x00000000u}, + {0x002813f2u, 0x00000000u, 0x00000000u}, + {0x00285412u, 0x00000000u, 0x00000000u}, + {0x51a89433u, 0x00000000u, 0x00000000u}, + {0x00291472u, 0x00000000u, 0x00000000u}, + {0x53295493u, 0x00000000u, 0x00000000u}, + {0x0029d4d2u, 0x00000000u, 0x00000000u}, + {0x002a14f2u, 0x00000000u, 0x00000000u}, + {0x002a9522u, 0x00000000u, 0x00000000u}, + {0x002ad552u, 0x00000000u, 0x00000000u}, + {0x56ab1573u, 0x00000000u, 0x00000000u}, + {0x002b95b2u, 0x00000000u, 0x00000000u}, + {0x582bd5d3u, 0x00000000u, 0x00000000u}, + {0x002c5612u, 0x00000000u, 0x00000000u}, + {0x002c9632u, 0x00000000u, 0x00000000u}, + {0x002d1662u, 0x00000000u, 0x00000000u}, + {0x5b2d5693u, 0x00000000u, 0x00000000u}, + {0x5c2dd6d3u, 0x00000000u, 0x00000000u}, + {0x5d2e5713u, 0x00000000u, 0x00000000u}, + {0x002ed752u, 0x00000000u, 0x00000000u}, + {0x002f5782u, 0x00000000u, 0x00000000u}, + {0x5faf97b3u, 0x00000000u, 0x00000000u}, + {0x60b017f3u, 0x00000000u, 0x00000000u}, + {0x61b09833u, 0x00000000u, 0x00000000u}, + {0x62b11873u, 0x00000000u, 0x00000000u}, + {0x003198b2u, 0x00000000u, 0x00000000u}, + {0x64b218e3u, 0x00000000u, 0x00000000u}, + {0x00329932u, 0x00000000u, 0x00000000u}, + {0x00331962u, 0x00000000u, 0x00000000u}, + {0x67335993u, 0x00000000u, 0x00000000u}, + {0x6833d9d3u, 0x00000000u, 0x00000000u}, + {0x69345a13u, 0x00000000u, 0x00000000u}, + {0x0034da52u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x00008032u, 0x00000000u, 0x00000000u}, + {0x00010062u, 0x00000000u, 0x00000000u}, + {0x00014092u, 0x00000000u, 0x00000000u}, + {0x0001c0c2u, 0x00000000u, 0x00000000u}, + {0x000200f2u, 0x00000000u, 0x00000000u}, + {0x00024112u, 0x00000000u, 0x00000000u}, + {0x0002c142u, 0x00000000u, 0x00000000u}, + {0x00030172u, 0x00000000u, 0x00000000u}, + {0x000381a2u, 0x00000000u, 0x00000000u}, + {0x0003c1d2u, 0x00000000u, 0x00000000u}, + {0x000401f2u, 0x00000000u, 0x00000000u}, + {0x00048222u, 0x00000000u, 0x00000000u}, + {0x0004c252u, 0x00000000u, 0x00000000u}, + {0x00054282u, 0x00000000u, 0x00000000u}, + {0x000582b2u, 0x00000000u, 0x00000000u}, + {0x000602e2u, 0x00000000u, 0x00000000u}, + {0x00064312u, 0x00000000u, 0x00000000u}, + {0x00068332u, 0x00000000u, 0x00000000u}, + {0x00070362u, 0x00000000u, 0x00000000u}, + {0x00074392u, 0x00000000u, 0x00000000u}, + {0x0007c3c2u, 0x00000000u, 0x00000000u}, + {0x000803f2u, 0x00000000u, 0x00000000u}, + {0x00084412u, 0x00000000u, 0x00000000u}, + {0x0008c442u, 0x00000000u, 0x00000000u}, + {0x00090472u, 0x00000000u, 0x00000000u}, + {0x13094493u, 0x00000000u, 0x00000000u}, + {0x0009c4d2u, 0x00000000u, 0x00000000u}, + {0x000a04f2u, 0x00000000u, 0x00000000u}, + {0x150a4513u, 0x00000000u, 0x00000000u}, + {0x000ac552u, 0x00000000u, 0x00000000u}, + {0x168b0573u, 0x00000000u, 0x00000000u}, + {0x000b85b2u, 0x00000000u, 0x00000000u}, + {0x180bc5d3u, 0x00000000u, 0x00000000u}, + {0x000c4612u, 0x00000000u, 0x00000000u}, + {0x000c8632u, 0x00000000u, 0x00000000u}, + {0x1a0cc653u, 0x00000000u, 0x00000000u}, + {0x000d4692u, 0x00000000u, 0x00000000u}, + {0x1b8d86b3u, 0x00000000u, 0x00000000u}, + {0x000e06f2u, 0x00000000u, 0x00000000u}, + {0x000e4712u, 0x00000000u, 0x00000000u}, + {0x1d8e8733u, 0x00000000u, 0x00000000u}, + {0x000f0772u, 0x00000000u, 0x00000000u}, + {0x1f0f4793u, 0x00000000u, 0x00000000u}, + {0x000fc7d2u, 0x00000000u, 0x00000000u}, + {0x209007f3u, 0x00000000u, 0x00000000u}, + {0x00108832u, 0x00000000u, 0x00000000u}, + {0x2210c853u, 0x00000000u, 0x00000000u}, + {0x23114893u, 0x00000000u, 0x00000000u}, + {0x2411c8d3u, 0x00000000u, 0x00000000u}, + {0x00124912u, 0x00000000u, 0x00000000u}, + {0x0012c942u, 0x00000000u, 0x00000000u}, + {0x26930973u, 0x00000000u, 0x00000000u}, + {0x0013c9c2u, 0x00000000u, 0x00000000u}, + {0x289409f3u, 0x00000000u, 0x00000000u}, + {0x00148a32u, 0x00000000u, 0x00000000u}, + {0x2a950a63u, 0x00000000u, 0x00000000u}, + {0x00158ab2u, 0x00000000u, 0x00000000u}, + {0x00160ae2u, 0x00000000u, 0x00000000u}, + {0x00168b22u, 0x00000000u, 0x00000000u}, + {0x2e16cb53u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x3017cbd3u, 0x00000000u, 0x00000000u}, + {0x00184c12u, 0x00000000u, 0x00000000u}, + {0x00188c32u, 0x00000000u, 0x00000000u}, + {0x00190c62u, 0x00000000u, 0x00000000u}, + {0x33998ca3u, 0x00000000u, 0x00000000u}, + {0x349a0cf3u, 0x00000000u, 0x00000000u}, + {0x361acd43u, 0x00000000u, 0x00000000u}, + {0x379b8da3u, 0x00000000u, 0x00000000u}, + {0x391c4e03u, 0x00000000u, 0x00000000u}, + {0x3a1cce53u, 0x00000000u, 0x00000000u}, + {0x3b1d4e94u, 0x00000077u, 0x00000000u}, + {0x3c9e0ef4u, 0x0000007au, 0x00000000u}, + {0x3e1ecf53u, 0x00000000u, 0x00000000u}, + {0x3f9f8fa3u, 0x00000000u, 0x00000000u}, + {0x41205003u, 0x00000000u, 0x00000000u}, + {0x4220d053u, 0x00000000u, 0x00000000u}, + {0x43a190a3u, 0x00000000u, 0x00000000u}, + {0x45225103u, 0x00000000u, 0x00000000u}, + {0x4622d153u, 0x00000000u, 0x00000000u}, + {0x47235194u, 0x0000008fu, 0x00000000u}, + {0x48a411f3u, 0x00000000u, 0x00000000u}, + {0x4a24d243u, 0x00000000u, 0x00000000u}, + {0x00255292u, 0x00000000u, 0x00000000u}, + {0x4ba592b3u, 0x00000000u, 0x00000000u}, + {0x002612f2u, 0x00000000u, 0x00000000u}, + {0x4d265313u, 0x00000000u, 0x00000000u}, + {0x0026d352u, 0x00000000u, 0x00000000u}, + {0x00271372u, 0x00000000u, 0x00000000u}, + {0x4f275393u, 0x00000000u, 0x00000000u}, + {0x0027d3d2u, 0x00000000u, 0x00000000u}, + {0x50a813f3u, 0x00000000u, 0x00000000u}, + {0x00289432u, 0x00000000u, 0x00000000u}, + {0x0028d452u, 0x00000000u, 0x00000000u}, + {0x52a91473u, 0x00000000u, 0x00000000u}, + {0x002994b2u, 0x00000000u, 0x00000000u}, + {0x5429d4d3u, 0x00000000u, 0x00000000u}, + {0x002a5512u, 0x00000000u, 0x00000000u}, + {0x55aa9533u, 0x00000000u, 0x00000000u}, + {0x002b1572u, 0x00000000u, 0x00000000u}, + {0x002b5592u, 0x00000000u, 0x00000000u}, + {0x57ab95b3u, 0x00000000u, 0x00000000u}, + {0x002c15f2u, 0x00000000u, 0x00000000u}, + {0x592c5613u, 0x00000000u, 0x00000000u}, + {0x002cd652u, 0x00000000u, 0x00000000u}, + {0x002d1672u, 0x00000000u, 0x00000000u}, + {0x002d96a2u, 0x00000000u, 0x00000000u}, + {0x002e16e2u, 0x00000000u, 0x00000000u}, + {0x5d2e5713u, 0x00000000u, 0x00000000u}, + {0x5e2ed753u, 0x00000000u, 0x00000000u}, + {0x5f2f5793u, 0x00000000u, 0x00000000u}, + {0x002fd7d2u, 0x00000000u, 0x00000000u}, + {0x61305803u, 0x00000000u, 0x00000000u}, + {0x0030d852u, 0x00000000u, 0x00000000u}, + {0x00315882u, 0x00000000u, 0x00000000u}, + {0x63b198b3u, 0x00000000u, 0x00000000u}, + {0x64b218f3u, 0x00000000u, 0x00000000u}, + {0x65b29933u, 0x00000000u, 0x00000000u}, + {0x00331972u, 0x00000000u, 0x00000000u}, + {0x67335993u, 0x00000000u, 0x00000000u}, + {0x6833d9d3u, 0x00000000u, 0x00000000u}, + {0x69345a13u, 0x00000000u, 0x00000000u}, + {0x0034da52u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x00008032u, 0x00000000u, 0x00000000u}, + {0x00010062u, 0x00000000u, 0x00000000u}, + {0x00014092u, 0x00000000u, 0x00000000u}, + {0x0001c0c2u, 0x00000000u, 0x00000000u}, + {0x000200f2u, 0x00000000u, 0x00000000u}, + {0x00024112u, 0x00000000u, 0x00000000u}, + {0x05828133u, 0x00000000u, 0x00000000u}, + {0x00030172u, 0x00000000u, 0x00000000u}, + {0x00034192u, 0x00000000u, 0x00000000u}, + {0x0003c1c2u, 0x00000000u, 0x00000000u}, + {0x000401f2u, 0x00000000u, 0x00000000u}, + {0x00048222u, 0x00000000u, 0x00000000u}, + {0x0004c252u, 0x00000000u, 0x00000000u}, + {0x00050272u, 0x00000000u, 0x00000000u}, + {0x0b054293u, 0x00000000u, 0x00000000u}, + {0x0005c2d2u, 0x00000000u, 0x00000000u}, + {0x0c8602f3u, 0x00000000u, 0x00000000u}, + {0x00068332u, 0x00000000u, 0x00000000u}, + {0x0006c352u, 0x00000000u, 0x00000000u}, + {0x00074382u, 0x00000000u, 0x00000000u}, + {0x000783b2u, 0x00000000u, 0x00000000u}, + {0x0007c3d2u, 0x00000000u, 0x00000000u}, + {0x00084402u, 0x00000000u, 0x00000000u}, + {0x00088432u, 0x00000000u, 0x00000000u}, + {0x1208c453u, 0x00000000u, 0x00000000u}, + {0x00094492u, 0x00000000u, 0x00000000u}, + {0x000984b2u, 0x00000000u, 0x00000000u}, + {0x1409c4d3u, 0x00000000u, 0x00000000u}, + {0x000a4512u, 0x00000000u, 0x00000000u}, + {0x158a8533u, 0x00000000u, 0x00000000u}, + {0x000b0572u, 0x00000000u, 0x00000000u}, + {0x000b4592u, 0x00000000u, 0x00000000u}, + {0x178b85b3u, 0x00000000u, 0x00000000u}, + {0x000c05f2u, 0x00000000u, 0x00000000u}, + {0x190c4613u, 0x00000000u, 0x00000000u}, + {0x000cc652u, 0x00000000u, 0x00000000u}, + {0x000d0672u, 0x00000000u, 0x00000000u}, + {0x000d86a2u, 0x00000000u, 0x00000000u}, + {0x000dc6d2u, 0x00000000u, 0x00000000u}, + {0x000e06f2u, 0x00000000u, 0x00000000u}, + {0x1d0e4713u, 0x00000000u, 0x00000000u}, + {0x000ec752u, 0x00000000u, 0x00000000u}, + {0x1e8f0773u, 0x00000000u, 0x00000000u}, + {0x000f87b2u, 0x00000000u, 0x00000000u}, + {0x000fc7d2u, 0x00000000u, 0x00000000u}, + {0x00104802u, 0x00000000u, 0x00000000u}, + {0x21908833u, 0x00000000u, 0x00000000u}, + {0x00110872u, 0x00000000u, 0x00000000u}, + {0x001188a2u, 0x00000000u, 0x00000000u}, + {0x2411c8d3u, 0x00000000u, 0x00000000u}, + {0x25124913u, 0x00000000u, 0x00000000u}, + {0x0012c952u, 0x00000000u, 0x00000000u}, + {0x00134982u, 0x00000000u, 0x00000000u}, + {0x0013c9c2u, 0x00000000u, 0x00000000u}, + {0x00144a02u, 0x00000000u, 0x00000000u}, + {0x29948a33u, 0x00000000u, 0x00000000u}, + {0x00150a72u, 0x00000000u, 0x00000000u}, + {0x2b958aa3u, 0x00000000u, 0x00000000u}, + {0x00160af2u, 0x00000000u, 0x00000000u}, + {0x00168b22u, 0x00000000u, 0x00000000u}, + {0x2e16cb53u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x0017cbd2u, 0x00000000u, 0x00000000u}, + {0x00184c02u, 0x00000000u, 0x00000000u}, + {0x31988c33u, 0x00000000u, 0x00000000u}, + {0x32990c73u, 0x00000000u, 0x00000000u}, + {0x33998cb3u, 0x00000000u, 0x00000000u}, + {0x349a0cf4u, 0x0000006au, 0x00000000u}, + {0x361acd53u, 0x00000000u, 0x00000000u}, + {0x371b4d93u, 0x00000000u, 0x00000000u}, + {0x389c0de3u, 0x00000000u, 0x00000000u}, + {0x3a1cce43u, 0x00000000u, 0x00000000u}, + {0x3b9d8ea3u, 0x00000000u, 0x00000000u}, + {0x3c9e0ef3u, 0x00000000u, 0x00000000u}, + {0x3d9e8f34u, 0x0000007cu, 0x00000000u}, + {0x3f1f4f94u, 0x0000007fu, 0x00000000u}, + {0x40a00ff3u, 0x00000000u, 0x00000000u}, + {0x41a09033u, 0x00000000u, 0x00000000u}, + {0x43215083u, 0x00000000u, 0x00000000u}, + {0x44a210e3u, 0x00000000u, 0x00000000u}, + {0x4622d143u, 0x00000000u, 0x00000000u}, + {0x47235193u, 0x00000000u, 0x00000000u}, + {0x4823d1d4u, 0x00000091u, 0x00000000u}, + {0x49a49233u, 0x00000000u, 0x00000000u}, + {0x4aa51273u, 0x00000000u, 0x00000000u}, + {0x002592b2u, 0x00000000u, 0x00000000u}, + {0x0025d2d2u, 0x00000000u, 0x00000000u}, + {0x00265302u, 0x00000000u, 0x00000000u}, + {0x00269332u, 0x00000000u, 0x00000000u}, + {0x00271362u, 0x00000000u, 0x00000000u}, + {0x00275392u, 0x00000000u, 0x00000000u}, + {0x002793b2u, 0x00000000u, 0x00000000u}, + {0x5027d3d3u, 0x00000000u, 0x00000000u}, + {0x00285412u, 0x00000000u, 0x00000000u}, + {0x51a89433u, 0x00000000u, 0x00000000u}, + {0x00291472u, 0x00000000u, 0x00000000u}, + {0x00295492u, 0x00000000u, 0x00000000u}, + {0x0029d4c2u, 0x00000000u, 0x00000000u}, + {0x002a14f2u, 0x00000000u, 0x00000000u}, + {0x552a5513u, 0x00000000u, 0x00000000u}, + {0x002ad552u, 0x00000000u, 0x00000000u}, + {0x002b1572u, 0x00000000u, 0x00000000u}, + {0x002b95a2u, 0x00000000u, 0x00000000u}, + {0x002bd5d2u, 0x00000000u, 0x00000000u}, + {0x58ac15f3u, 0x00000000u, 0x00000000u}, + {0x002c9632u, 0x00000000u, 0x00000000u}, + {0x002cd652u, 0x00000000u, 0x00000000u}, + {0x5aad1673u, 0x00000000u, 0x00000000u}, + {0x5bad96b3u, 0x00000000u, 0x00000000u}, + {0x002e16f2u, 0x00000000u, 0x00000000u}, + {0x002e9722u, 0x00000000u, 0x00000000u}, + {0x5e2ed753u, 0x00000000u, 0x00000000u}, + {0x5f2f5793u, 0x00000000u, 0x00000000u}, + {0x602fd7d3u, 0x00000000u, 0x00000000u}, + {0x61305813u, 0x00000000u, 0x00000000u}, + {0x0030d852u, 0x00000000u, 0x00000000u}, + {0x62b11873u, 0x00000000u, 0x00000000u}, + {0x0031d8c2u, 0x00000000u, 0x00000000u}, + {0x64b218f3u, 0x00000000u, 0x00000000u}, + {0x65b29933u, 0x00000000u, 0x00000000u}, + {0x00331972u, 0x00000000u, 0x00000000u}, + {0x003399a2u, 0x00000000u, 0x00000000u}, + {0x6833d9d3u, 0x00000000u, 0x00000000u}, + {0x69345a13u, 0x00000000u, 0x00000000u}, + {0x0034da52u, 0x00000000u, 0x00000000u}, + {0x00004002u, 0x00000000u, 0x00000000u}, + {0x00008032u, 0x00000000u, 0x00000000u}, + {0x00010062u, 0x00000000u, 0x00000000u}, + {0x00014092u, 0x00000000u, 0x00000000u}, + {0x000180b2u, 0x00000000u, 0x00000000u}, + {0x000200e2u, 0x00000000u, 0x00000000u}, + {0x00024112u, 0x00000000u, 0x00000000u}, + {0x0002c142u, 0x00000000u, 0x00000000u}, + {0x00030172u, 0x00000000u, 0x00000000u}, + {0x00034192u, 0x00000000u, 0x00000000u}, + {0x078381b3u, 0x00000000u, 0x00000000u}, + {0x000401f2u, 0x00000000u, 0x00000000u}, + {0x00044212u, 0x00000000u, 0x00000000u}, + {0x0004c242u, 0x00000000u, 0x00000000u}, + {0x00050272u, 0x00000000u, 0x00000000u}, + {0x00054292u, 0x00000000u, 0x00000000u}, + {0x0005c2c2u, 0x00000000u, 0x00000000u}, + {0x000602f2u, 0x00000000u, 0x00000000u}, + {0x0d064313u, 0x00000000u, 0x00000000u}, + {0x0006c352u, 0x00000000u, 0x00000000u}, + {0x00070372u, 0x00000000u, 0x00000000u}, + {0x000783a2u, 0x00000000u, 0x00000000u}, + {0x0007c3d2u, 0x00000000u, 0x00000000u}, + {0x000803f2u, 0x00000000u, 0x00000000u}, + {0x11084413u, 0x00000000u, 0x00000000u}, + {0x0008c452u, 0x00000000u, 0x00000000u}, + {0x00090472u, 0x00000000u, 0x00000000u}, + {0x000984a2u, 0x00000000u, 0x00000000u}, + {0x0009c4d2u, 0x00000000u, 0x00000000u}, + {0x148a04f3u, 0x00000000u, 0x00000000u}, + {0x000a8532u, 0x00000000u, 0x00000000u}, + {0x000ac552u, 0x00000000u, 0x00000000u}, + {0x168b0573u, 0x00000000u, 0x00000000u}, + {0x000b85b2u, 0x00000000u, 0x00000000u}, + {0x000bc5d2u, 0x00000000u, 0x00000000u}, + {0x000c4602u, 0x00000000u, 0x00000000u}, + {0x000c8632u, 0x00000000u, 0x00000000u}, + {0x000cc652u, 0x00000000u, 0x00000000u}, + {0x000d4682u, 0x00000000u, 0x00000000u}, + {0x000d86b2u, 0x00000000u, 0x00000000u}, + {0x000dc6d2u, 0x00000000u, 0x00000000u}, + {0x000e4702u, 0x00000000u, 0x00000000u}, + {0x000e8732u, 0x00000000u, 0x00000000u}, + {0x000f0762u, 0x00000000u, 0x00000000u}, + {0x000f4792u, 0x00000000u, 0x00000000u}, + {0x000f87b2u, 0x00000000u, 0x00000000u}, + {0x001007e2u, 0x00000000u, 0x00000000u}, + {0x00104812u, 0x00000000u, 0x00000000u}, + {0x21908833u, 0x00000000u, 0x00000000u}, + {0x22910873u, 0x00000000u, 0x00000000u}, + {0x239188b3u, 0x00000000u, 0x00000000u}, + {0x249208f3u, 0x00000000u, 0x00000000u}, + {0x00128932u, 0x00000000u, 0x00000000u}, + {0x2612c953u, 0x00000000u, 0x00000000u}, + {0x27134993u, 0x00000000u, 0x00000000u}, + {0x0013c9d2u, 0x00000000u, 0x00000000u}, + {0x00144a02u, 0x00000000u, 0x00000000u}, + {0x0014ca42u, 0x00000000u, 0x00000000u}, + {0x2a950a73u, 0x00000000u, 0x00000000u}, + {0x00158ab2u, 0x00000000u, 0x00000000u}, + {0x00160ae2u, 0x00000000u, 0x00000000u}, + {0x00168b22u, 0x00000000u, 0x00000000u}, + {0x2e16cb53u, 0x00000000u, 0x00000000u}, + {0x2f174b93u, 0x00000000u, 0x00000000u}, + {0x0017cbd2u, 0x00000000u, 0x00000000u}, + {0x00184c02u, 0x00000000u, 0x00000000u}, + {0x00188c32u, 0x00000000u, 0x00000000u}, + {0x00190c62u, 0x00000000u, 0x00000000u}, + {0x33194c94u, 0x00000067u, 0x00000000u}, + {0x349a0cf3u, 0x00000000u, 0x00000000u}, + {0x359a8d34u, 0x0000006cu, 0x00000000u}, + {0x371b4d93u, 0x00000000u, 0x00000000u}, + {0x389c0de3u, 0x00000000u, 0x00000000u}, + {0x3a1cce43u, 0x00000000u, 0x00000000u}, + {0x3b1d4e93u, 0x00000000u, 0x00000000u}, + {0x3c1dced4u, 0x00000079u, 0x00000000u}, + {0x3d9e8f33u, 0x00000000u, 0x00000000u}, + {0x3f1f4f83u, 0x00000000u, 0x00000000u}, + {0x401fcfd3u, 0x00000000u, 0x00000000u}, + {0x41a09023u, 0x00000000u, 0x00000000u}, + {0x42a11074u, 0x00000086u, 0x00000000u}, + {0x4421d0d3u, 0x00000000u, 0x00000000u}, + {0x45225113u, 0x00000000u, 0x00000000u}, + {0x46a31163u, 0x00000000u, 0x00000000u}, + {0x4823d1c3u, 0x00000000u, 0x00000000u}, + {0x49245213u, 0x00000000u, 0x00000000u}, + {0x00251262u, 0x00000000u, 0x00000000u}, + {0x00255292u, 0x00000000u, 0x00000000u}, + {0x0025d2c2u, 0x00000000u, 0x00000000u}, + {0x002612f2u, 0x00000000u, 0x00000000u}, + {0x4d265313u, 0x00000000u, 0x00000000u}, + {0x0026d352u, 0x00000000u, 0x00000000u}, + {0x00271372u, 0x00000000u, 0x00000000u}, + {0x002793a2u, 0x00000000u, 0x00000000u}, + {0x0027d3d2u, 0x00000000u, 0x00000000u}, + {0x002813f2u, 0x00000000u, 0x00000000u}, + {0x00289422u, 0x00000000u, 0x00000000u}, + {0x0028d452u, 0x00000000u, 0x00000000u}, + {0x52a91473u, 0x00000000u, 0x00000000u}, + {0x002994b2u, 0x00000000u, 0x00000000u}, + {0x5429d4d3u, 0x00000000u, 0x00000000u}, + {0x002a5512u, 0x00000000u, 0x00000000u}, + {0x002a9532u, 0x00000000u, 0x00000000u}, + {0x002ad552u, 0x00000000u, 0x00000000u}, + {0x002b5582u, 0x00000000u, 0x00000000u}, + {0x002b95b2u, 0x00000000u, 0x00000000u}, + {0x582bd5d3u, 0x00000000u, 0x00000000u}, + {0x002c5612u, 0x00000000u, 0x00000000u}, + {0x002c9632u, 0x00000000u, 0x00000000u}, + {0x002d1662u, 0x00000000u, 0x00000000u}, + {0x002d5692u, 0x00000000u, 0x00000000u}, + {0x002dd6c2u, 0x00000000u, 0x00000000u}, + {0x5cae16f3u, 0x00000000u, 0x00000000u}, + {0x5dae9733u, 0x00000000u, 0x00000000u}, + {0x5eaf1773u, 0x00000000u, 0x00000000u}, + {0x002f97b2u, 0x00000000u, 0x00000000u}, + {0x003017e2u, 0x00000000u, 0x00000000u}, + {0x61305813u, 0x00000000u, 0x00000000u}, + {0x6230d853u, 0x00000000u, 0x00000000u}, + {0x63315893u, 0x00000000u, 0x00000000u}, + {0x0031d8d2u, 0x00000000u, 0x00000000u}, + {0x64b218f3u, 0x00000000u, 0x00000000u}, + {0x65b29933u, 0x00000000u, 0x00000000u}, + {0x66b31973u, 0x00000000u, 0x00000000u}, + {0x003399b2u, 0x00000000u, 0x00000000u}, + {0x003419e2u, 0x00000000u, 0x00000000u}, + {0x69345a13u, 0x00000000u, 0x00000000u}, + {0x0034da52u, 0x00000000u, 0x00000000u}, +}; +static_assert(sizeof(kFc2ListsPacked) == 49920, "packed list constant extent"); +#endif + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 + +// ---------------------------------------------------------------------------------------------------------------- +// PTX helpers +// ---------------------------------------------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t smemU32(void const* p) +{ + return static_cast(__cvta_generic_to_shared(p)); +} + +__device__ __forceinline__ uint64_t globalTimer() +{ + uint64_t t; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)); + return t; +} + +__device__ __forceinline__ void mbarInit(uint32_t bar, uint32_t count) +{ + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" ::"r"(bar), "r"(count) : "memory"); +} + +__device__ __forceinline__ void mbarExpectTx(uint32_t bar, uint32_t bytes) +{ + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;" ::"r"(bar), "r"(bytes) : "memory"); +} + +__device__ __forceinline__ bool mbarTryWait(uint32_t bar, uint32_t parity) +{ + uint32_t ok; + asm volatile( + "{\n" + ".reg .pred p;\n" + "mbarrier.try_wait.parity.shared::cta.b64 p, [%1], %2;\n" + "selp.u32 %0, 1, 0, p;\n" + "}\n" + : "=r"(ok) + : "r"(bar), "r"(parity) + : "memory"); + return ok != 0; +} + +__device__ __forceinline__ void bulkCopy(uint32_t dst, void const* src, uint32_t bytes, uint32_t bar) +{ + asm volatile("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];" ::"r"(dst), + "l"(src), "r"(bytes), "r"(bar) + : "memory"); +} + +// The same with an L2 cache policy (weights streamed once per layer: evict first, so they do not push the kernel's +// code, the activations and the KV rows out of L2). +__device__ __forceinline__ void bulkCopyHint( + uint32_t dst, void const* src, uint32_t bytes, uint32_t bar, uint64_t policy) +{ + asm volatile( + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [%0], [%1], %2, [%3], %4;" :: + "r"(dst), + "l"(src), "r"(bytes), "r"(bar), "l"(policy) + : "memory"); +} + +__device__ __forceinline__ void prefetchBulkL2(void const* src, uint32_t bytes) +{ + asm volatile("cp.async.bulk.prefetch.L2.global [%0], %1;" ::"l"(src), "r"(bytes) : "memory"); +} + +__device__ __forceinline__ void prefetchBulkL2Hint(void const* src, uint32_t bytes, uint64_t policy) +{ + asm volatile("cp.async.bulk.prefetch.L2.global.L2::cache_hint [%0], %1, %2;" ::"l"(src), "r"(bytes), "l"(policy) + : "memory"); +} + +__device__ __forceinline__ uint64_t policyEvictFirst() +{ + uint64_t p; + asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(p)); + return p; +} + +__device__ __forceinline__ void fenceProxyAsyncSmem() +{ + asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); +} + +__device__ __forceinline__ void fenceProxyAsyncGlobal() +{ + asm volatile("fence.proxy.async.global;" ::: "memory"); +} + +__device__ __forceinline__ uint32_t ldAcquire(uint32_t const* p) +{ + uint32_t v; + asm volatile("ld.acquire.gpu.global.u32 %0, [%1];" : "=r"(v) : "l"(p) : "memory"); + return v; +} + +__device__ __forceinline__ void redReleaseAdd(uint32_t* p, uint32_t v) +{ + asm volatile("red.release.gpu.global.add.u32 [%0], %1;" ::"l"(p), "r"(v) : "memory"); +} + +__device__ __forceinline__ void redRelaxedAdd(uint32_t* p, uint32_t v) +{ + asm volatile("red.relaxed.gpu.global.add.u32 [%0], %1;" ::"l"(p), "r"(v) : "memory"); +} + +__device__ __forceinline__ uint4 ldcg128(void const* p) +{ + uint4 v; + asm volatile("ld.global.cg.v4.u32 {%0, %1, %2, %3}, [%4];" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "l"(p)); + return v; +} + +__device__ __forceinline__ uint4 ldVolatile128(void const* p) +{ + uint4 v; + asm volatile("ld.volatile.global.v4.u32 {%0, %1, %2, %3}, [%4];" + : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) + : "l"(p) + : "memory"); + return v; +} + +__device__ __forceinline__ void st128(void* p, uint4 v) +{ + asm volatile("st.global.v4.b32 [%0], {%1, %2, %3, %4};" ::"l"(p), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w) + : "memory"); +} + +// st128 under an L2 cache policy (createpolicy). +__device__ __forceinline__ void st128Hint(void* p, uint4 v, uint64_t policy) +{ + asm volatile("st.global.L2::cache_hint.v4.b32 [%0], {%1, %2, %3, %4}, %5;" ::"l"(p), "r"(v.x), "r"(v.y), "r"(v.z), + "r"(v.w), "l"(policy) + : "memory"); +} + +__device__ __forceinline__ uint4 lds128(uint32_t addr) +{ + uint4 v; + asm volatile("ld.shared.v4.u32 {%0, %1, %2, %3}, [%4];" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "r"(addr)); + return v; +} + +__device__ __forceinline__ uint16_t lds16(uint32_t addr) +{ + uint16_t v; + asm volatile("ld.shared.u16 %0, [%1];" : "=h"(v) : "r"(addr)); + return v; +} + +__device__ __forceinline__ uint32_t lds32(uint32_t addr) +{ + uint32_t v; + asm volatile("ld.shared.u32 %0, [%1];" : "=r"(v) : "r"(addr)); + return v; +} + +__device__ __forceinline__ uint2 lds64(uint32_t addr) +{ + uint2 v; + asm volatile("ld.shared.v2.u32 {%0, %1}, [%2];" : "=r"(v.x), "=r"(v.y) : "r"(addr)); + return v; +} + +__device__ __forceinline__ void sts32(uint32_t addr, uint32_t v) +{ + asm volatile("st.shared.u32 [%0], %1;" ::"r"(addr), "r"(v) : "memory"); +} + +__device__ __forceinline__ void sts64(uint32_t addr, uint2 v) +{ + asm volatile("st.shared.v2.u32 [%0], {%1, %2};" ::"r"(addr), "r"(v.x), "r"(v.y) : "memory"); +} + +__device__ __forceinline__ void sts128(uint32_t addr, uint4 v) +{ + asm volatile("st.shared.v4.u32 [%0], {%1, %2, %3, %4};" ::"r"(addr), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w) + : "memory"); +} + +__device__ __forceinline__ void redsAdd(uint32_t addr, uint32_t v) +{ + asm volatile("red.shared.add.u32 [%0], %1;" ::"r"(addr), "r"(v) : "memory"); +} + +// As redsAdd, without the memory clobber: the compiler keeps it in order with the other volatile asm (the shared loads +// and stores here) but need not treat it as touching every address. +__device__ __forceinline__ void redsAddUnordered(uint32_t addr, uint32_t v) +{ + asm volatile("red.shared.add.u32 [%0], %1;" ::"r"(addr), "r"(v)); +} + +__device__ __forceinline__ uint32_t atomsAdd(uint32_t addr, uint32_t v) +{ + uint32_t old; + asm volatile("atom.shared.add.u32 %0, [%1], %2;" : "=r"(old) : "r"(addr), "r"(v) : "memory"); + return old; +} + +__device__ __forceinline__ void mmaBf16( + float (&d)[4], uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1) +{ + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, " + "{%0, %1, %2, %3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)); +} + +__device__ __forceinline__ void mmaF16( + float (&d)[4], uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1) +{ + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, " + "{%0, %1, %2, %3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)); +} + +__device__ __forceinline__ void ldmatrixX4(uint32_t addr, uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3) +{ + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];" + : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) + : "r"(addr)); +} + +__device__ __forceinline__ void ldmatrixX4Trans(uint32_t addr, uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3) +{ + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];" + : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) + : "r"(addr)); +} + +__device__ __forceinline__ void ldmatrixX2(uint32_t addr, uint32_t& r0, uint32_t& r1) +{ + asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0, %1}, [%2];" : "=r"(r0), "=r"(r1) : "r"(addr)); +} + +// Two E4M3 values (low 16 bits) as two exact FP16 values. +__device__ __forceinline__ uint32_t e4m3x2ToHalf2(uint32_t pair) +{ + __half2_raw const h = __nv_cvt_fp8x2_to_halfraw2(static_cast<__nv_fp8x2_storage_t>(pair & 0xffffU), __NV_E4M3); + return static_cast(h.x) | (static_cast(h.y) << 16); +} + +// Two E2M1 values (one byte: element 2k in the low nibble) as two exact FP16 values. +__device__ __forceinline__ uint32_t e2m1x2ToHalf2(uint32_t byte) +{ + __half2_raw const h = __nv_cvt_fp4x2_to_halfraw2(static_cast<__nv_fp4x2_storage_t>(byte & 0xffU), __NV_E2M1); + return static_cast(h.x) | (static_cast(h.y) << 16); +} + +__device__ __forceinline__ uint32_t hmul2(uint32_t a, uint32_t b) +{ + uint32_t d; + asm("mul.rn.f16x2 %0, %1, %2;" : "=r"(d) : "r"(a), "r"(b)); + return d; +} + +__device__ __forceinline__ uint8_t toE4m3(float x) +{ + return static_cast(__nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3)); +} + +// Two values as E4M3 (round to nearest, saturating) in the low 16 bits: lo in bits 0-7. +__device__ __forceinline__ uint32_t toE4m3x2(float lo, float hi) +{ + return static_cast(__nv_cvt_float2_to_fp8x2(make_float2(lo, hi), __NV_SATFINITE, __NV_E4M3)); +} + +__device__ __forceinline__ float rcpApproxFtz(float x) +{ + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + +__device__ __forceinline__ float ex2ApproxFtz(float x) +{ + float y; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + +// E2M1 of (lo, hi) (round to nearest, saturating): lo in the low nibble of the returned byte. +__device__ __forceinline__ uint32_t toE2m1x2(float lo, float hi) +{ + uint32_t v; + asm("{\n" + ".reg .b8 b;\n" + "cvt.rn.satfinite.e2m1x2.f32 b, %2, %1;\n" + "mov.b32 %0, {b, b, b, b};\n" + "}" + : "=r"(v) + : "f"(lo), "f"(hi)); + return v & 0xffu; +} + +__device__ __forceinline__ float bf16Round(float x) +{ + return __bfloat162float(__float2bfloat16_rn(x)); +} + +__device__ __forceinline__ uint32_t packBf16(float lo, float hi) +{ + __nv_bfloat162 const v = __floats2bfloat162_rn(lo, hi); + return *reinterpret_cast(&v); +} + +__device__ __forceinline__ float bf16Lo(uint32_t v) +{ + return __uint_as_float(v << 16); +} + +__device__ __forceinline__ float bf16Hi(uint32_t v) +{ + return __uint_as_float(v & 0xffff0000u); +} + +__device__ __forceinline__ uint4 ldPoll128(void const* p) +{ + uint4 v; + asm volatile("ld.volatile.global.v4.u32 {%0, %1, %2, %3}, [%4];" + : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) + : "l"(p)); + return v; +} + +__device__ __forceinline__ uint2 ldPoll64(void const* p) +{ + uint2 v; + asm volatile("ld.volatile.global.v2.u32 {%0, %1}, [%2];" : "=r"(v.x), "=r"(v.y) : "l"(p)); + return v; +} + +__device__ __forceinline__ uint32_t ldPoll32(void const* p) +{ + uint32_t v; + asm volatile("ld.volatile.global.u32 %0, [%1];" : "=r"(v) : "l"(p)); + return v; +} + +// One 64-bit word (single-copy atomic against stU64's store, unlike a two-element vector). +__device__ __forceinline__ uint64_t ldPollU64(void const* p) +{ + uint64_t v; + asm volatile("ld.volatile.global.u64 %0, [%1];" : "=l"(v) : "l"(p)); + return v; +} + +// A strong (relaxed, GPU-scope) store: with ldPollU64's volatile load a morally strong pair, so a poller sees the whole +// word or none of it (PTX promises single-copy atomicity only between morally strong accesses; a weak store is not). +__device__ __forceinline__ void stU64(void* p, uint64_t v) +{ + asm volatile("st.relaxed.gpu.global.u64 [%0], %1;" ::"l"(p), "l"(v) : "memory"); +} + +// Read-only loads the compiler keeps where they are written (issued before the polls that follow them). +__device__ __forceinline__ int ldPinnedS32(int const* p) +{ + int v; + asm volatile("ld.global.nc.s32 %0, [%1];" : "=r"(v) : "l"(p)); + return v; +} + +__device__ __forceinline__ uint32_t ldPinnedU16(void const* p) +{ + uint16_t v; + asm volatile("ld.global.nc.u16 %0, [%1];" : "=h"(v) : "l"(p)); + return v; +} + +__device__ __forceinline__ float4 ldPinnedF4(void const* p) +{ + float4 v; + asm volatile("ld.global.nc.v4.f32 {%0, %1, %2, %3}, [%4];" : "=f"(v.x), "=f"(v.y), "=f"(v.z), "=f"(v.w) : "l"(p)); + return v; +} + +__device__ __forceinline__ void st64(void* p, uint2 v) +{ + asm volatile("st.global.v2.b32 [%0], {%1, %2};" ::"l"(p), "r"(v.x), "r"(v.y) : "memory"); +} + +__device__ __forceinline__ void st32(void* p, uint32_t v) +{ + asm volatile("st.global.b32 [%0], %1;" ::"l"(p), "r"(v) : "memory"); +} + +__device__ __forceinline__ void prefetchL2(void const* p, uint32_t bytes) +{ + asm volatile("cp.async.bulk.prefetch.L2.global [%0], %1;" ::"l"(p), "r"(bytes) : "memory"); +} + +// ---------------------------------------------------------------------------------------------------------------- +// Tensor-core NVFP4 MMA (tcgen05 kind::mxf4nvf4 block16): D[128 weight rows][8 tokens] FP32 in TMEM, A and B E2M1 in +// shared memory (K-major SWIZZLE_64B), their UE4M3 block scales in TMEM. +// ---------------------------------------------------------------------------------------------------------------- +constexpr uint32_t kIdescNvf4 = (1u << 7) | (1u << 10) | ((8u >> 3) << 17) | ((128u >> 4) << 24); // M 128, N 8 +// K-major SWIZZLE_64B operand descriptor without its start address: 8-row x 64 B atoms 512 B apart, version 1. +constexpr uint64_t kDescSw64 = (1ull << 16) | (static_cast(512 >> 4) << 32) | (1ull << 46) | (4ull << 61); +// tcgen05.cp 32x128b source descriptor without its start address: 8-row x 16 B core matrices 128 B apart. +constexpr uint64_t kDescSf = (static_cast(128 >> 4) << 32) | (1ull << 46); +constexpr uint32_t kTmemCols = 512; +constexpr uint32_t kTmemD = 0; // accumulators: FC1 8 columns, FC2 6 tiles x 8 +constexpr uint32_t kTmemSfa = 64; // weight scales of ring slot rs: [64 + 48 rs, +48), 4 columns per K step +constexpr uint32_t kTmemSfb = 256; // FC1 token scales of the 96 K steps: per K step a column of scales and a zero one +constexpr uint32_t kTmemSfb2 = 448; // FC2 token scales (2 K steps) + +__device__ __forceinline__ void tcFenceBefore() +{ + asm volatile("tcgen05.fence::before_thread_sync;" ::: "memory"); +} + +__device__ __forceinline__ void tcFenceAfter() +{ + asm volatile("tcgen05.fence::after_thread_sync;" ::: "memory"); +} + +// The following five are executed by a whole (converged) warp. +__device__ __forceinline__ void tmemAlloc(uint32_t dst, uint32_t cols) +{ + asm volatile("tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" ::"r"(dst), "r"(cols) : "memory"); + asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;" ::: "memory"); +} + +__device__ __forceinline__ void tmemDealloc(uint32_t taddr, uint32_t cols) +{ + asm volatile("tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" ::"r"(taddr), "r"(cols) : "memory"); +} + +// 512 B of block scales (shared memory, 128x4 layout) into 4 TMEM columns; the elected lane issues. +__device__ __forceinline__ void utccpElect(uint32_t taddr, uint32_t src) +{ + uint64_t const sdesc = kDescSf | ((src & 0x3ffffu) >> 4); + asm volatile( + "{\n" + ".reg .pred e;\n" + "elect.sync _|e, 0xffffffff;\n" + "@e tcgen05.cp.cta_group::1.32x128b.warpx4 [%0], %1;\n" + "}\n" ::"r"(taddr), + "l"(sdesc) + : "memory"); +} + +// D (+)= A B for one K step of 64; a and b are shared addresses of the K step's first byte (atom-aligned rows). +__device__ __forceinline__ void mmaNvf4Elect( + uint32_t d, uint32_t a, uint32_t b, uint32_t sfa, uint32_t sfb, uint32_t acc) +{ + uint64_t const da = kDescSw64 | ((a & 0x3ffffu) >> 4); + uint64_t const db = kDescSw64 | ((b & 0x3ffffu) >> 4); + asm volatile( + "{\n" + ".reg .pred e, p;\n" + "elect.sync _|e, 0xffffffff;\n" + "setp.ne.b32 p, %6, 0;\n" + "@e tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16 [%0], %1, %2, %3, [%4], [%5], p;\n" + "}\n" ::"r"(d), + "l"(da), "l"(db), "r"(kIdescNvf4), "r"(sfa), "r"(sfb), "r"(acc) + : "memory"); +} + +// One lane of the (converged) warp: true in exactly one lane. A batch of the single-thread tcgen05 instructions +// below under one election compiles to straight code (an election per instruction added a divergence check and a +// fallback copy of every instruction). +__device__ __forceinline__ bool electOne() +{ + uint32_t e; + asm volatile( + "{\n" + ".reg .pred p;\n" + "elect.sync _|p, 0xffffffff;\n" + "selp.u32 %0, 1, 0, p;\n" + "}\n" + : "=r"(e)); + return e != 0; +} + +#if GLM52_IO_MODE == 2 || GLM52_IO_MODE == 3 +// As utccp1, all four 8-row groups from the same 128 bytes (stride 0): rows 8-31 repeat rows 0-7. +__device__ __forceinline__ void utccp1Rows8(uint32_t taddr, uint32_t src) +{ + uint64_t const sdesc = (1ull << 46) | ((src & 0x3ffffu) >> 4); + asm volatile("tcgen05.cp.cta_group::1.32x128b.warpx4 [%0], %1;" ::"r"(taddr), "l"(sdesc) : "memory"); +} + +#endif + +__device__ __forceinline__ void utccp1(uint32_t taddr, uint32_t src) +{ + uint64_t const sdesc = kDescSf | ((src & 0x3ffffu) >> 4); + asm volatile("tcgen05.cp.cta_group::1.32x128b.warpx4 [%0], %1;" ::"r"(taddr), "l"(sdesc) : "memory"); +} + +__device__ __forceinline__ void mmaNvf4(uint32_t d, uint32_t a, uint32_t b, uint32_t sfa, uint32_t sfb, uint32_t acc) +{ + uint64_t const da = kDescSw64 | ((a & 0x3ffffu) >> 4); + uint64_t const db = kDescSw64 | ((b & 0x3ffffu) >> 4); + asm volatile( + "{\n" + ".reg .pred p;\n" + "setp.ne.b32 p, %6, 0;\n" + "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16 [%0], %1, %2, %3, [%4], [%5], p;\n" + "}\n" ::"r"(d), + "l"(da), "l"(db), "r"(kIdescNvf4), "r"(sfa), "r"(sfb), "r"(acc) + : "memory"); +} + +__device__ __forceinline__ void commit1(uint32_t bar) +{ + asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];" ::"r"(bar) : "memory"); +} + +// K-major SWIZZLE_128B operand descriptor without its start address: 8-row x 128 B atoms 1024 B apart, version 1. +constexpr uint64_t kDescSw128 = (1ull << 16) | (static_cast(1024 >> 4) << 32) | (1ull << 46) | (2ull << 61); +// tcgen05.mma kind::f8f6f4: E4M3 x E4M3 -> FP32, both operands K-major, M 128, N 256. +constexpr uint32_t kIdescF8 = (1u << 4) | ((256u >> 3) << 17) | ((128u >> 4) << 24); +constexpr uint32_t kIdescF8N32 = (1u << 4) | ((32u >> 3) << 17) | ((128u >> 4) << 24); // one row's 32 heads + +// D (+)= A B^T for one K step of 32 E4M3 values; a and b are shared addresses of the step's first byte in +// SWIZZLE_128B atoms (A: 128 rows, B: 256 rows). +__device__ __forceinline__ void mmaF8(uint32_t d, uint32_t a, uint32_t b, uint32_t acc, uint32_t idesc = kIdescF8) +{ + uint64_t const da = kDescSw128 | ((a & 0x3ffffu) >> 4); + uint64_t const db = kDescSw128 | ((b & 0x3ffffu) >> 4); + asm volatile( + "{\n" + ".reg .pred p;\n" + "setp.ne.b32 p, %4, 0;\n" + "tcgen05.mma.cta_group::1.kind::f8f6f4 [%0], %1, %2, %3, p;\n" + "}\n" ::"r"(d), + "l"(da), "l"(db), "r"(idesc), "r"(acc) + : "memory"); +} + +// One box of a 2-D tensor map (global tensor-map address) into shared memory on barrier bar (one thread). +__device__ __forceinline__ void tmaLoad2d(uint32_t dst, void const* tmap, int x, int y, uint32_t bar) +{ + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3}], [%4];" ::"r"( + dst), + "l"(tmap), "r"(x), "r"(y), "r"(bar) + : "memory"); +} + +// The same with an L2 cache policy. +__device__ __forceinline__ void tmaLoad2dHint( + uint32_t dst, void const* tmap, int x, int y, uint32_t bar, uint64_t policy) +{ + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint " + "[%0], [%1, {%2, %3}], [%4], %5;" ::"r"(dst), + "l"(tmap), "r"(x), "r"(y), "r"(bar), "l"(policy) + : "memory"); +} + +// 32 columns of the warp's TMEM lane quarter, one register each (the caller waits). +__device__ __forceinline__ void tmemLd32(uint32_t taddr, uint32_t (&r)[32]) +{ + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x32.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, " + "%16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), "=r"(r[8]), + "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]), "=r"(r[16]), + "=r"(r[17]), "=r"(r[18]), "=r"(r[19]), "=r"(r[20]), "=r"(r[21]), "=r"(r[22]), "=r"(r[23]), "=r"(r[24]), + "=r"(r[25]), "=r"(r[26]), "=r"(r[27]), "=r"(r[28]), "=r"(r[29]), "=r"(r[30]), "=r"(r[31]) + : "r"(taddr) + : "memory"); + asm volatile("tcgen05.wait::ld.sync.aligned;" + : "+r"(r[0]), "+r"(r[1]), "+r"(r[2]), "+r"(r[3]), "+r"(r[4]), "+r"(r[5]), "+r"(r[6]), "+r"(r[7]), + "+r"(r[8]), "+r"(r[9]), "+r"(r[10]), "+r"(r[11]), "+r"(r[12]), "+r"(r[13]), "+r"(r[14]), "+r"(r[15]), + "+r"(r[16]), "+r"(r[17]), "+r"(r[18]), "+r"(r[19]), "+r"(r[20]), "+r"(r[21]), "+r"(r[22]), "+r"(r[23]), + "+r"(r[24]), "+r"(r[25]), "+r"(r[26]), "+r"(r[27]), "+r"(r[28]), "+r"(r[29]), "+r"(r[30]), "+r"(r[31]) + : + : "memory"); +} + +// Columns [taddr, taddr + 128) of the warp's TMEM lane quarter: four 32-column loads in flight, one wait (the empty +// statements after it order the other registers' uses behind it). +__device__ __forceinline__ void tmemLd128(uint32_t taddr, uint32_t (&r)[128]) +{ +#pragma unroll + for (int c = 0; c < 4; ++c) + { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x32.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, " + "%15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];" + : "=r"(r[32 * c + 0]), "=r"(r[32 * c + 1]), "=r"(r[32 * c + 2]), "=r"(r[32 * c + 3]), "=r"(r[32 * c + 4]), + "=r"(r[32 * c + 5]), "=r"(r[32 * c + 6]), "=r"(r[32 * c + 7]), "=r"(r[32 * c + 8]), "=r"(r[32 * c + 9]), + "=r"(r[32 * c + 10]), "=r"(r[32 * c + 11]), "=r"(r[32 * c + 12]), "=r"(r[32 * c + 13]), + "=r"(r[32 * c + 14]), "=r"(r[32 * c + 15]), "=r"(r[32 * c + 16]), "=r"(r[32 * c + 17]), + "=r"(r[32 * c + 18]), "=r"(r[32 * c + 19]), "=r"(r[32 * c + 20]), "=r"(r[32 * c + 21]), + "=r"(r[32 * c + 22]), "=r"(r[32 * c + 23]), "=r"(r[32 * c + 24]), "=r"(r[32 * c + 25]), + "=r"(r[32 * c + 26]), "=r"(r[32 * c + 27]), "=r"(r[32 * c + 28]), "=r"(r[32 * c + 29]), + "=r"(r[32 * c + 30]), "=r"(r[32 * c + 31]) + : "r"(taddr + 32u * static_cast(c)) + : "memory"); + } + asm volatile("tcgen05.wait::ld.sync.aligned;" + : "+r"(r[0]), "+r"(r[1]), "+r"(r[2]), "+r"(r[3]), "+r"(r[4]), "+r"(r[5]), "+r"(r[6]), "+r"(r[7]), + "+r"(r[8]), "+r"(r[9]), "+r"(r[10]), "+r"(r[11]), "+r"(r[12]), "+r"(r[13]), "+r"(r[14]), "+r"(r[15]), + "+r"(r[16]), "+r"(r[17]), "+r"(r[18]), "+r"(r[19]), "+r"(r[20]), "+r"(r[21]), "+r"(r[22]), "+r"(r[23]), + "+r"(r[24]), "+r"(r[25]), "+r"(r[26]), "+r"(r[27]), "+r"(r[28]), "+r"(r[29]), "+r"(r[30]), "+r"(r[31]) + : + : "memory"); + asm volatile("" + : "+r"(r[32]), "+r"(r[33]), "+r"(r[34]), "+r"(r[35]), "+r"(r[36]), "+r"(r[37]), "+r"(r[38]), + "+r"(r[39]), "+r"(r[40]), "+r"(r[41]), "+r"(r[42]), "+r"(r[43]), "+r"(r[44]), "+r"(r[45]), "+r"(r[46]), + "+r"(r[47]), "+r"(r[48]), "+r"(r[49]), "+r"(r[50]), "+r"(r[51]), "+r"(r[52]), "+r"(r[53]), "+r"(r[54]), + "+r"(r[55]), "+r"(r[56]), "+r"(r[57]), "+r"(r[58]), "+r"(r[59]), "+r"(r[60]), "+r"(r[61]), "+r"(r[62]), + "+r"(r[63])); + asm volatile("" + : "+r"(r[64]), "+r"(r[65]), "+r"(r[66]), "+r"(r[67]), "+r"(r[68]), "+r"(r[69]), "+r"(r[70]), + "+r"(r[71]), "+r"(r[72]), "+r"(r[73]), "+r"(r[74]), "+r"(r[75]), "+r"(r[76]), "+r"(r[77]), "+r"(r[78]), + "+r"(r[79]), "+r"(r[80]), "+r"(r[81]), "+r"(r[82]), "+r"(r[83]), "+r"(r[84]), "+r"(r[85]), "+r"(r[86]), + "+r"(r[87]), "+r"(r[88]), "+r"(r[89]), "+r"(r[90]), "+r"(r[91]), "+r"(r[92]), "+r"(r[93]), "+r"(r[94]), + "+r"(r[95])); + asm volatile("" + : "+r"(r[96]), "+r"(r[97]), "+r"(r[98]), "+r"(r[99]), "+r"(r[100]), "+r"(r[101]), "+r"(r[102]), + "+r"(r[103]), "+r"(r[104]), "+r"(r[105]), "+r"(r[106]), "+r"(r[107]), "+r"(r[108]), "+r"(r[109]), + "+r"(r[110]), "+r"(r[111]), "+r"(r[112]), "+r"(r[113]), "+r"(r[114]), "+r"(r[115]), "+r"(r[116]), + "+r"(r[117]), "+r"(r[118]), "+r"(r[119]), "+r"(r[120]), "+r"(r[121]), "+r"(r[122]), "+r"(r[123]), + "+r"(r[124]), "+r"(r[125]), "+r"(r[126]), "+r"(r[127])); +} + +// The indexer's E4M3 scale of a row with this |max|: max(amax, 1e-12) / 448 (FP32 constants), rounded up to a power +// of two (UE8M0), as the stock prologue and Q tail. +__device__ __forceinline__ float ue8m0Scale(float amax) +{ + float const ratio = __fmul_rn(fmaxf(amax, __uint_as_float(0x2B8CBCCCu)), __uint_as_float(0x3B124925u)); + uint32_t const bits = __float_as_uint(ratio); + uint32_t const e = (bits & 0x7F800000u) + ((bits & 0x007FFFFFu) != 0 ? 0x00800000u : 0u); + return __uint_as_float(e); +} + +// Order-preserving u32 of an FP32 value (larger value, larger key). +__device__ __forceinline__ uint32_t orderKey(float f) +{ + uint32_t const u = __float_as_uint(f); + return u ^ ((u >> 31) != 0 ? 0xffffffffu : 0x80000000u); +} + +// The barrier at shared address bar arrives once the MMAs issued so far are complete. +__device__ __forceinline__ void commitElect(uint32_t bar) +{ + asm volatile( + "{\n" + ".reg .pred e;\n" + "elect.sync _|e, 0xffffffff;\n" + "@e tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];\n" + "}\n" ::"r"(bar) + : "memory"); +} + +// Columns [taddr, taddr + kCols) of the warp's TMEM lane quarter (lane = row). +template +__device__ __forceinline__ void tmemLd(uint32_t taddr, float (&v)[kCols]) +{ + static_assert(kCols == 8 || kCols == 16, "8 or 16 columns"); + uint32_t r[kCols]; + if constexpr (kCols == 8) + { + asm volatile("tcgen05.ld.sync.aligned.32x32b.x8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]) + : "r"(taddr) + : "memory"); + } + else + { + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x16.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, " + "%15}, [%16];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), + "=r"(r[8]), "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]) + : "r"(taddr) + : "memory"); + } + asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); +#pragma unroll + for (int i = 0; i < kCols; ++i) + { + v[i] = __uint_as_float(r[i]); + } +} + +// Columns [taddr, taddr + 24) of the warp's TMEM lane quarter as three 8-column tiles: both loads are issued before the +// one wait (the registers are tied to the wait, so no use moves above it). +__device__ __forceinline__ void tmemLd3x8(uint32_t taddr, float (&v)[3][8]) +{ + uint32_t r[24]; + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x16.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, " + "%15}, [%16];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]), "=r"(r[4]), "=r"(r[5]), "=r"(r[6]), "=r"(r[7]), "=r"(r[8]), + "=r"(r[9]), "=r"(r[10]), "=r"(r[11]), "=r"(r[12]), "=r"(r[13]), "=r"(r[14]), "=r"(r[15]) + : "r"(taddr) + : "memory"); + asm volatile( + "tcgen05.ld.sync.aligned.32x32b.x8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" + : "=r"(r[16]), "=r"(r[17]), "=r"(r[18]), "=r"(r[19]), "=r"(r[20]), "=r"(r[21]), "=r"(r[22]), "=r"(r[23]) + : "r"(taddr + 16) + : "memory"); + asm volatile("tcgen05.wait::ld.sync.aligned;" + : "+r"(r[0]), "+r"(r[1]), "+r"(r[2]), "+r"(r[3]), "+r"(r[4]), "+r"(r[5]), "+r"(r[6]), "+r"(r[7]), + "+r"(r[8]), "+r"(r[9]), "+r"(r[10]), "+r"(r[11]), "+r"(r[12]), "+r"(r[13]), "+r"(r[14]), "+r"(r[15]), + "+r"(r[16]), "+r"(r[17]), "+r"(r[18]), "+r"(r[19]), "+r"(r[20]), "+r"(r[21]), "+r"(r[22]), "+r"(r[23]) + : + : "memory"); +#pragma unroll + for (int i = 0; i < 3; ++i) + { +#pragma unroll + for (int j = 0; j < 8; ++j) + { + v[i][j] = __uint_as_float(r[8 * i + j]); + } + } +} + +// ---------------------------------------------------------------------------------------------------------------- +// Kernel context +// ---------------------------------------------------------------------------------------------------------------- +struct Ctx +{ + int64_t const* p; + uint8_t* smem; + uint32_t smemBase; + uint32_t* counters; + int* err; + uint64_t timeoutNs; + uint8_t* arena; // this launch's arena copy + int parity; + int rank; + uint64_t* stamps; // this CTA's stamps, or null + int stampRole; // the running unit's index in the CTA's list + int64_t flags; // variant bits (Flag) + int numCopies; // copies in the launch rotation + int leader; // a leader layer (kPLeader) +}; + +static_assert(sizeof(Ctx) <= 128, "Ctx exceeds its shared-memory slot"); + +#if GLM52_FLAGS_MODE != 0 +#ifndef GLM52_STATIC_FLAGS +#define GLM52_STATIC_FLAGS 0ll +#endif +// Options selected per arm: the host passes an arm's flags as GLM52_STATIC_FLAGS, and these bits of them are fixed into +// the build (kFlagMergeDry, kFlagLandProbe, kFlagMoeL2, kFlagMoeL2Normal, kFlagSliceMerge, kFlagPageMap2; bits 43-45: +// the layout pad). kFlagFirstEarly, +// kFlagHopSelfClear, kFlagSelFast, kFlagPageBalance, kFlagAttnIlp, kFlagHopClearLate, kFlagWarmInHop, kFlagHPrime, +// kFlagXPrime, kFlagHop1Lines, kFlagRstdShared, kFlagXFrag, kFlagUnionSide, kFlagUnionLean and kFlagCandWarps are +// always on. +constexpr int64_t kArmOptionFlags + = kFlagMergeDry | kFlagLandProbe | kFlagMoeL2 | kFlagMoeL2Normal | kFlagSliceMerge | kFlagPageMap2 | (7ll << 43); +constexpr int64_t kSpecializedDynamicFlags = kFlagEarlyX | kFlagNoStamps | kFlagNextChained; +constexpr int64_t kSpecializedStaticFlags = 704ll +#if GLM52_FLAGS_MODE >= 2 + | (1ll << 38) | (1ll << 46) | (1ll << 49) +#endif +#if GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5 + | (1ll << 54) +#endif + | kFlagFirstEarly | kFlagHopSelfClear | kFlagSelFast | kFlagPageBalance | kFlagAttnIlp | kFlagHopClearLate + | kFlagWarmInHop | kFlagHPrime | kFlagXPrime | kFlagHop1Lines | kFlagRstdShared | kFlagXFrag | kFlagUnionSide + | kFlagUnionLean | kFlagCandWarps | (static_cast(GLM52_STATIC_FLAGS) & kArmOptionFlags); +// The clear units the owners (and the next launch's fused-A) wait for: with kFlagHopSelfClear the hop buffers' units +// are not scheduled (the H1 units empty those slots), the top-k exchange's and the arena's units still count. +constexpr int kHopSlotClearUnits = 2 * kHopBytes / kClearUnitBytes; +constexpr int kCountedClear + = kCountedClearUnits - ((kSpecializedStaticFlags & kFlagHopSelfClear) != 0 ? kHopSlotClearUnits : 0); +static_assert(kRouterLg == 0 || (kSpecializedStaticFlags & kFlagSliceMerge) == 0, "kArLg shares kArPStats' space"); +static_assert(kRouterLg == 0 + || ((kSpecializedStaticFlags & kFlagHPrime) != 0 && (kSpecializedStaticFlags & kFlagRstdShared) != 0), + "the reduced logits are h' logits: the routing scales them by 1 / rms"); + +// Inline at each read, including noinline helper bodies: optional branches +// become constants while capture-dependent EarlyX/NoStamps remain runtime. +__device__ __forceinline__ int64_t specializedFlags(Ctx const& ctx) +{ + return (ctx.flags & kSpecializedDynamicFlags) | kSpecializedStaticFlags; +} +#endif + +template +__device__ __forceinline__ T* ptr(Ctx const& ctx, int idx) +{ + return reinterpret_cast(ctx.p[idx]); +} + +template +__device__ __forceinline__ T* arena(Ctx const& ctx, int off) +{ + return reinterpret_cast(ctx.arena + off); +} + +__device__ __forceinline__ void subStamp(Ctx const& ctx, int k) +{ + if (GLM52_STAMPS(ctx) != nullptr && threadIdx.x == 0) + { + GLM52_STAMPS(ctx)[kMaxRoles * 4 + 4 + k] = globalTimer(); + if (k >= 40 && k < 72) + { + GLM52_STAMPS(ctx)[kMaxRoles * 4 + 4 + 96 + k - 40] = clock64(); + } + } +} + +// The running unit's inputs are in (thread 0's view; the last call of the unit wins). +__device__ __forceinline__ void stampInput(Ctx const& ctx) +{ + if (GLM52_STAMPS(ctx) != nullptr && threadIdx.x == 0) + { + GLM52_STAMPS(ctx)[ctx.stampRole * 4 + 1] = globalTimer(); + } +} + +// A watchdog expired: the record goes to host memory (readable after the trap), then the kernel traps. +__device__ __noinline__ void fail(Ctx const& ctx, int code, int a, int b) +{ + int volatile* err = ctx.err; + err[1] = blockIdx.x; + err[2] = threadIdx.x; + err[3] = a; + err[4] = b; + err[0] = code; + __threadfence_system(); + __trap(); +} + +// Half way to the timeout, a waiting warp's lane 0 logs (code, thread, a, b) into its CTA's slot of the record +// (entries 16 + 4 cta), so a hang shows what every CTA waits for, not only the first to trap. +__device__ __noinline__ void logWaiter(Ctx const& ctx, int code, int a, int b) +{ + int volatile* err = ctx.err + 16 + 4 * blockIdx.x; + err[1] = static_cast(threadIdx.x); + err[2] = a; + err[3] = b; + err[0] = code; + __threadfence_system(); +} + +// The watchdog of a spin (out of line, so every wait inlines only its turn count): the wait began at about t0. +__device__ __noinline__ void spinSlow(Ctx const& ctx, uint64_t t0, int code, int a, int b) +{ + uint64_t const waited = globalTimer() - t0; + if (waited > ctx.timeoutNs) + { + fail(ctx, code, a, b); + } + if (waited > ctx.timeoutNs / 2 && (threadIdx.x & 31) == 0) + { + logWaiter(ctx, code, a, b); + } +} + +// One more turn of a spin. t0: the clock at the first turn, plus one per turn since (1 ns per turn of ~0.5 us); every +// 256 turns the watchdog runs. (Backing off here measured slower: 8 us per layer.) +__device__ __forceinline__ void spinCheck(Ctx const& ctx, uint64_t& t0, int code, int a, int b) +{ +#if GLM52_LEAN + return; +#endif + if (t0 == 0) + { + t0 = globalTimer(); + } + else if ((++t0 & 0xffull) == 0) + { + spinSlow(ctx, t0, code, a, b); + } +} + +// The exit test of a poll loop, taken by the whole warp at once (all 32 lanes call it). A lane that leaves a spin early +// waits at the reconvergence point while the others spin, and after a long spin it is let go on alone: the warp then +// stays split (a __syncwarp() after the loop is dropped by ptxas as redundant), so every later warp collective takes +// its slow path and the rest issues once per part (the leader top-k's candidate step ran 8.4 instead of 2.5 us). +__device__ __forceinline__ bool warpAll(bool ok) +{ + return __all_sync(0xffffffffu, ok); +} + +// ---------------------------------------------------------------------------------------------------------------- +// Data as its own flag: every 32-bit word of a buffer is kEmpty until its producer stores it. +// ---------------------------------------------------------------------------------------------------------------- +__device__ __forceinline__ uint32_t lamport(uint32_t w) +{ + return w == kEmpty ? 0u : w; +} + +__device__ __forceinline__ bool full4(uint4 v) +{ + return (v.x != kEmpty) & (v.y != kEmpty) & (v.z != kEmpty) & (v.w != kEmpty); +} + +__device__ __forceinline__ bool full2(uint2 v) +{ + return (v.x != kEmpty) & (v.y != kEmpty); +} + +__device__ __forceinline__ uint4 poll4(Ctx const& ctx, void const* p, int code) +{ + uint4 v = ldPoll128(p); + uint64_t t0 = 0; + while (!full4(v)) + { + spinCheck(ctx, t0, code, 0, 0); + v = ldPoll128(p); + } + return v; +} + +__device__ __forceinline__ uint32_t poll1(Ctx const& ctx, void const* p, int code) +{ + uint32_t v = ldPoll32(p); + uint64_t t0 = 0; + while (v == kEmpty) + { + spinCheck(ctx, t0, code, 0, 0); + v = ldPoll32(p); + } + return v; +} + +// kN 16-byte vectors at src + stride * i: all loads in flight together, then again for the ones not complete. +template +__device__ __forceinline__ void pollStrided(Ctx const& ctx, uint4 (&v)[kN], uint8_t const* src, int stride, int code) +{ +#pragma unroll + for (int i = 0; i < kN; ++i) + { + v[i] = ldPoll128(src + stride * i); + } + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int i = 0; i < kN; ++i) + { + ok &= full4(v[i]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, code, stride, 0); +#pragma unroll + for (int i = 0; i < kN; ++i) + { + if (!full4(v[i])) + { + v[i] = ldPoll128(src + stride * i); + } + } + } +} + +// nVec 16-byte vectors from src to shared memory at dst; vector j is handled by thread j mod nThreads of the threads +// [first, first + nThreads) (kN >= its count). +template +__device__ __forceinline__ void pollCopy( + Ctx const& ctx, uint8_t* dst, uint8_t const* src, int nVec, int first, int nThreads, int code) +{ + int const me = static_cast(threadIdx.x) - first; + uint4 v[kN]; +#pragma unroll + for (int i = 0; i < kN; ++i) + { + int const j = me + nThreads * i; + v[i] = j < nVec ? ldPoll128(src + 16 * j) : make_uint4(0, 0, 0, 0); + } + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int i = 0; i < kN; ++i) + { + ok &= full4(v[i]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, code, nVec, 0); +#pragma unroll + for (int i = 0; i < kN; ++i) + { + if (!full4(v[i])) + { + v[i] = ldPoll128(src + 16 * (me + nThreads * i)); + } + } + } +#pragma unroll + for (int i = 0; i < kN; ++i) + { + int const j = me + nThreads * i; + if (j < nVec) + { + *reinterpret_cast(dst + 16 * j) = v[i]; + } + } +} + +// Warp 0 polls one canary word per producer unit (addrOf(i), i < n <= 64) until none is empty; then the CTA proceeds. +template +__device__ __forceinline__ void waitCanaries(Ctx const& ctx, int n, AddrOf addrOf, int code) +{ + if (threadIdx.x < 32) + { + int const lane = threadIdx.x; + uint8_t const* a0 = lane < n ? addrOf(lane) : nullptr; + uint8_t const* a1 = lane + 32 < n ? addrOf(lane + 32) : nullptr; + uint32_t v0 = a0 != nullptr ? ldPoll32(a0) : 0u; + uint32_t v1 = a1 != nullptr ? ldPoll32(a1) : 0u; + uint64_t t0 = 0; + while (!__all_sync(0xffffffffu, (v0 != kEmpty) & (v1 != kEmpty))) + { + spinCheck(ctx, t0, code, n, 0); + if (v0 == kEmpty) + { + v0 = ldPoll32(a0); + } + if (v1 == kEmpty) + { + v1 = ldPoll32(a1); + } + } + } + __syncthreads(); +} + +// Thread 0 waits until counter >= target; the CTA then proceeds (bar.sync orders the CTA's later reads). +__device__ __forceinline__ void waitCounter(Ctx const& ctx, int idx, uint32_t target, int code) +{ + if (threadIdx.x == 0) + { + uint32_t const* c = ctx.counters + idx; + uint64_t t0 = 0; + // One acquire load per turn (the watchdog takes the value already read: a second load doubled the turn). + uint32_t v; + while ((v = ldAcquire(c)) < target) + { + spinCheck(ctx, t0, code, idx, static_cast(v)); + } + } + __syncthreads(); +} + +// Every CTA of the grid: the CTA's global writes so far, then +1 on the counter, then wait until all CTAs have +// arrived (the counter's set is this launch's; the CTA then proceeds). The barrier orders the CTA's writes before +// thread 0's release (red.release.gpu: MEMBAR.ALL.GPU, then the add), so no separate fence. +__device__ __forceinline__ void gridBarrier(Ctx const& ctx, int idx, int code) +{ + __syncthreads(); + if (threadIdx.x == 0) + { + redReleaseAdd(ctx.counters + idx, 1u); + uint64_t t0 = 0; + while (ldAcquire(ctx.counters + idx) < static_cast(kCtas)) + { + spinCheck(ctx, t0, code, idx, 0); + } + } + __syncthreads(); +} + +// The CTA's global writes so far, then +1 on the counter (as gridBarrier: the barrier, then one release). +__device__ __forceinline__ void signalCounter(Ctx const& ctx, int idx) +{ + __syncthreads(); + if (threadIdx.x == 0) + { + redReleaseAdd(ctx.counters + idx, 1u); + } +} + +__device__ __forceinline__ void waitBar(Ctx const& ctx, uint32_t bar, uint32_t parity, int code) +{ + if (!mbarTryWait(bar, parity)) + { + uint64_t t0 = 0; + while (!mbarTryWait(bar, parity)) + { + spinCheck(ctx, t0, code, static_cast(bar), static_cast(parity)); + } + } + if (GLM52_STAMPS(ctx) != nullptr && threadIdx.x == 0) + { + uint64_t* s = GLM52_STAMPS(ctx) + ctx.stampRole * 4 + 2; + if (*s == 0) + { + *s = globalTimer(); + } + } +} + +// Bulk copy of [src, src + bytes) to shared offset dst on barrier bar (thread 0 only), one request (a request takes +// ~0.3 us to issue); with evictFirst under the L2 evict-first policy. +__device__ __forceinline__ void copyPieces( + uint32_t dst, uint8_t const* src, uint32_t bytes, uint32_t bar, bool evictFirst) +{ + constexpr uint32_t kPiece = 1u << 19; + uint64_t const policy = evictFirst ? policyEvictFirst() : 0; + for (uint32_t off = 0; off < bytes; off += kPiece) + { + uint32_t const n = bytes - off < kPiece ? bytes - off : kPiece; + if (evictFirst) + { + bulkCopyHint(dst + off, src + off, n, bar, policy); + } + else + { + bulkCopy(dst + off, src + off, n, bar); + } + } +} + +// ---------------------------------------------------------------------------------------------------------------- +// GEMM building blocks: M = weight rows (16 per tile), N = 8 rows, K in 64-wide chunks. +// +// A packed (BF16): per (tile, chunk) 2048 B = 4 pieces x 32 threads x 16 B; thread (g, c) holds row g's elements +// [64q + 16c, +16) in pieces 0-1 and row g + 8's in pieces 2-3. A packed (NVFP4): per (tile, chunk) 512 B = 32 threads +// x 16 B (row g's bytes [32q + 8c, +8), then row g + 8's), and 64 B of scales (thread: row g's and row g + 8's E4M3 +// scale of the 16-block 4q + c). B natural [8][K]: thread (g, c) reads row g's elements [64q + 16c, +16). +// In k step s of a chunk, MMA k slots {2c, 2c + 1} take elements 4s, 4s + 1 of the thread's 16, slots {2c + 8, 2c + 9} +// take 4s + 2, 4s + 3, in A and in B alike. +// ---------------------------------------------------------------------------------------------------------------- +template +__device__ __forceinline__ void loadB(uint4 (&b)[2], T const* gB, int ldb, int chunk) +{ + int const lane = threadIdx.x & 31; + T const* src = gB + static_cast(lane >> 2) * ldb + chunk * 64 + 16 * (lane & 3); + b[0] = ldcg128(src); + b[1] = ldcg128(src + 8); +} + +template +__device__ __forceinline__ void loadBSmem(uint4 (&b)[2], T const* sB, int ldb, int chunk) +{ + int const lane = threadIdx.x & 31; + T const* src = sB + (lane >> 2) * ldb + chunk * 64 + 16 * (lane & 3); + b[0] = *reinterpret_cast(src); + b[1] = *reinterpret_cast(src + 8); +} + +// The B fragments of chunks [chunk0, chunk0 + kN) of a 16-bit [8][ld] activation in the arena, polled. +template +__device__ __forceinline__ void pollB( + Ctx const& ctx, uint4 (&b)[kN][2], uint8_t const* gB, int ld, int chunk0, int code) +{ + int const lane = threadIdx.x & 31; + uint8_t const* src = gB + ((lane >> 2) * ld + chunk0 * 64 + 16 * (lane & 3)) * 2; +#pragma unroll + for (int i = 0; i < kN; ++i) + { + b[i][0] = ldPoll128(src + 128 * i); + b[i][1] = ldPoll128(src + 128 * i + 16); + } + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int i = 0; i < kN; ++i) + { + ok &= full4(b[i][0]) & full4(b[i][1]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, code, chunk0, 0); +#pragma unroll + for (int i = 0; i < kN; ++i) + { + if (!full4(b[i][0])) + { + b[i][0] = ldPoll128(src + 128 * i); + } + if (!full4(b[i][1])) + { + b[i][1] = ldPoll128(src + 128 * i + 16); + } + } + } +} + +// Byte offset of x' (token t, column col) in the arena's x (kFlagXFrag: fused-A's fragment order, else [8][6144]). +__device__ __forceinline__ size_t xArOff(int t, int col) +{ + if constexpr ((kSpecializedStaticFlags & kFlagXFrag) != 0) + { + return static_cast(col >> 6) * 1024 + ((col >> 3) & 1) * 512 + (4 * t + ((col >> 4) & 3)) * 16 + + 2 * (col & 7); + } + return (static_cast(t) * kHidden + col) * 2; +} + +// pollB for x' in fragment order (kFlagXFrag): chunk j's fragment b[.][h] of lane l at j 1024 + 512 h + 16 l. +template +__device__ __forceinline__ void pollBFrag(Ctx const& ctx, uint4 (&b)[kN][2], uint8_t const* gB, int chunk0, int code) +{ + int const lane = threadIdx.x & 31; + uint8_t const* src = gB + chunk0 * 1024 + 16 * lane; +#pragma unroll + for (int i = 0; i < kN; ++i) + { + b[i][0] = ldPoll128(src + 1024 * i); + b[i][1] = ldPoll128(src + 1024 * i + 512); + } + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int i = 0; i < kN; ++i) + { + ok &= full4(b[i][0]) & full4(b[i][1]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, code, chunk0, 0); +#pragma unroll + for (int i = 0; i < kN; ++i) + { + if (!full4(b[i][0])) + { + b[i][0] = ldPoll128(src + 1024 * i); + } + if (!full4(b[i][1])) + { + b[i][1] = ldPoll128(src + 1024 * i + 512); + } + } + } +} + +// acc += A(tile, chunk) x B(chunk), BF16. +__device__ __forceinline__ void mmaChunkBf16(float (&acc)[4], uint32_t aBlock, uint4 const (&b)[2]) +{ + int const lane = threadIdx.x & 31; + uint32_t const a = aBlock + lane * 16; + uint4 const p0 = lds128(a); + uint4 const p1 = lds128(a + 512); + uint4 const p2 = lds128(a + 1024); + uint4 const p3 = lds128(a + 1536); + mmaBf16(acc, p0.x, p2.x, p0.y, p2.y, b[0].x, b[0].y); + mmaBf16(acc, p0.z, p2.z, p0.w, p2.w, b[0].z, b[0].w); + mmaBf16(acc, p1.x, p3.x, p1.y, p3.y, b[1].x, b[1].y); + mmaBf16(acc, p1.z, p3.z, p1.w, p3.w, b[1].z, b[1].w); +} + +// Sum of the warps' accumulators of kTiles tiles in warp order, as out[row][token] (FP32) in the reduction area. +// Rows: 16 * tile + (g, g + 8); tokens 2c, 2c + 1. +template +__device__ __forceinline__ void reduceWarps(Ctx const& ctx, float const (&acc)[kTiles][4], float* out) +{ + float* red = reinterpret_cast(ctx.smem + kSmemRed); + int const warp = threadIdx.x >> 5; + int const lane = threadIdx.x & 31; +#pragma unroll + for (int t = 0; t < kTiles; ++t) + { + *reinterpret_cast(red + ((warp * kTiles + t) * 32 + lane) * 4) + = make_float4(acc[t][0], acc[t][1], acc[t][2], acc[t][3]); + } + __syncthreads(); + for (int i = threadIdx.x; i < kTiles * 32 * 4; i += kThreads) + { + int const t = i / 128; + int const l = (i / 4) % 32; + int const r = i % 4; + float s = 0.f; +#pragma unroll + for (int w = 0; w < kWarps; ++w) + { + s += red[((w * kTiles + t) * 32 + l) * 4 + r]; + } + int const row = 16 * t + (l >> 2) + (r >= 2 ? 8 : 0); + int const tok = 2 * (l & 3) + (r & 1); + out[row * kTok + tok] = s; + } + __syncthreads(); +} + +// Threads 0-63: rows [16 tile, +16) of out[row][token] as BF16 pairs into the [8][ld] activation at dst (one word +// each: token t, rows 2 pr and 2 pr + 1), times scale[t] if given. +__device__ __forceinline__ void storeTileBf16( + float const* out, uint8_t* dst, int ld, int row0, float const* scale = nullptr) +{ + if (threadIdx.x < 64) + { + int const tok = threadIdx.x / 8; + int const pr = threadIdx.x % 8; + float const sc = scale != nullptr ? scale[tok] : 1.f; + uint32_t const w = packBf16(out[(2 * pr) * kTok + tok] * sc, out[(2 * pr + 1) * kTok + tok] * sc); + st32(dst + (tok * ld + row0 + 2 * pr) * 2, lamport(w)); + } +} + +// ---------------------------------------------------------------------------------------------------------------- +// Unit weights: where they come from and how many bytes (static units only). +// ---------------------------------------------------------------------------------------------------------------- +__device__ __forceinline__ void unitWeights( + Ctx const& ctx, int stage, int unit, uint8_t const*& src0, uint32_t& n0, uint8_t const*& src1, uint32_t& n1) +{ + src0 = nullptr; + src1 = nullptr; + n0 = 0; + n1 = 0; + switch (stage) + { + case kStFusedA: + src0 = ptr(ctx, kPWFa) + static_cast(unit) * kFaChunks * kBf16Block; + n0 = kFaChunks * kBf16Block; + break; + case kStQb: + src0 = ptr(ctx, kPWQb) + static_cast(unit) * kQbUnitTiles * kQbChunks * kBf16Block; + n0 = kQbUnitTiles * kQbChunks * kBf16Block; + break; + case kStOproj: + src0 = ptr(ctx, kPWO) + static_cast(unit) * 2 * kOChunks * kBf16Block; + n0 = 2 * kOChunks * kBf16Block; + break; + case kStRouterGemv: + { + int const et = unit / 4; + int const kq = unit % 4; + src0 = ptr(ctx, kPWR) + (static_cast(et) * kRChunks + 24 * kq) * kBf16Block; + n0 = 24 * kBf16Block; + break; + } + case kStIdxQ: + src0 = ptr(ctx, kPIdxWq) + static_cast(unit) * 2 * kIdxQChunks * kBf16Block; + n0 = 2 * kIdxQChunks * kBf16Block; + break; + case kStOprojLg: + // o_proj's two tiles, then the logit tile's O part (chunks 0-31; the residual chunks' fragments come from + // global memory). + src0 = ptr(ctx, kPWO) + static_cast(unit) * 2 * kOChunks * kBf16Block; + n0 = 2 * kOChunks * kBf16Block; + src1 = ptr(ctx, kPWLg) + static_cast(unit - kLgOprojFirst) * kLgChunks * kBf16Block; + n1 = kOChunks * kBf16Block; + break; + case kStSharedGu: + { + int const ft = unit / 4; + int const kq = unit % 4; + src0 = ptr(ctx, kPWGu) + (static_cast(ft) * kRChunks + 24 * kq) * kBf16Block; + src1 = ptr(ctx, kPWGu) + (static_cast(8 + ft) * kRChunks + 24 * kq) * kBf16Block; + n0 = 24 * kBf16Block; + n1 = 24 * kBf16Block; + break; + } + default: break; + } +} + +// Thread 0 at entry (kFlagL2Deferred, leaders): the indexer Q weights of the CTA's IdxQ unit, whose copy waits for +// fused-A's slot, into L2 behind the first requests. +__device__ __noinline__ void prefetchDeferred(Ctx const& ctx, int const* sRoles) +{ +#pragma unroll 1 + for (int i = 0; i < kMaxRoles && sRoles[i] != 0; ++i) + { + if ((sRoles[i] >> 16) == kStIdxQ) + { + uint8_t const* src0; + uint8_t const* src1; + uint32_t n0, n1; + unitWeights(ctx, kStIdxQ, sRoles[i] & 0xffff, src0, n0, src1, n1); + prefetchBulkL2(src0, n0); + } + } +} + +#if GLM52_SETUP_MODE == 0 +__device__ __noinline__ void issueUnitWeights(Ctx const& ctx, int role, int smemOff, uint32_t bar) +{ + // Thread 0 only; every thread's earlier shared-memory reads are complete (the caller synchronized). + uint8_t const* src0; + uint8_t const* src1; + uint32_t n0, n1; + unitWeights(ctx, role >> 16, role & 0xffff, src0, n0, src1, n1); + if (n0 + n1 == 0) + { + return; + } + fenceProxyAsyncSmem(); + mbarExpectTx(bar, n0 + n1); + bool const evictFirst = (GLM52_FLAGS(ctx) & kFlagEvictFirst) != 0; + copyPieces(ctx.smemBase + smemOff, src0, n0, bar, evictFirst); + if (n1) + { + copyPieces(ctx.smemBase + smemOff + n0, src1, n1, bar, evictFirst); + } +} +#else +__device__ __noinline__ void issueUnitWeights(Ctx const& ctx, int j, uint32_t bar) +{ + // Thread 0 after the original caller's join; the per-layer table is immutable. + int const* roles = reinterpret_cast(ctx.smem + kSmemSmall + kSmallRoles); + int64_t const* w = reinterpret_cast(ctx.smem + kSmemSmall + kSmallWeights) + 3 * j; + uint8_t const* src0 = reinterpret_cast(w[0]); + uint8_t const* src1 = reinterpret_cast(w[1]); + uint32_t const n0 = static_cast(w[2]); + uint32_t const n1 = static_cast(static_cast(w[2]) >> 32); + if (n0 + n1 == 0) + return; + uint32_t const dst = ctx.smemBase + static_cast(roles[kMaxRoles + j]); + fenceProxyAsyncSmem(); + mbarExpectTx(bar, n0 + n1); + bool const evictFirst = (GLM52_FLAGS(ctx) & kFlagEvictFirst) != 0; + copyPieces(dst, src0, n0, bar, evictFirst); + if (n1) + copyPieces(dst + n0, src1, n1, bar, evictFirst); +} + +// Filled once, before capture, using the unchanged address/length function. +__global__ void glm52WeightTableKernel(int64_t const* params, int64_t* out) +{ + int const c = static_cast(blockIdx.x); + int const j = static_cast(threadIdx.x); + Ctx ctx{}; + ctx.p = params; + int const role = reinterpret_cast(params + kPNumParams)[c * kMaxRoles + j]; + uint8_t const* src0 = nullptr; + uint8_t const* src1 = nullptr; + uint32_t n0 = 0, n1 = 0; + if (role != 0) + unitWeights(ctx, role >> 16, role & 0xffff, src0, n0, src1, n1); + int64_t* o = out + (static_cast(c) * kMaxRoles + j) * 3; + o[0] = reinterpret_cast(src0); + o[1] = reinterpret_cast(src1); + o[2] = static_cast(n0) | (static_cast(n1) << 32); +} +#endif + +// Thread 0, inside unit `blocker` once its weight slot is free: the weights of its stage-`stage` units waiting for that +// slot go out now, and those units are marked issued for the main loop. +__device__ __noinline__ void issueBlockedEarly(Ctx const& ctx, int blocker, int stage) +{ + int* sRoles = reinterpret_cast(ctx.smem + kSmemSmall + kSmallRoles); +#pragma unroll 1 + for (int j = blocker + 1; j < kMaxRoles && sRoles[j] != 0; ++j) + { + if ((sRoles[j] >> 16) == stage && sRoles[2 * kMaxRoles + j] == blocker) + { +#if GLM52_SETUP_MODE == 0 + issueUnitWeights(ctx, sRoles[j], sRoles[kMaxRoles + j], ctx.smemBase + kSmemSmall + kSmallBars + 8 * j); +#else + issueUnitWeights(ctx, j, ctx.smemBase + kSmemSmall + kSmallBars + 8 * j); +#endif + sRoles[2 * kMaxRoles + j] = -3; + } + } +} + +#include "glmMegakernelStages.cuh" + +// ---------------------------------------------------------------------------------------------------------------- +// Kernel +// ---------------------------------------------------------------------------------------------------------------- +// Runs one unit of the CTA's list. +__device__ __forceinline__ void runUnit(Ctx const& c, int stage, int unit, uint32_t off, uint32_t bar, + __nv_bfloat16 const* x, __nv_bfloat16 const* residual, __nv_bfloat16* xOut, __nv_bfloat16* resOut, int const* topk, + int const* blockTable, int const* kvLen) +{ + switch (stage) + { + case kStFusedA: stageFusedA(c, unit, off, bar, x); break; + case kStQb: stageQb(c, unit, off, bar, kvLen); break; + case kStKvAppend: stageKvAppend(c, kvLen, blockTable); break; + case kStIdxK: stageIdxK(c, kvLen); break; + case kStIdxQ: stageIdxQ(c, unit, off, bar, kvLen); break; + case kStIdxSel: stageIdxSel(c, off, kvLen); break; + case kStAttn: stageAttn(c, unit, off, kvLen, blockTable, topk); break; + case kStMerge: stageMerge(c, unit); break; + case kStOproj: stageOproj(c, unit, off, bar); break; + case kStClear: stageClear(c, unit); break; + case kStH1: stageHop(c, unit, false, residual, xOut, resOut); break; + case kStRouterGemv: + if constexpr (kRouterLg != 2) + { + stageRouterGemv(c, unit, off, bar); + } + break; + case kStOprojLg: + if constexpr (kRouterLg != 0) + { + stageOprojLg(c, unit, off, bar, residual); + } + break; + case kStSharedGu: stageSharedGu(c, unit, off, bar); break; + case kStMoe: stageMoe(c, (GLM52_FLAGS(c) & kFlagOsInMoe) ? unit - 1 : -1, off); break; + case kStOwnerShared: + if (!(GLM52_FLAGS(c) & kFlagOsInMoe)) + { + stageOwnerShared(c, unit); + } + break; + case kStOwner: stageOwner(c, unit); break; + case kStH2: stageHop(c, unit, true, residual, xOut, resOut); break; + default: break; + } +} + +__global__ void __launch_bounds__(kThreads, 1) glm52PlainLayerKernel(int64_t const* __restrict__ params, + __nv_bfloat16 const* __restrict__ x, __nv_bfloat16 const* __restrict__ residual, __nv_bfloat16* __restrict__ xOut, + __nv_bfloat16* __restrict__ resOut, int const* __restrict__ topk, int const* __restrict__ blockTable, + int const* __restrict__ kvLen) +{ + extern __shared__ __align__(1024) uint8_t smemRaw[]; +#if GLM52_SETUP_MODE == 1 + // kFlagFirstEarly: the first unit's weights (when the setup would request them) before anything else. + bool firstIssued = false; + if constexpr ((kSpecializedStaticFlags & kFlagFirstEarly) != 0) + { + if (threadIdx.x == 0) + { + int const* tables = reinterpret_cast(params + kPNumParams); + int64_t const* wt + = params + kPNumParams + 3 * kCtas * kMaxRoles / 2 + static_cast(blockIdx.x) * 3 * kMaxRoles; + int const issue0 = __ldg(tables + (2 * kCtas + static_cast(blockIdx.x)) * kMaxRoles); + int const off0 = __ldg(tables + (kCtas + static_cast(blockIdx.x)) * kMaxRoles); + int64_t const w0 = __ldg(wt); + int64_t const w1 = __ldg(wt + 1); + int64_t const w2 = __ldg(wt + 2); + uint32_t const n0 = static_cast(w2); + uint32_t const n1 = static_cast(static_cast(w2) >> 32); + if (issue0 == -1 && n0 + n1 != 0) + { + uint32_t const sb = smemU32(smemRaw); + uint32_t const bar0 = sb + kSmemSmall + kSmallBars; + mbarInit(bar0, 1); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + mbarExpectTx(bar0, n0 + n1); + bool const evictFirst = (kSpecializedStaticFlags & kFlagEvictFirst) != 0; + uint32_t const dst = sb + static_cast(off0); + copyPieces(dst, reinterpret_cast(w0), n0, bar0, evictFirst); + if (n1) + { + copyPieces(dst + n0, reinterpret_cast(w1), n1, bar0, evictFirst); + } + firstIssued = true; + } + } + } +#endif + // The parameter block into shared memory: every later read of it is a shared load (L1 is a few KiB next to the + // units' shared memory, so global reads of it went to L2 again after each poll or barrier). + int64_t* sParams = reinterpret_cast(smemRaw + kSmemSmall + kSmallParams); + int* sRoles = reinterpret_cast(smemRaw + kSmemSmall + kSmallRoles); + if (threadIdx.x < kPNumParams) + { + sParams[threadIdx.x] = params[threadIdx.x]; + } + // The CTA's rows of the role tables, appended to the parameter block (units, shared offsets, issue points), in the + // same round trip. + if (threadIdx.x < 3 * kMaxRoles) + { + int const* tables = reinterpret_cast(params + kPNumParams); + int const table = threadIdx.x / kMaxRoles; + sRoles[threadIdx.x] + = tables[(table * kCtas + static_cast(blockIdx.x)) * kMaxRoles + threadIdx.x % kMaxRoles]; + } +#if GLM52_SETUP_MODE == 1 + if (threadIdx.x >= 64 && threadIdx.x < 64 + 3 * kMaxRoles) + { + int64_t const* table = params + kPNumParams + 3 * kCtas * kMaxRoles / 2; + reinterpret_cast(smemRaw + kSmemSmall + kSmallWeights)[threadIdx.x - 64] + = table[static_cast(blockIdx.x) * 3 * kMaxRoles + threadIdx.x - 64]; + } +#endif + __syncthreads(); + params = sParams; + // The context in shared memory: thread 0 fills it (its reads in the units are shared loads, never evicted). + Ctx& ctx = *reinterpret_cast(smemRaw + kSmemSmall + kSmallCtx); + if (threadIdx.x == 0) + { + ctx.p = params; + ctx.smem = smemRaw; + ctx.smemBase = smemU32(smemRaw); + ctx.parity = static_cast(params[kPCopy]); + ctx.counters = reinterpret_cast(params[kPCounters]) + ctx.parity * kNumCounters; + ctx.err = reinterpret_cast(params[kPErr]); + ctx.timeoutNs = static_cast(params[kPTimeoutNs]); + ctx.rank = static_cast(params[kPRank]); + ctx.stamps = params[kPStamps] != 0 && !(params[kPFlags] & kFlagNoStamps) + ? reinterpret_cast(params[kPStamps]) + static_cast(blockIdx.x) * kStampsPerCta + : nullptr; + ctx.stampRole = 0; + ctx.flags = params[kPFlags]; + ctx.numCopies = static_cast(params[kPNumCopies]); + ctx.leader = static_cast(params[kPLeader]); + ctx.arena = reinterpret_cast(params[kPArena]) + static_cast(ctx.parity) * kArenaBytes; + } + __syncthreads(); + int const tid = threadIdx.x; + uint64_t const entry = globalTimer(); + // The weights first: thread 0 initializes the barriers and requests the weights of the units that need no free + // slot right away (a CTA that starts late, on an SM the previous layer released last, has them in flight ~1.5 us + // sooner than after the rest of the setup); warp 1 takes the MoE stage's TMEM meanwhile. The TMEM is taken before + // the dependents are released: a later kernel's CTA sharing this SM then waits for it instead of holding TMEM this + // CTA needs while it waits for this grid (released at the MoE stage's end). +#if GLM52_SETUP_MODE == 0 + if (tid == 0) + { +#pragma unroll 1 + for (int i = 0; i < kNumBars; ++i) + { + mbarInit(ctx.smemBase + kSmemSmall + kSmallBars + 8 * i, 1); + } + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); +#pragma unroll 1 + for (int i = 0; i < kMaxRoles && sRoles[i] != 0; ++i) + { + if (sRoles[2 * kMaxRoles + i] == -1) + { +#if GLM52_SETUP_MODE == 0 + issueUnitWeights(ctx, sRoles[i], sRoles[kMaxRoles + i], ctx.smemBase + kSmemSmall + kSmallBars + 8 * i); +#else + issueUnitWeights(ctx, i, ctx.smemBase + kSmemSmall + kSmallBars + 8 * i); +#endif + } + } + if (ctx.leader && (GLM52_FLAGS(ctx) & kFlagL2Deferred)) + { + prefetchDeferred(ctx, sRoles); + } + } +#else + uint64_t landIssued = 0; + if (tid < 32) + { + static_assert(kNumBars <= 32, "one lane per barrier"); + if (tid < kNumBars && !(tid == 0 && firstIssued)) + mbarInit(ctx.smemBase + kSmemSmall + kSmallBars + 8 * tid, 1); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); + __syncwarp(); + if (tid == 0) + { +#pragma unroll 1 + for (int i = 0; i < kMaxRoles && sRoles[i] != 0; ++i) + { + if (sRoles[2 * kMaxRoles + i] == -1 && !(i == 0 && firstIssued)) + issueUnitWeights(ctx, i, ctx.smemBase + kSmemSmall + kSmallBars + 8 * i); + } + if constexpr ((kSpecializedStaticFlags & kFlagLandProbe) != 0) + { + landIssued = globalTimer(); + } + if (ctx.leader && (GLM52_FLAGS(ctx) & kFlagL2Deferred)) + prefetchDeferred(ctx, sRoles); + } + } +#endif + else if (tid >= 32 && tid < 64) + { + tmemAlloc(smemU32(smemRaw + kSmemSmall + kSmallScalars + 4 * 120), kTmemCols); + } + // The stamps (one word per thread; the CTA's synchronization below orders them before the units' stamps). + if (GLM52_STAMPS(ctx) != nullptr && tid >= 64) + { + for (int i = tid - 64; i < kStampsPerCta; i += kThreads - 64) + { + GLM52_STAMPS(ctx)[i] = i == kMaxRoles * 4 ? entry : 0ull; + } + } + // The SWIZZLE_64B operands need 512-byte aligned shared addresses (the MoE ring and h start at offsets of 512 B). + if (params[kPHopBytes] != kHopBytes || params[kPMagic] != kMagic || (ctx.smemBase & 1023u) != 0 + || params[kPMaxRoles] != kMaxRoles) + { + __trap(); + } + reinterpret_cast(smemRaw + kSmemSmall + kSmallMoe)[tid] = 0; + tcFenceBefore(); + __syncthreads(); + tcFenceAfter(); + int nRoles = 0; +#pragma unroll 1 + while (nRoles < kMaxRoles && sRoles[nRoles] != 0) + { + ++nRoles; + } + // The next launch's counters (last used by launch n + 1 - copies <= n - 2, finished before this launch began). + { + uint32_t* next + = reinterpret_cast(params[kPCounters]) + ((ctx.parity + 1) % ctx.numCopies) * kNumCounters; + for (int i = static_cast(blockIdx.x) * kThreads + tid; i < kNumCounters; i += kCtas * kThreads) + { + next[i] = 0; + } + } + // The previous kernel's outputs (x, residual, the step inputs, the leader's top-k) are read after the + // grid-dependency wait; with kFlagEarlyX a CTA whose first unit is fused-A runs it before the wait, its x from the + // arena. + bool const early = (GLM52_FLAGS(ctx) & kFlagEarlyX) && nRoles > 0 && (sRoles[0] >> 16) == kStFusedA; + auto dependencyWait = [&]() + { + asm volatile("griddepcontrol.wait;" ::: "memory"); + asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); + if (GLM52_STAMPS(ctx) != nullptr && tid == 0) + { + GLM52_STAMPS(ctx)[kMaxRoles * 4 + 1] = globalTimer(); + } + if (early && (GLM52_FLAGS(ctx) & kFlagCheckX) && blockIdx.x == 0) + { + uint8_t const* axb = arena(ctx, kArX); + uint32_t const* gx = reinterpret_cast(x); + for (int i = tid; i < kTok * kHidden / 2; i += kThreads) + { + uint32_t const* const aw + = reinterpret_cast(axb + xArOff(2 * i / kHidden, 2 * i % kHidden)); + if constexpr ((kSpecializedStaticFlags & kFlagXPrime) != 0) + { + // x' x 1 / rms against x: the same math, rounded at other points (about 2^-7 relative apart). + float const* q = arena(ctx, kArXSsq) + 4 * (2 * i / kHidden); + float const r = rsqrtf(((((0.f + __ldcg(q)) + __ldcg(q + 1)) + __ldcg(q + 2)) + __ldcg(q + 3)) + / static_cast(kHidden) + + kEps); + uint32_t const a = __ldcg(aw); + uint32_t const g = __ldcg(gx + i); + if (fabsf(bf16Lo(a) * r - bf16Lo(g)) > 0.015625f * fabsf(bf16Lo(g)) + 1e-30f + || fabsf(bf16Hi(a) * r - bf16Hi(g)) > 0.015625f * fabsf(bf16Hi(g)) + 1e-30f) + { + fail(ctx, 98, i, static_cast(a)); + } + } + else if (__ldcg(aw) != lamport(__ldcg(gx + i))) + { + fail(ctx, 98, i, static_cast(__ldcg(aw))); + } + } + } + }; + if constexpr ((kSpecializedStaticFlags & kFlagLandProbe) != 0) + { + if (GLM52_STAMPS(ctx) != nullptr && tid == 0) + { + GLM52_STAMPS(ctx)[kMaxRoles * 4 + 4 + 128] = landIssued; + } + } + if (!early) + { + dependencyWait(); + } + + for (int i = 0; i < nRoles; ++i) + { + if (early && i == 1) + { + dependencyWait(); + } + int const role = sRoles[i]; + int const stage = role >> 16; + int const unit = role & 0xffff; + uint32_t const off = static_cast(sRoles[kMaxRoles + i]); + uint32_t const bar = ctx.smemBase + kSmemSmall + kSmallBars + 8 * i; + if (GLM52_STAMPS(ctx) != nullptr && tid == 0) + { + GLM52_STAMPS(ctx)[i * 4] = globalTimer(); + } + if (tid == 0) + { + ctx.stampRole = i; // read by thread 0 only (its stamps) + } + if (GLM52_STAMPS(ctx) != nullptr && tid == 0) + { + // The CTA's progress in host memory (stamped runs): after a trap, the unit each CTA was in. + reinterpret_cast(ctx.err)[16 + 4 * kCtas + blockIdx.x] = (stage << 16) | (i << 8) | 1; + } + runUnit(ctx, stage, unit, off, bar, x, residual, xOut, resOut, topk, blockTable, kvLen); + // Every thread fences its generic shared accesses of the unit for the async proxy before the join (the + // documented pattern): later units' weight copies and gathers, issued by one thread after it, land on them. + fenceProxyAsyncSmem(); + __syncthreads(); + if (GLM52_STAMPS(ctx) != nullptr && tid == 0) + { + GLM52_STAMPS(ctx)[i * 4 + 3] = globalTimer(); + } + if (stage == kStMoe && (GLM52_FLAGS(ctx) & kFlagHoldExit) && tid == 0) + { + atomicAdd(ctx.counters + kCMoeDone, 1u); + } + if (tid == 0) + { + bool const attnNext = i + 1 < nRoles && (sRoles[i + 1] >> 16) == kStAttn; +#pragma unroll 1 + for (int j = i + 1; j < nRoles; ++j) + { + if (sRoles[2 * kMaxRoles + j] == i) + { + if (attnNext && (sRoles[j] >> 16) == kStOproj) + { + sRoles[2 * kMaxRoles + j] = -4; // requested by the attention unit (after its gathers / Q) + } + else + { +#if GLM52_SETUP_MODE == 0 + issueUnitWeights( + ctx, sRoles[j], sRoles[kMaxRoles + j], ctx.smemBase + kSmemSmall + kSmallBars + 8 * j); +#else + issueUnitWeights(ctx, j, ctx.smemBase + kSmemSmall + kSmallBars + 8 * j); +#endif + } + } + } + } + } + + if (early && nRoles == 1) + { + dependencyWait(); + } + __syncthreads(); + if (GLM52_FLAGS(ctx) & kFlagHoldExit) + { + waitCounter(ctx, kCMoeDone, kCtas, 150); + } + if (GLM52_STAMPS(ctx) != nullptr && tid == 0) + { + GLM52_STAMPS(ctx)[kMaxRoles * 4 + 2] = globalTimer(); + reinterpret_cast(ctx.err)[16 + 4 * kCtas + blockIdx.x] = 0xffff; + } + // The layout pad: code placement alone moves the slice by up to ~7 us (lessons/megakernel/code-placement-moves-the- + // slice.md), so the stage functions' offset in the 128-byte fetch line is a build parameter (k never-run stores). + constexpr int kLayoutPad = static_cast((kSpecializedStaticFlags >> 43) & 7); + if constexpr (kLayoutPad > 0) + { + if (ctx.p[kPMagic] == 0) + { + uint32_t const a = smemU32(smemRaw); + asm volatile("st.shared.u32 [%0], %1;" ::"r"(a), "r"(tid) : "memory"); + if constexpr (kLayoutPad > 1) + { + asm volatile("st.shared.u32 [%0+16], %1;" ::"r"(a), "r"(tid) : "memory"); + } + if constexpr (kLayoutPad > 2) + { + asm volatile("st.shared.u32 [%0+32], %1;" ::"r"(a), "r"(tid) : "memory"); + } + if constexpr (kLayoutPad > 3) + { + asm volatile("st.shared.u32 [%0+48], %1;" ::"r"(a), "r"(tid) : "memory"); + } + if constexpr (kLayoutPad > 4) + { + asm volatile("st.shared.u32 [%0+64], %1;" ::"r"(a), "r"(tid) : "memory"); + } + if constexpr (kLayoutPad > 5) + { + asm volatile("st.shared.u32 [%0+80], %1;" ::"r"(a), "r"(tid) : "memory"); + } + if constexpr (kLayoutPad > 6) + { + asm volatile("st.shared.u32 [%0+96], %1;" ::"r"(a), "r"(tid) : "memory"); + } + } + } +} + +#else + +__global__ void glm52PlainLayerKernel(int64_t const*, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16*, + __nv_bfloat16*, int const*, int const*, int const*) +{ +} + +#if GLM52_SETUP_MODE == 1 +__global__ void glm52WeightTableKernel(int64_t const*, int64_t*) {} +#endif + +#endif + +} // namespace glm52 + +// ---------------------------------------------------------------------------------------------------------------- +// Native launch and layout queries for the Torch binding. +// ---------------------------------------------------------------------------------------------------------------- + +int glm52PlainLayerSmemBytes() +{ + return glm52::kSmemBytes; +} + +int glm52PlainLayerNumParams() +{ + return glm52::kPNumParams; +} + +int64_t glm52PlainLayerMagic() +{ + return glm52::kMagic; +} + +int glm52PlainLayerArenaBytes() +{ + return glm52::kArenaBytes; +} + +int glm52PlainLayerClearUnits() +{ + return glm52::kClearUnits; +} + +int glm52PlainLayerNumCounters() +{ + return glm52::kNumCounters; +} + +int glm52PlainLayerCandCap() +{ + return glm52::kCandCap; +} + +int glm52PlainLayerMaxKeys() +{ + return glm52::kIdxMaxKeys; +} + +// GLM52_ROUTER_LG of this build (the host's role tables: 1 adds the logit units, 2 also drops the router GEMV units). +int glm52PlainLayerRouterLg() +{ + return glm52::kRouterLg; +} + +// GLM52_ROUTER_LG's multicast workspace per launch copy, after the top-k exchange's copies. +int glm52PlainLayerLgBytes() +{ + return glm52::kLgBytes; +} + +// The MoE cut's class weight i (W, OS, SG, H1; glm52::kFc2ListsCut), for the host's check of slot 0's FC1 partials. +int glm52PlainLayerCutWeight(int i) +{ + return i >= 0 && i < 4 ? glm52::kFc2ListsCut[i] : 0; +} + +// The 128-byte CUtensorMap of a leader's indexer K cache (rows of 128 B from pool, numRows of them) with SWIZZLE_128B +// boxes of 64 rows (one block's keys), written to out (host memory); a CUresult. +int glm52PlainLayerIdxTmap(void* out, void* pool, unsigned long long numRows) +{ + cuuint64_t dims[2] = {128, static_cast(numRows)}; + cuuint64_t strides[1] = {128}; + cuuint32_t box[2] = {128, 64}; + cuuint32_t elem[2] = {1, 1}; + CUresult const r = cuTensorMapEncodeTiled(static_cast(out), CU_TENSOR_MAP_DATA_TYPE_UINT8, 2, pool, + dims, strides, box, elem, CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, + CU_TENSOR_MAP_L2_PROMOTION_L2_128B, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + return static_cast(r); +} + +#if GLM52_SETUP_MODE == 1 +int glm52PlainLayerWeightTable(void const* params, void* out, void* stream) +{ + glm52::glm52WeightTableKernel<<(stream)>>>( + static_cast(params), static_cast(out)); + return static_cast(cudaGetLastError()); +} +#endif + +int glm52PlainLayerInit() +{ +#if defined(PLACEHOLDER_KERNELS) + return static_cast(cudaErrorNotSupported); +#else + int dev = 0; + int optInBytes = 0; + cudaGetDevice(&dev); + cudaDeviceGetAttribute(&optInBytes, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); + if (glm52::kSmemBytes > optInBytes) + { + // Larger than the portable opt-in: the oversized shared-memory mode (the max-dynamic-size attribute is + // ignored in it). + return static_cast(cudaFuncSetAttribute(glm52::glm52PlainLayerKernel, cudaFuncAttributeSharedMemoryMode, + cudaSharedMemoryModePreferOversizedSharedMemory)); + } + cudaError_t e = cudaFuncSetAttribute( + glm52::glm52PlainLayerKernel, cudaFuncAttributeMaxDynamicSharedMemorySize, glm52::kSmemBytes); + if (e != cudaSuccess) + { + return static_cast(e); + } + e = cudaFuncSetAttribute( + glm52::glm52PlainLayerKernel, cudaFuncAttributePreferredSharedMemoryCarveout, cudaSharedmemCarveoutMaxShared); + return static_cast(e); +#endif +} + +int glm52PlainLayerLaunch(void const* params, void const* x, void const* residual, void* xOut, void* resOut, + void const* topk, void const* blockTable, void const* kvLen, void* stream, int pdl) +{ +#if defined(PLACEHOLDER_KERNELS) + return static_cast(cudaErrorNotSupported); +#else + cudaLaunchConfig_t config{}; + config.gridDim = dim3(glm52::kCtas); + config.blockDim = dim3(glm52::kThreads); + config.dynamicSmemBytes = glm52::kSmemBytes; + config.stream = static_cast(stream); + cudaLaunchAttribute attrs[2]{}; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = pdl ? 1 : 0; + attrs[1].id = cudaLaunchAttributePreferredSharedMemoryCarveout; + attrs[1].val.sharedMemCarveout = 100; + config.attrs = attrs; + config.numAttrs = 2; + cudaError_t const e = cudaLaunchKernelEx(&config, glm52::glm52PlainLayerKernel, static_cast(params), + static_cast<__nv_bfloat16 const*>(x), static_cast<__nv_bfloat16 const*>(residual), + static_cast<__nv_bfloat16*>(xOut), static_cast<__nv_bfloat16*>(resOut), static_cast(topk), + static_cast(blockTable), static_cast(kvLen)); + return static_cast(e); +#endif +} + +} // namespace kernels::glm_moe_dsa::megakernel + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.h b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.h new file mode 100644 index 000000000000..78fcd90270b6 --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.h @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "tensorrt_llm/modeling_v2/common/namespace.h" +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels::glm_moe_dsa::megakernel +{ +int glm52PlainLayerSmemBytes(); +int glm52PlainLayerNumParams(); +int64_t glm52PlainLayerMagic(); +int glm52PlainLayerArenaBytes(); +int glm52PlainLayerClearUnits(); +int glm52PlainLayerNumCounters(); +int glm52PlainLayerCandCap(); +int glm52PlainLayerMaxKeys(); +int glm52PlainLayerRouterLg(); +int glm52PlainLayerLgBytes(); +int glm52PlainLayerIdxTmap(void* out, void* pool, unsigned long long numRows); +int glm52PlainLayerWeightTable(void const* params, void* out, void* stream); +int glm52PlainLayerInit(); +int glm52PlainLayerLaunch(void const* params, void const* x, void const* residual, void* xOut, void* resOut, + void const* topk, void const* blockTable, void const* kvLen, void* stream, int pdl); +} // namespace kernels::glm_moe_dsa::megakernel + +TRTLLM_MV2_NAMESPACE_END diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp new file mode 100644 index 000000000000..9f27d4a9345d --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.h" +#include "tensorrt_llm/modeling_v2/common/torchLibrary.h" +#include "tensorrt_llm/thop/thUtils.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace mega = TRTLLM_MV2_NS::kernels::glm_moe_dsa::megakernel; + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace torch_ext +{ +namespace +{ +constexpr int64_t kRows = 8; +constexpr int64_t kHidden = 6144; +constexpr int64_t kTopK = 2048; +constexpr int64_t kCtas = 212; +constexpr int64_t kMaxRoles = 16; +constexpr int64_t kWeightWords = kCtas * kMaxRoles * 3; + +void checkMegakernelDevice(at::Tensor const& tensor) +{ + CHECK_TH_CUDA(tensor); + auto const* props = at::cuda::getDeviceProperties(tensor.get_device()); + TORCH_CHECK(props->major == 10 && props->minor == 7 && props->multiProcessorCount == kCtas, + "glm_megakernel requires SM107 with 212 SMs"); +} + +void checkPackedParams(at::Tensor const& params) +{ + CHECK_TH_CUDA(params); + TORCH_CHECK(params.scalar_type() == at::kLong && params.dim() == 1 && params.is_contiguous() + && params.numel() == mega::glm52PlainLayerNumParams() + kWeightWords / 2 + kWeightWords, + "glm_megakernel: params must contain the int64 parameter block, role tables, and weight table"); +} + +void rejectMegakernelFunctionalization(c10::OperatorHandle const&, torch::jit::Stack*) +{ + TORCH_CHECK( + false, "glm_megakernel uses packed device pointers and multicast state; functionalization is unsupported"); +} +} // namespace + +std::vector glmMegakernelInfo() +{ + return {mega::glm52PlainLayerMagic(), mega::glm52PlainLayerNumParams(), mega::glm52PlainLayerArenaBytes(), + mega::glm52PlainLayerClearUnits(), mega::glm52PlainLayerSmemBytes(), mega::glm52PlainLayerNumCounters(), + mega::glm52PlainLayerCandCap(), mega::glm52PlainLayerMaxKeys(), mega::glm52PlainLayerRouterLg(), + mega::glm52PlainLayerLgBytes()}; +} + +at::Tensor glmMegakernelIndexerTmap(at::Tensor const& pool, int64_t rows) +{ + checkMegakernelDevice(pool); + int64_t const availableBytes = pool.storage().nbytes() - pool.storage_offset() * pool.element_size(); + TORCH_CHECK(rows > 0 && rows <= availableBytes / 128, + "glm_megakernel_indexer_tmap: pool storage must contain rows of 128 bytes"); + c10::cuda::CUDAGuard const deviceGuard(pool.device()); + auto out = at::empty({128}, at::TensorOptions().dtype(at::kByte).device(at::kCPU)); + alignas(64) CUtensorMap tensorMap{}; + auto const result = mega::glm52PlainLayerIdxTmap(&tensorMap, pool.data_ptr(), rows); + TORCH_CHECK(result == CUDA_SUCCESS, "glm_megakernel_indexer_tmap: CUDA driver error ", result); + std::copy_n(reinterpret_cast(&tensorMap), sizeof(tensorMap), out.data_ptr()); + return out; +} + +void glmMegakernelWeightTable(at::Tensor& params, int64_t offset) +{ + checkMegakernelDevice(params); + checkPackedParams(params); + TORCH_CHECK(offset == mega::glm52PlainLayerNumParams() + kWeightWords / 2, + "glm_megakernel_weight_table: invalid weight table offset"); + c10::cuda::CUDAGuard const deviceGuard(params.device()); + C10_CUDA_CHECK(static_cast(mega::glm52PlainLayerInit())); + C10_CUDA_CHECK(static_cast(mega::glm52PlainLayerWeightTable( + params.data_ptr(), params.data_ptr() + offset, at::cuda::getCurrentCUDAStream(params.get_device())))); +} + +std::tuple glmMegakernel(at::Tensor const& params, at::Tensor const& x, + at::Tensor const& residual, at::Tensor& topk, at::Tensor const& blockTable, at::Tensor const& kvLens, + std::vector const& state, std::vector const& kvCache, bool pdl) +{ + checkMegakernelDevice(x); + checkPackedParams(params); + TORCH_CHECK(x.scalar_type() == at::kBFloat16 && x.dim() == 2 && x.size(0) == kRows && x.size(1) == kHidden + && x.is_contiguous(), + "glm_megakernel: x must be contiguous BF16 [8, 6144]"); + TORCH_CHECK(residual.device() == x.device() && residual.scalar_type() == x.scalar_type() + && residual.sizes() == x.sizes() && residual.is_contiguous() && params.device() == x.device(), + "glm_megakernel: residual and params must match the input device and residual shape"); + TORCH_CHECK(topk.device() == x.device() && topk.scalar_type() == at::kInt && topk.dim() == 2 + && topk.size(0) == kRows && topk.size(1) == kTopK && topk.is_contiguous(), + "glm_megakernel: topk must be contiguous int32 [8, 2048]"); + TORCH_CHECK(blockTable.device() == x.device() && blockTable.scalar_type() == at::kInt && blockTable.dim() == 1 + && blockTable.numel() > 0 && blockTable.is_contiguous(), + "glm_megakernel: block_table must be a contiguous int32 row"); + TORCH_CHECK(kvLens.device() == x.device() && kvLens.scalar_type() == at::kInt && kvLens.numel() == 1 + && kvLens.is_contiguous(), + "glm_megakernel: kv_lens must contain one int32 request length"); + TORCH_CHECK(!state.empty() && !kvCache.empty(), "glm_megakernel: mutable state and KV pools must be supplied"); + for (auto const& tensor : state) + { + TORCH_CHECK(tensor.device() == x.device() || (tensor.is_cpu() && tensor.is_pinned()), + "glm_megakernel: state must be on the input device or pinned host memory"); + } + for (auto const& tensor : kvCache) + { + TORCH_CHECK(tensor.device() == x.device(), "glm_megakernel: KV pools must be on the input device"); + } + c10::cuda::CUDAGuard const deviceGuard(x.device()); + auto xOut = at::empty_like(x); + auto resOut = at::empty_like(residual); + C10_CUDA_CHECK(static_cast(mega::glm52PlainLayerLaunch(params.data_ptr(), x.data_ptr(), + residual.data_ptr(), xOut.data_ptr(), resOut.data_ptr(), topk.data_ptr(), blockTable.data_ptr(), + kvLens.data_ptr(), at::cuda::getCurrentCUDAStream(x.get_device()), pdl))); + return {xOut, resOut}; +} +} // namespace torch_ext + +TRTLLM_MV2_NAMESPACE_END + +TRTLLM_MV2_LIBRARY_FRAGMENT(m) +{ + m.def("glm_megakernel_info() -> int[]", &TRTLLM_MV2_NS::torch_ext::glmMegakernelInfo); + m.def("glm_megakernel_indexer_tmap(Tensor pool, int rows) -> Tensor"); + m.def("glm_megakernel_weight_table(Tensor(a!) params, int offset) -> ()"); + m.def( + "glm_megakernel(Tensor params, Tensor x, Tensor residual, Tensor(a!) topk, Tensor block_table, " + "Tensor kv_lens, Tensor(b!)[] state, Tensor(c!)[] kv_cache, bool pdl) -> (Tensor, Tensor)"); +} + +TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) +{ + m.impl("glm_megakernel_indexer_tmap", &TRTLLM_MV2_NS::torch_ext::glmMegakernelIndexerTmap); + m.impl("glm_megakernel_weight_table", &TRTLLM_MV2_NS::torch_ext::glmMegakernelWeightTable); + m.impl("glm_megakernel", &TRTLLM_MV2_NS::torch_ext::glmMegakernel); +} + +TRTLLM_MV2_LIBRARY_IMPL(Functionalize, m) +{ + m.impl("glm_megakernel", + torch::CppFunction::makeFromBoxedFunction<&TRTLLM_MV2_NS::torch_ext::rejectMegakernelFunctionalization>()); + m.impl("glm_megakernel_weight_table", + torch::CppFunction::makeFromBoxedFunction<&TRTLLM_MV2_NS::torch_ext::rejectMegakernelFunctionalization>()); +} diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelStages.cuh b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelStages.cuh new file mode 100644 index 000000000000..5241846ef3cc --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelStages.cuh @@ -0,0 +1,8183 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +// Device stages included inside the glm52 namespace by glmMegakernel.cu. + +// ---------------------------------------------------------------------------------------------------------------- +// Stages +// ---------------------------------------------------------------------------------------------------------------- + +// kFlagXPrime (fused-A's warp 0): 1 / rms of token (lane % 8) from the previous launch's four H2 partials of it +// (kArXSsq), v their first load; one exit test for the whole warp. +__device__ __forceinline__ float xRcp(Ctx const& ctx, uint4 v, int code) +{ + float const* p = arena(ctx, kArXSsq) + 4 * (threadIdx.x & 7); + uint64_t t0 = 0; + while (!warpAll(full4(v))) + { + spinCheck(ctx, t0, code, 0, 0); + if (!full4(v)) + { + v = ldPoll128(p); + } + } + float const S + = (((0.f + __uint_as_float(v.x)) + __uint_as_float(v.y)) + __uint_as_float(v.z)) + __uint_as_float(v.w); + return rsqrtf(S / static_cast(kHidden) + kEps); +} + +// fused-A: rows [16 tile, +16) of [q_a | c_kv | k_pe] = x W_A^T, K = 6144. +__device__ __noinline__ void stageFusedA(Ctx const& ctx, int tile, uint32_t wOff, uint32_t bar, __nv_bfloat16 const* x) +{ + int const warp = threadIdx.x >> 5; + if constexpr ((kSpecializedStaticFlags & kFlagLandProbe) != 0) + { + if (GLM52_STAMPS(ctx) != nullptr && threadIdx.x == 0) + { + while (!mbarTryWait(bar, 0)) + { + } + subStamp(ctx, 129); + } + } + constexpr int kPer = kFaChunks / kWarps; + uint4 b[kPer][2]; + if (GLM52_FLAGS(ctx) & kFlagEarlyX) + { + // Before the dependency wait the previous launch may still be running: its clear units empty this arena copy's + // x (left non-empty from four launches back) before its H2 writes the new one, and the rows this unit stores + // (kArY, kArIdxKw), and nothing else orders those clears before this read and those stores. So first its clear + // count, as its H2 waits for it before publishing (its copy's counters were zeroed before it began and are not + // zeroed again until this launch is done); then the poll. + if (threadIdx.x == 0) + { + uint32_t const* c = reinterpret_cast(ctx.p[kPCounters]) + + ((ctx.parity + ctx.numCopies - 1) % ctx.numCopies) * kNumCounters + kCClear; + uint64_t t0 = 0; + uint32_t v; + while ((v = ldAcquire(c)) < static_cast(kCountedClear)) + { + spinCheck(ctx, t0, 99, kCClear, static_cast(v)); + } + } + __syncthreads(); + if constexpr ((kSpecializedStaticFlags & kFlagXFrag) != 0) + { + pollBFrag(ctx, b, arena(ctx, kArX), warp * kPer, 100); + } + else + { + pollB(ctx, b, arena(ctx, kArX), kHidden, warp * kPer, 100); + } + } + else + { +#pragma unroll + for (int i = 0; i < kPer; ++i) + { + loadB(b[i], x, kHidden, warp * kPer + i); + } + } + // kFlagXPrime: the arena x is x' = res' x gamma; warp 0 takes the tokens' 1 / rms (lane l: token l % 8) from the + // previous launch's H2 partials, first load now (they follow their x' by ~0.1 us), the rest after its MMAs. + bool const xPrime = (kSpecializedStaticFlags & kFlagXPrime) != 0 && (GLM52_FLAGS(ctx) & kFlagEarlyX) && warp == 0; + uint4 xSsq = make_uint4(kEmpty, kEmpty, kEmpty, kEmpty); + if (xPrime) + { + xSsq = ldPoll128(arena(ctx, kArXSsq) + 4 * (threadIdx.x & 7)); + } + float acc[1][4]; + bool const idxqEarly = ctx.leader && (GLM52_FLAGS(ctx) & kFlagIdxQEarly) && threadIdx.x == 0; + stampInput(ctx); + waitBar(ctx, bar, 0, 101); + if (idxqEarly) + { + prefetchDeferred(ctx, reinterpret_cast(ctx.smem + kSmemSmall + kSmallRoles)); + } + // Four interleaved accumulator chains (one chain of 48 dependent MMAs was the unit's critical path). + float part[4][4] = {}; +#pragma unroll + for (int i = 0; i < kPer; ++i) + { + mmaChunkBf16(part[i & 3], ctx.smemBase + wOff + (warp * kPer + i) * kBf16Block, b[i]); + } +#pragma unroll + for (int r = 0; r < 4; ++r) + { + acc[0][r] = (part[0][r] + part[1][r]) + (part[2][r] + part[3][r]); + } + float* sRcp = reinterpret_cast(ctx.smem + kSmemRed + 8704); + if (xPrime) + { + float const rcp = xRcp(ctx, xSsq, 148); + if (threadIdx.x < kTok) + { + sRcp[threadIdx.x] = rcp; + } + } + float* out = reinterpret_cast(ctx.smem + kSmemRed + 8192); + reduceWarps<1>(ctx, acc, out); + float const* scale + = (kSpecializedStaticFlags & kFlagXPrime) != 0 && (GLM52_FLAGS(ctx) & kFlagEarlyX) ? sRcp : nullptr; + if (idxqEarly) + { + // Every warp's MMAs read the weight slot before the reduction's barrier: the IdxQ unit it blocks gets it now. + issueBlockedEarly(ctx, 0, kStIdxQ); + } + // Tiles 164-173 (leader layers): the indexer's K and head-weight rows. + if (tile < kFaRows / 16) + { + storeTileBf16(out, arena(ctx, kArY), kFaRows, 16 * tile, scale); + } + else + { + storeTileBf16(out, arena(ctx, kArIdxKw), kIdxKwCols, 16 * tile - kFaRows, scale); + } +} + +// q_b': rows [48 u, +48) of Q = rstd_q * (q_a W'^T), RoPE on the rope columns, E4M3. +__device__ __noinline__ void stageQb(Ctx const& ctx, int unit, uint32_t wOff, uint32_t bar, int const* kvLenPtr) +{ + int const warp = threadIdx.x >> 5; + int const lane = threadIdx.x & 31; + if constexpr ((kSpecializedStaticFlags & kFlagLandProbe) != 0) + { + if (GLM52_STAMPS(ctx) != nullptr && threadIdx.x == 0) + { + while (!mbarTryWait(bar, 0)) + { + } + subStamp(ctx, 129); + } + } + float* sc = reinterpret_cast(ctx.smem + kSmemSmall + kSmallScalars); + // The RoPE factors of this thread's epilogue rows first: their loads overlap the q_a poll. + float4 rope = make_float4(1.f, 0.f, 1.f, 0.f); + { + int const col = (48 * unit + 4 * (threadIdx.x % 12)) % 576; + if (threadIdx.x < 12 * kTok && col >= 512) + { + int const pos = ldPinnedS32(kvLenPtr) - kTok + static_cast(threadIdx.x) / 12; + rope = ldPinnedF4(ptr(ctx, kPRope) + static_cast(pos) * 128 + (col - 512)); + } + } + constexpr int kPer = kQbChunks / kWarps; + uint4 b[kPer][2]; + pollB(ctx, b, arena(ctx, kArY), kFaRows, warp * kPer, 102); + stampInput(ctx); + subStamp(ctx, 0); + // Sum of squares of q_a per row: thread (g, c) holds 16 of row g's elements per chunk. + float ssq = 0.f; +#pragma unroll + for (int i = 0; i < kPer; ++i) + { +#pragma unroll + for (int h = 0; h < 2; ++h) + { + uint32_t const w4[4] = {b[i][h].x, b[i][h].y, b[i][h].z, b[i][h].w}; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + float const lo = bf16Lo(w4[j]); + float const hi = bf16Hi(w4[j]); + ssq = ssq + lo * lo + hi * hi; + } + } + } + ssq += __shfl_xor_sync(0xffffffffu, ssq, 1); + ssq += __shfl_xor_sync(0xffffffffu, ssq, 2); + if ((lane & 3) == 0) + { + sc[warp * kTok + (lane >> 2)] = ssq; + } + waitBar(ctx, bar, 0, 103); + subStamp(ctx, 1); + float acc[kQbUnitTiles][4]; +#pragma unroll + for (int t = 0; t < kQbUnitTiles; ++t) + { + acc[t][0] = acc[t][1] = acc[t][2] = acc[t][3] = 0.f; + } +#pragma unroll + for (int i = 0; i < kPer; ++i) + { +#pragma unroll + for (int t = 0; t < kQbUnitTiles; ++t) + { + mmaChunkBf16(acc[t], ctx.smemBase + wOff + (t * kQbChunks + warp * kPer + i) * kBf16Block, b[i]); + } + } + subStamp(ctx, 2); + float* out = reinterpret_cast(ctx.smem + kSmemRed + 12288); + reduceWarps(ctx, acc, out); + subStamp(ctx, 3); + if (threadIdx.x < kTok) + { + float s = 0.f; + for (int w = 0; w < kWarps; ++w) + { + s += sc[w * kTok + threadIdx.x]; + } + sc[64 + threadIdx.x] = rsqrtf(s / 2048.f + kEps); + } + __syncthreads(); + // Epilogue: thread (token, 4 rows): 8 x 12 = 96 threads, one word of 4 E4M3 values each. + if (threadIdx.x < 12 * kTok) + { + int const tok = threadIdx.x / 12; + int const q4 = threadIdx.x % 12; + int const row0 = 48 * unit + 4 * q4; + float const r = sc[64 + tok]; + float v[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + v[j] = out[(4 * q4 + j) * kTok + tok] * r; + } + if (row0 % 576 >= 512) + { + float const cs[4] = {rope.x, rope.y, rope.z, rope.w}; +#pragma unroll + for (int pp = 0; pp < 2; ++pp) + { + float const cosv = cs[2 * pp]; + float const sinv = cs[2 * pp + 1]; + float const n0 = cosv * v[2 * pp] - sinv * v[2 * pp + 1]; + float const n1 = cosv * v[2 * pp + 1] + sinv * v[2 * pp]; + v[2 * pp] = n0; + v[2 * pp + 1] = n1; + } + } + uint32_t const w = static_cast(toE4m3(v[0])) | (static_cast(toE4m3(v[1])) << 8) + | (static_cast(toE4m3(v[2])) << 16) | (static_cast(toE4m3(v[3])) << 24); + st32(arena(ctx, kArQ8) + tok * kQbRows + row0, lamport(w)); + } + subStamp(ctx, 4); +} + +// KV append: this step's 8 KV rows (kv_a RMSNorm, k_pe RoPE, E4M3) into the pool and into the arena (this step's +// attention reads them from there); warp t = row t. +__device__ __noinline__ void stageKvAppend(Ctx const& ctx, int const* kvLenPtr, int const* blockTable) +{ + int const warp = threadIdx.x >> 5; + int const lane = threadIdx.x & 31; + int const tok = warp; + uint8_t const* y = arena(ctx, kArY) + (tok * kFaRows + 2048) * 2; + // Step constants first (their loads overlap the poll of y): position, page, RoPE factors, the lane's gammas. + int const L = __ldg(kvLenPtr); + int const pos = L - kTok + tok; + int const page = __ldg(blockTable + (pos >> 6)); + float4 cs0 = make_float4(0.f, 0.f, 0.f, 0.f); + float4 cs1 = cs0; + if (lane < 8) + { + float4 const* cs + = reinterpret_cast(ptr(ctx, kPRope) + static_cast(pos) * 128) + + 2 * lane; + cs0 = __ldg(cs); + cs1 = __ldg(cs + 1); + } + __nv_bfloat16 const* gamma = ptr<__nv_bfloat16 const>(ctx, kPGammaKv); + uint4 const gm0 = __ldg(reinterpret_cast(gamma + 8 * lane)); + uint4 const gm1 = __ldg(reinterpret_cast(gamma + 8 * (lane + 32))); + // Lane l: latent vectors l and l + 32 (8 values each), then the rope vector l (lanes < 8; others re-read l). + uint4 v[3]; + { + uint8_t const* srcs[3] = {y + 16 * lane, y + 16 * (lane + 32), y + 1024 + 16 * (lane & 7)}; +#pragma unroll + for (int i = 0; i < 3; ++i) + { + v[i] = ldPoll128(srcs[i]); + } + uint64_t t0 = 0; + while (!warpAll(full4(v[0]) & full4(v[1]) & full4(v[2]))) + { + spinCheck(ctx, t0, 104, tok, 0); +#pragma unroll + for (int i = 0; i < 3; ++i) + { + if (!full4(v[i])) + { + v[i] = ldPoll128(srcs[i]); + } + } + } + } + stampInput(ctx); + float lat[16]; + { + uint32_t const w[8] = {v[0].x, v[0].y, v[0].z, v[0].w, v[1].x, v[1].y, v[1].z, v[1].w}; +#pragma unroll + for (int j = 0; j < 8; ++j) + { + lat[2 * j] = bf16Lo(w[j]); + lat[2 * j + 1] = bf16Hi(w[j]); + } + } + float ssq = 0.f; +#pragma unroll + for (int j = 0; j < 16; ++j) + { + ssq += lat[j] * lat[j]; + } +#pragma unroll + for (int off = 16; off > 0; off >>= 1) + { + ssq += __shfl_xor_sync(0xffffffffu, ssq, off); + } + float const rstd = rsqrtf(ssq / 512.f + kEps); + int64_t const row = static_cast(page) * ctx.p[kPStrideRows] + (pos & 63) + ctx.p[kPLayerRows]; + uint8_t* dst = ptr(ctx, kPKvBase) + row * kRowBytes; + uint8_t* dstA = arena(ctx, kArKv) + tok * kRowBytes; + float gms[16]; + { + uint32_t const gw[8] = {gm0.x, gm0.y, gm0.z, gm0.w, gm1.x, gm1.y, gm1.z, gm1.w}; +#pragma unroll + for (int j = 0; j < 8; ++j) + { + gms[2 * j] = bf16Lo(gw[j]); + gms[2 * j + 1] = bf16Hi(gw[j]); + } + } + uint32_t out[4]; +#pragma unroll + for (int q = 0; q < 4; ++q) + { + uint32_t w = 0; +#pragma unroll + for (int e = 0; e < 4; ++e) + { + int const j = 4 * q + e; + w |= static_cast(toE4m3(lat[j] * (rstd * (gms[j] * 1.f)))) << (8 * e); + } + out[q] = lamport(w); + } + st64(dst + 8 * lane, make_uint2(out[0], out[1])); + st64(dst + 8 * (lane + 32), make_uint2(out[2], out[3])); + st64(dstA + 8 * lane, make_uint2(out[0], out[1])); + st64(dstA + 8 * (lane + 32), make_uint2(out[2], out[3])); + if (lane < 8) + { + float k[8]; + uint32_t const w[4] = {v[2].x, v[2].y, v[2].z, v[2].w}; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + k[2 * j] = bf16Lo(w[j]); + k[2 * j + 1] = bf16Hi(w[j]); + } + float const cs[8] = {cs0.x, cs0.y, cs0.z, cs0.w, cs1.x, cs1.y, cs1.z, cs1.w}; + uint32_t ro[2] = {0, 0}; +#pragma unroll + for (int pp = 0; pp < 4; ++pp) + { + float const cosv = cs[2 * pp]; + float const sinv = cs[2 * pp + 1]; + uint32_t const r0 = toE4m3((cosv * k[2 * pp] - sinv * k[2 * pp + 1]) * 1.f); + uint32_t const r1 = toE4m3((cosv * k[2 * pp + 1] + sinv * k[2 * pp]) * 1.f); + ro[pp >> 1] |= (r0 | (r1 << 8)) << (16 * (pp & 1)); + } + uint2 const rw = make_uint2(lamport(ro[0]), lamport(ro[1])); + st64(dst + 512 + 8 * lane, rw); + st64(dstA + 512 + 8 * lane, rw); + } +} + +// ---------------------------------------------------------------------------------------------------------------- +// Leader layers: the DSA indexer (numerics as the stock fused prologue / Q tail: FP32 math on BF16-rounded +// intermediates, GPT-J RoPE on dims 0-63, UE8M0 E4M3 scales with the MUFU reciprocal) +// ---------------------------------------------------------------------------------------------------------------- +// This step's 8 indexer keys (warp t = row t, lane l = dims 4l .. 4l + 3): LayerNorm (eps 1e-6), RoPE, E4M3 with the +// row's scale; into the cache at the row's position and into the arena (the logits of the page holding it read it +// there: the page may have been read before this store). +__device__ __noinline__ void stageIdxK(Ctx const& ctx, int const* kvLenPtr) +{ + int const t = threadIdx.x >> 5; + int const lane = threadIdx.x & 31; + int const pos = __ldg(kvLenPtr) - kTok + t; + // Constants first (their loads overlap the poll): LayerNorm weight and bias, cos / sin of pairs 2l, 2l + 1. + float4 const lw = __ldg(reinterpret_cast(ptr(ctx, kPIdxKnW)) + lane); + float4 const lb = __ldg(reinterpret_cast(ptr(ctx, kPIdxKnB)) + lane); + float2 cs = make_float2(1.f, 1.f); + float2 sn = make_float2(0.f, 0.f); + if (lane < 16) + { + float const* row = ptr(ctx, kPIdxRope) + static_cast(pos) * 64; + cs = __ldg(reinterpret_cast(row) + lane); + sn = __ldg(reinterpret_cast(row + 32) + lane); + } + int const blk = __ldg(ptr(ctx, kPIdxBlocks) + (pos >> 6)); + uint8_t const* src = arena(ctx, kArIdxKw) + (t * kIdxKwCols + 4 * lane) * 2; + uint2 v = ldPoll64(src); + { + uint64_t t0 = 0; + while (!__all_sync(0xffffffffu, full2(v))) + { + spinCheck(ctx, t0, 150, t, 0); + if (!full2(v)) + { + v = ldPoll64(src); + } + } + } + stampInput(ctx); + float const x[4] = {bf16Lo(v.x), bf16Hi(v.x), bf16Lo(v.y), bf16Hi(v.y)}; + float sum = (x[0] + x[1]) + (x[2] + x[3]); +#pragma unroll + for (int off = 16; off > 0; off >>= 1) + { + sum += __shfl_xor_sync(0xffffffffu, sum, off); + } + float const mean = sum / static_cast(kIdxDim); + float kc[4]; + float var = 0.f; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + kc[i] = x[i] - mean; + var = fmaf(kc[i], kc[i], var); + } +#pragma unroll + for (int off = 16; off > 0; off >>= 1) + { + var += __shfl_xor_sync(0xffffffffu, var, off); + } + float const rstd = rsqrtf(var / static_cast(kIdxDim) + 1e-6f); + float const w[4] = {lw.x, lw.y, lw.z, lw.w}; + float const b[4] = {lb.x, lb.y, lb.z, lb.w}; + float k[4]; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + k[i] = bf16Round(fmaf(kc[i] * rstd, w[i], b[i])); + } + if (lane < 16) + { + // GPT-J pairs (4l, 4l + 1), (4l + 2, 4l + 3): x cos -/+ partner sin, as fma(x, cos, rn(+-partner sin)). + float const r0 = bf16Round(fmaf(k[0], cs.x, __fmul_rn(-k[1], sn.x))); + float const r1 = bf16Round(fmaf(k[1], cs.x, __fmul_rn(k[0], sn.x))); + float const r2 = bf16Round(fmaf(k[2], cs.y, __fmul_rn(-k[3], sn.y))); + float const r3 = bf16Round(fmaf(k[3], cs.y, __fmul_rn(k[2], sn.y))); + k[0] = r0; + k[1] = r1; + k[2] = r2; + k[3] = r3; + } + float amax = fmaxf(fmaxf(fabsf(k[0]), fabsf(k[1])), fmaxf(fabsf(k[2]), fabsf(k[3]))); +#pragma unroll + for (int off = 16; off > 0; off >>= 1) + { + amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, off)); + } + float const scale = ue8m0Scale(amax); + float const inv = rcpApproxFtz(scale); + uint32_t const word = static_cast(toE4m3(__fmul_rn(k[0], inv))) + | (static_cast(toE4m3(__fmul_rn(k[1], inv))) << 8) + | (static_cast(toE4m3(__fmul_rn(k[2], inv))) << 16) + | (static_cast(toE4m3(__fmul_rn(k[3], inv))) << 24); + uint8_t* page = ptr(ctx, kPIdxPool) + static_cast(blk) * ctx.p[kPIdxBlockStride]; + st32(page + (pos & 63) * kIdxDim + 4 * lane, word); + st32(arena(ctx, kArIdxKNew) + t * kIdxDim + 4 * lane, lamport(word)); + if (lane == 0) + { + st32(page + kIdxPageKeys * kIdxDim + (pos & 63) * 4, __float_as_uint(scale)); + st32(arena(ctx, kArIdxKNewS) + 4 * t, lamport(__float_as_uint(scale))); + } +} + +__device__ __noinline__ void selEpilogue( + uint32_t taddr, float const* sW, float* sLog, int kt, int hh, float ks, int tMin, float* dbg, int64_t dbgLd); + +// kFlagPageBalance: each IdxQ CTA holds kIdxBalDeficit pages fewer than each CTA without an IdxQ unit [kIdxQUnits, +// kCtas): the first kBalFree * kIdxBalDeficit pages round-robin over the latter, the rest round-robin over all CTAs. +// The CTAs without an IdxQ unit start IdxSel, and so their page loads, ~3.5 us earlier; with 7 pages more their logits +// end with the IdxQ CTAs' at 102k-900k keys (mk-t15 stamps: the earlier split, 14 pages on them and then 16 on each +// IdxQ CTA, left them 1.4 us late at 102k and the IdxQ CTAs 3.5 us late at 335k). Else CTA c holds the pages c, c + +// 212, ... +constexpr bool kPageBal = (kSpecializedStaticFlags & kFlagPageBalance) != 0; +constexpr int kBalFree = kCtas - kIdxQUnits; +static_assert( + kIdxBalDeficit + (kIdxMaxKeys / kIdxPageKeys - kBalFree * kIdxBalDeficit + kCtas - 1) / kCtas <= kIdxMaxPagesRs + && (kIdxMaxKeys / kIdxPageKeys + kCtas - 1) / kCtas <= kIdxMaxPagesRs, + "the indexer's longest context: every CTA's logits fit"); + +// kFlagPageMap2: the key pages water-filled over four CTA groups in the order of their IdxSel starts (CTAs 128-163 at +// ~1.7 us, 174-211 at ~5.1, 164-173 at ~5.7, the IdxQ CTAs 0-127 at ~6.3; mk-t09 stamps, leader-front.md P3): group g +// takes T - kMapS(g) pages per CTA for the largest T the pages fill, the rest go one each to the CTAs in group order. +// Pages are numbered group by group (CTA-interleaved within a group), then the rest. +constexpr bool kPageMap2 = (kSpecializedStaticFlags & kFlagPageMap2) != 0; + +// Group g's CTA count, first CTA, first CTA in group order, and pages fewer than group 0's (its IdxSel start's lead +// over group 0 / ~0.24 us per page). +__host__ __device__ constexpr int kMapN(int g) +{ + return g == 0 ? 36 : g == 1 ? 38 : g == 2 ? 10 : 128; +} + +__host__ __device__ constexpr int kMapC0(int g) +{ + return g == 0 ? 128 : g == 1 ? 174 : g == 2 ? 164 : 0; +} + +__host__ __device__ constexpr int kMapOrd(int g) +{ + return g == 0 ? 0 : g == 1 ? 36 : g == 2 ? 74 : 84; +} + +__host__ __device__ constexpr int kMapS(int g) +{ + return g == 0 ? 0 : g == 1 ? 2 : g == 2 ? 5 : 7; +} + +static_assert(kMapN(0) + kMapN(1) + kMapN(2) + kMapN(3) == kCtas && kMapC0(3) == 0 && kMapN(3) == kIdxQUnits, + "the page map's groups"); +static_assert((kIdxMaxKeys / kIdxPageKeys + 38 * 2 + 10 * 5 + 128 * 7) / kCtas + 1 <= kIdxMaxPagesRs, + "the page map at the indexer's longest context"); + +struct IdxPageMap +{ + int T; // group g holds max(0, T - kMapS(g)) pages per CTA + int B; // pages in the group blocks + int R; // the rest: one page each for the first R CTAs in group order +}; + +__device__ __forceinline__ int idxMapGroup(int cta) +{ + return cta < 128 ? 3 : cta < 164 ? 0 : cta < 174 ? 2 : 1; +} + +__device__ __forceinline__ IdxPageMap idxPageMap(int nPages) +{ + int g = 0; + int A = kMapN(0); + int S = kMapN(0) * kMapS(0); + int T = (nPages + S) / A; + while (g < 3 && T > kMapS(g + 1)) + { + ++g; + A += kMapN(g); + S += kMapN(g) * kMapS(g); + T = (nPages + S) / A; + } + int b = 0; +#pragma unroll + for (int i = 0; i < 4; ++i) + { + b += kMapN(i) * max(0, T - kMapS(i)); + } + return IdxPageMap{T, b, nPages - b}; +} + +// Group g's pages per CTA and its first page. +__device__ __forceinline__ int idxMapK(IdxPageMap const& m, int g) +{ + return max(0, m.T - kMapS(g)); +} + +__device__ __forceinline__ int idxMapBase(IdxPageMap const& m, int g) +{ + int b = 0; +#pragma unroll + for (int i = 0; i < 3; ++i) + { + b += i < g ? kMapN(i) * max(0, m.T - kMapS(i)) : 0; + } + return b; +} + +// The global page of the CTA's page j (kPageBal: a CTA without an IdxQ unit holds the pages (cta - kIdxQUnits) + +// kBalFree j for j < kIdxBalDeficit, then every CTA the pages kBalFree * kIdxBalDeficit + cta, + 212, ...). +__device__ __forceinline__ int idxPageOf(int cta, int j, int nPages) +{ + if constexpr (kPageMap2) + { + IdxPageMap const m = idxPageMap(nPages); + int const g = idxMapGroup(cta); + return j < idxMapK(m, g) ? idxMapBase(m, g) + j * kMapN(g) + (cta - kMapC0(g)) + : m.B + kMapOrd(g) + (cta - kMapC0(g)); + } + if constexpr (kPageBal) + { + if (cta >= kIdxQUnits && j < kIdxBalDeficit) + { + return (cta - kIdxQUnits) + kBalFree * j; + } + return kBalFree * kIdxBalDeficit + cta + kCtas * (j - (cta >= kIdxQUnits ? kIdxBalDeficit : 0)); + } + return cta + kCtas * j; +} + +// The CTA's index of global page p, or -1 when another CTA holds it. +__device__ __forceinline__ int idxPageSlot(int cta, int p, int nPages) +{ + if constexpr (kPageMap2) + { + IdxPageMap const m = idxPageMap(nPages); + int const g = idxMapGroup(cta); + if (p >= m.B) + { + return p - m.B == kMapOrd(g) + (cta - kMapC0(g)) ? idxMapK(m, g) : -1; + } + int const q = p - idxMapBase(m, g); + return q >= 0 && q < idxMapK(m, g) * kMapN(g) && q % kMapN(g) == cta - kMapC0(g) ? q / kMapN(g) : -1; + } + if constexpr (kPageBal) + { + int const F = kBalFree * kIdxBalDeficit; + if (p < F) + { + return cta >= kIdxQUnits && p % kBalFree == cta - kIdxQUnits ? p / kBalFree : -1; + } + return (p - F) % kCtas == cta ? (cta >= kIdxQUnits ? kIdxBalDeficit : 0) + (p - F) / kCtas : -1; + } + return p >= cta && (p - cta) % kCtas == 0 ? (p - cta) / kCtas : -1; +} + +// The CTA's IdxSel pages for L keys. +__device__ __forceinline__ int idxPagesOf(int cta, int nPages) +{ + if constexpr (kPageMap2) + { + IdxPageMap const m = idxPageMap(nPages); + int const g = idxMapGroup(cta); + return idxMapK(m, g) + (kMapOrd(g) + (cta - kMapC0(g)) < m.R ? 1 : 0); + } + if constexpr (kPageBal) + { + int const F = min(nPages, kBalFree * kIdxBalDeficit); + int const X = nPages - kBalFree * kIdxBalDeficit; // the round-robin pages + int const extra = cta < X ? (X - 1 - cta) / kCtas + 1 : 0; + if (cta >= kIdxQUnits) + { + int const c = cta - kIdxQUnits; + return (c < F ? (F - 1 - c) / kBalFree + 1 : 0) + extra; + } + return extra; + } + return cta < nPages ? (nPages - 1 - cta) / kCtas + 1 : 0; +} + +__device__ __forceinline__ int idxSelMyPages(int L) +{ + return idxPagesOf(blockIdx.x, (L + kIdxPageKeys - 1) / kIdxPageKeys); +} + +// The CTA's IdxSel key pages [first, first + count) (SWIZZLE_128B boxes of the pool's tensor map) and their scales into +// the IdxSel region at a0 (page first + j into slot slot0 + j), on bar (0: kBarIdxK): warp 0, lane j takes page +// first + j, its block index from sBlk (the CTA's table) or loaded (in parallel with the others'). Read once per +// leader: L2 evict-first, as the weights (with the default policy these ~19.5 MB and the KV rows pushed what the next +// layers reuse out of L2: -46 us per slice, mk-v75a). +__device__ __noinline__ void idxSelIssuePages(Ctx const& ctx, uint32_t a0, int first, int count, int nPages, + int slot0 = 0, uint32_t bar = 0, int const* sBlk = nullptr) +{ + int const lane = threadIdx.x & 31; + int const cta = blockIdx.x; + if (threadIdx.x >= 32 || count <= 0) + { + return; + } + uint32_t const barK = bar != 0 ? bar : ctx.smemBase + kSmemSmall + kSmallBars + kBarIdxK * 8; + int const blk = lane >= count ? 0 + : sBlk != nullptr ? sBlk[first + lane] + : __ldg(ptr(ctx, kPIdxBlocks) + idxPageOf(cta, first + lane, nPages)); + if (lane == 0) + { + fenceProxyAsyncSmem(); + mbarExpectTx(barK, static_cast(count) * (8192u + 256u)); + } + __syncwarp(); + if (lane < count) + { + int64_t const stride = ctx.p[kPIdxBlockStride]; + int const y = static_cast(static_cast(blk) * (stride / 128)); + uint8_t const* scales + = ptr(ctx, kPIdxPool) + static_cast(blk) * stride + kIdxPageKeys * kIdxDim; + uint64_t const policy = policyEvictFirst(); + tmaLoad2dHint(a0 + kSelK + (slot0 + lane) * 8192, ptr(ctx, kPIdxTmap), 0, y, barK, policy); + bulkCopyHint(a0 + kSelKs + (slot0 + lane) * 256, scales, 256, barK, policy); + } +} + +// Leader: the context fits in the top-k (row t has L - 7 + t <= L keys), so the top-k is every valid key whatever the +// logits: no indexer Q, no key pages, logits or selection; the attention takes keys [0, valid) itself (the stock top-k +// as a set). The new keys still go into the indexer cache (IdxK). +__device__ __forceinline__ bool idxAllKeys(int L) +{ + return L <= kTopK; +} + +// Indexer Q rows [32 u, +32) (a quarter of head u / 4): rstd(q_a) (q_a Wf^T) rounded to BF16 (the stock wq_b output; +// Wf = wq_b with the q_a norm weight folded in), RoPE on the head's dims 0-63 (GPT-J pairs are adjacent rows), BF16; +// then the head's E4M3 scale from its four quarters' |max| (the head's other units run alongside), the quarter in +// E4M3, and (quarter 0) the head weight. +__device__ __noinline__ void stageIdxQ(Ctx const& ctx, int unit, uint32_t wOff, uint32_t bar, int const* kvLenPtr) +{ + if (idxAllKeys(__ldg(kvLenPtr))) + { + // The unit's weights were requested: they land before a later unit takes the slot. + waitBar(ctx, bar, 0, 152); + return; + } + int const warp = threadIdx.x >> 5; + int const lane = threadIdx.x & 31; + float* sc = reinterpret_cast(ctx.smem + kSmemSmall + kSmallScalars); + // Epilogue thread (row t = tid / 16, row pair p = tid % 16): its cos / sin first (their loads overlap the poll). + int const et = static_cast(threadIdx.x) / 16; + int const ep = static_cast(threadIdx.x) % 16; + int const row0 = kIdxQUnitRows * unit + 2 * ep; + int const dim0 = row0 % kIdxDim; + float cosv = 1.f; + float sinv = 0.f; + if (threadIdx.x < 16 * kTok && dim0 < 64) + { + float const* cs = ptr(ctx, kPIdxRope) + static_cast(__ldg(kvLenPtr) - kTok + et) * 64; + cosv = __ldg(cs + dim0 / 2); + sinv = __ldg(cs + 32 + dim0 / 2); + } + constexpr int kPer = kIdxQChunks / kWarps; + uint4 b[kPer][2]; + pollB(ctx, b, arena(ctx, kArY), kFaRows, warp * kPer, 151); + stampInput(ctx); + // Quarter 0: the head's weight word now (fused-A wrote it with q_a), its round trip under this unit's work. + int const head = kIdxQUnitRows * unit / kIdxDim; + int const quarter = (kIdxQUnitRows * unit % kIdxDim) / kIdxQUnitRows; + uint8_t const* const kwWord = arena(ctx, kArIdxKw) + (et * kIdxKwCols + 128 + (head & ~1)) * 2; + uint32_t const kwEarly = threadIdx.x < 16 * kTok && quarter == 0 && ep == 0 ? ldPoll32(kwWord) : kEmpty; + // Sum of squares of q_a per row: thread (g, c) holds 16 of row g's elements per chunk. + float ssq = 0.f; +#pragma unroll + for (int i = 0; i < kPer; ++i) + { +#pragma unroll + for (int h = 0; h < 2; ++h) + { + uint32_t const w4[4] = {b[i][h].x, b[i][h].y, b[i][h].z, b[i][h].w}; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + float const lo = bf16Lo(w4[j]); + float const hi = bf16Hi(w4[j]); + ssq = ssq + lo * lo + hi * hi; + } + } + } + ssq += __shfl_xor_sync(0xffffffffu, ssq, 1); + ssq += __shfl_xor_sync(0xffffffffu, ssq, 2); + if ((lane & 3) == 0) + { + sc[warp * kTok + (lane >> 2)] = ssq; + } + // The leader's logits epilogue once on a dummy tile while this unit's weights land (it runs cold in the CTA's + // IdxSel unit otherwise): TMEM columns no MMA has written, the results into the reduction area. + tcFenceAfter(); +#if GLM52_FLAGS_MODE != 4 && GLM52_FLAGS_MODE != 5 + selEpilogue(*reinterpret_cast(ctx.smem + kSmemSmall + kSmallScalars + 4 * 120) + 256u + + (static_cast(32 * (warp & 3)) << 16) + 128u * static_cast(warp >> 2), + reinterpret_cast(ctx.smem + kSmemRed + 4096), reinterpret_cast(ctx.smem + kSmemRed), + 32 * (warp & 3) + lane, warp >> 2, 0.f, 0, nullptr, 0); +#endif + __syncthreads(); // finish dummy scratch accesses before real reduction writes + waitBar(ctx, bar, 0, 152); + float acc[2][4] = {{0.f, 0.f, 0.f, 0.f}, {0.f, 0.f, 0.f, 0.f}}; +#pragma unroll + for (int i = 0; i < kPer; ++i) + { +#pragma unroll + for (int tl = 0; tl < 2; ++tl) + { + mmaChunkBf16(acc[tl], ctx.smemBase + wOff + (tl * kIdxQChunks + warp * kPer + i) * kBf16Block, b[i]); + } + } + float* out = reinterpret_cast(ctx.smem + kSmemRed + 12288); + reduceWarps<2>(ctx, acc, out); + subStamp(ctx, 69); + if (threadIdx.x < kTok) + { + float s = 0.f; + for (int w = 0; w < kWarps; ++w) + { + s += sc[w * kTok + threadIdx.x]; + } + sc[64 + threadIdx.x] = rsqrtf(s / 2048.f + kEps); + } + __syncthreads(); + if (threadIdx.x < 16 * kTok) + { + float const r = sc[64 + et]; + float x0 = bf16Round(out[(2 * ep) * kTok + et] * r); + float x1 = bf16Round(out[(2 * ep + 1) * kTok + et] * r); + if (dim0 < 64) + { + float const y0 = bf16Round(fmaf(x0, cosv, __fmul_rn(-x1, sinv))); + float const y1 = bf16Round(fmaf(x1, cosv, __fmul_rn(x0, sinv))); + x0 = y0; + x1 = y1; + } + if ((GLM52_FLAGS(ctx) & kFlagIdxAllRows) == 0) + { + // The row shard: BF16 Q and the head's raw weight; the IdxSel CTAs take the head's E4M3 scale from its 128 + // values (the four quarters' |max| exchange, one L2 round trip, is off this unit's path). + st32(arena(ctx, kArIdxQb) + ((et * kIdxHeads + head) * kIdxDim + dim0) * 2, + lamport(packBf16(x0, x1))); + if (quarter == 0 && ep == 0) + { + uint32_t const wr = kwEarly != kEmpty ? kwEarly : poll1(ctx, kwWord, 154); + st32(arena(ctx, kArIdxW) + et * kIdxHeads + head, + lamport(__float_as_uint((head & 1) ? bf16Hi(wr) : bf16Lo(wr)))); + } + subStamp(ctx, 72); + return; + } + // The quarter's |max| of row et: its 16 lanes (lanes 0-15 hold row 2w, 16-31 row 2w + 1). + float m = fmaxf(fabsf(x0), fabsf(x1)); +#pragma unroll + for (int off = 8; off > 0; off >>= 1) + { + m = fmaxf(m, __shfl_xor_sync(0xffffffffu, m, off)); + } + float* amaxRow = arena(ctx, kArIdxAmax) + (et * kIdxHeads + head) * 4; + if (ep == 0) + { + st32(amaxRow + quarter, lamport(__float_as_uint(m))); + } + subStamp(ctx, 70); + uint4 const am = poll4(ctx, amaxRow, 153); + subStamp(ctx, 71); + float const scale = ue8m0Scale(fmaxf( + fmaxf(__uint_as_float(am.x), __uint_as_float(am.y)), fmaxf(__uint_as_float(am.z), __uint_as_float(am.w)))); + float const inv = rcpApproxFtz(scale); + // Lane pairs (ep even, ep + 1) make the 32-bit word of dims dim0 .. dim0 + 3. + uint32_t const two = toE4m3x2(__fmul_rn(x0, inv), __fmul_rn(x1, inv)); + uint32_t const next = __shfl_down_sync(0xffffffffu, two, 1); + if ((ep & 1) == 0) + { + st32(arena(ctx, kArIdxQ8) + (et * kIdxHeads + head) * kIdxDim + dim0, lamport(two | (next << 16))); + } + if (quarter == 0 && ep == 0) + { + uint32_t const wr = kwEarly != kEmpty ? kwEarly : poll1(ctx, kwWord, 154); + float const wraw = (head & 1) ? bf16Hi(wr) : bf16Lo(wr); + st32(arena(ctx, kArIdxW) + et * kIdxHeads + head, + lamport(__float_as_uint(__fmul_rn(__fmul_rn(wraw, scale), kIdxWeightScale)))); + } + } + subStamp(ctx, 72); +} + +// Block-wide, over a 2048-bin histogram: the largest bin whose count from the top reaches need (1 <= need <= the +// total), the count strictly above it and its own count, to every thread (thread j sums bins [8 j, +8); sSel[2..5) +// and sSel[8..16) are scratch). +__device__ __noinline__ void blockCrossBin( + uint32_t const* hist, uint32_t need, uint32_t* sSel, int& bin, uint32_t& above, uint32_t& inBin) +{ + int const tid = threadIdx.x; + int const lane = tid & 31; + int const warp = tid >> 5; + uint4 const v0 = reinterpret_cast(hist)[2 * tid]; + uint4 const v1 = reinterpret_cast(hist)[2 * tid + 1]; + uint32_t const c[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + uint32_t s = 0; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + s += c[i]; + } + // Inclusive suffix sum over the warp's lanes [lane, 32) (the higher lanes hold the larger bins), then the warps'. + uint32_t incl = s; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) + { + uint32_t const v = __shfl_down_sync(0xffffffffu, incl, off); + incl += lane + off < 32 ? v : 0u; + } + if (lane == 0) + { + sSel[8 + warp] = incl; + } + __syncthreads(); + uint32_t higher = 0; +#pragma unroll + for (int w = 0; w < kWarps; ++w) + { + higher += w > warp ? sSel[8 + w] : 0u; + } + uint32_t const aboveMe = higher + incl - s; + if (aboveMe < need && aboveMe + s >= need) + { + uint32_t acc = aboveMe; +#pragma unroll + for (int i = 7; i >= 0; --i) + { + if (acc + c[i] >= need) + { + sSel[2] = static_cast(8 * tid + i); + sSel[3] = acc; + sSel[4] = c[i]; + break; + } + acc += c[i]; + } + } + __syncthreads(); + bin = static_cast(sSel[2]); + above = sSel[3]; + inBin = sSel[4]; +} + +// Block-wide exclusive scan of two per-thread counts in thread order (scratch: 16 words at shared address scratch; one +// barrier inside). +__device__ __forceinline__ void blockScan2(uint32_t a, uint32_t b, uint32_t scratch, uint32_t& baseA, uint32_t& baseB) +{ + int const lane = threadIdx.x & 31; + int const warp = threadIdx.x >> 5; + uint32_t ia = a; + uint32_t ib = b; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) + { + uint32_t const va = __shfl_up_sync(0xffffffffu, ia, off); + uint32_t const vb = __shfl_up_sync(0xffffffffu, ib, off); + ia += lane >= off ? va : 0u; + ib += lane >= off ? vb : 0u; + } + if (lane == 31) + { + sts32(scratch + 4 * warp, ia); + sts32(scratch + 4 * (kWarps + warp), ib); + } + __syncthreads(); + uint32_t pa = 0; + uint32_t pb = 0; +#pragma unroll + for (int w = 0; w < kWarps; ++w) + { + uint32_t const sa = lds32(scratch + 4 * w); + uint32_t const sb = lds32(scratch + 4 * (kWarps + w)); + pa += w < warp ? sa : 0u; + pb += w < warp ? sb : 0u; + } + baseA = pa + ia - a; + baseB = pb + ib - b; +} + +// Check mode: 128 accumulator columns of the warp's TMEM lane quarter from taddr into dd (this lane's row). +__device__ __noinline__ void dumpTileD(uint32_t taddr, float* dd) +{ +#pragma unroll 1 + for (int c = 0; c < 4; ++c) + { + uint32_t r[32]; + tmemLd32(taddr + 32u * static_cast(c), r); +#pragma unroll + for (int j = 0; j < 32; ++j) + { + dd[32 * c + j] = __uint_as_float(r[j]); + } + } +} + +// IdxSel epilogue of one tile for this thread (key kt of the tile, rows 4 hh .. + 3, accumulator columns [taddr, +// +128)): logit(t) = ks * sum_h w(t, h) relu(D[t][h]) into sLog[t][kt], -inf for rows t < tMin (the key is past the +// row's last); dbg != nullptr: also into dbg[t * dbgLd] (check mode). Out of line: the same code warms up on a dummy +// tile. +__device__ __noinline__ void selEpilogue( + uint32_t taddr, float const* sW, float* sLog, int kt, int hh, float ks, int tMin, float* dbg, int64_t dbgLd) +{ + uint32_t r[128]; + tmemLd128(taddr, r); +#pragma unroll + for (int tt = 0; tt < 4; ++tt) + { + int const t = 4 * hh + tt; + // Four independent chains over the heads (h mod 4), then their sum. + float4 const* w4 = reinterpret_cast(sW + 32 * t); + float c0 = 0.f; + float c1 = 0.f; + float c2 = 0.f; + float c3 = 0.f; +#pragma unroll + for (int h4 = 0; h4 < 8; ++h4) + { + float4 const wv = w4[h4]; + c0 = fmaf(wv.x, fmaxf(__uint_as_float(r[32 * tt + 4 * h4]), 0.f), c0); + c1 = fmaf(wv.y, fmaxf(__uint_as_float(r[32 * tt + 4 * h4 + 1]), 0.f), c1); + c2 = fmaf(wv.z, fmaxf(__uint_as_float(r[32 * tt + 4 * h4 + 2]), 0.f), c2); + c3 = fmaf(wv.w, fmaxf(__uint_as_float(r[32 * tt + 4 * h4 + 3]), 0.f), c3); + } + float const acc = (c0 + c1) + (c2 + c3); + float const lgt = t >= tMin ? acc * ks : -INFINITY; + sLog[t * 128 + kt] = lgt; + if (dbg != nullptr) + { + dbg[t * dbgLd] = lgt; + } + } +} + +// IdxSel epilogue of one tile under the row shard (warps 0-3; key kt of the tile, the rank's row: accumulator columns +// [taddr, +32) = its 32 heads): logit = ks * sum_h w(h) relu(D[h]) into sLog[kt], -inf when the key is past the row's +// last (valid false); dbg != nullptr: also into *dbg (check mode). Returns the logit. +__device__ __noinline__ float selEpilogue1( + uint32_t taddr, float const* sW, float* sLog, int kt, float ks, bool valid, float* dbg) +{ + uint32_t r[32]; + tmemLd32(taddr, r); + float4 const* w4 = reinterpret_cast(sW); + float c0 = 0.f; + float c1 = 0.f; + float c2 = 0.f; + float c3 = 0.f; +#pragma unroll + for (int h4 = 0; h4 < 8; ++h4) + { + float4 const wv = w4[h4]; + c0 = fmaf(wv.x, fmaxf(__uint_as_float(r[4 * h4]), 0.f), c0); + c1 = fmaf(wv.y, fmaxf(__uint_as_float(r[4 * h4 + 1]), 0.f), c1); + c2 = fmaf(wv.z, fmaxf(__uint_as_float(r[4 * h4 + 2]), 0.f), c2); + c3 = fmaf(wv.w, fmaxf(__uint_as_float(r[4 * h4 + 3]), 0.f), c3); + } + float const acc = (c0 + c1) + (c2 + c3); + float const lgt = valid ? acc * ks : -INFINITY; + sLog[kt] = lgt; + if (dbg != nullptr) + { + *dbg = lgt; + } + return lgt; +} + +// IdxSel tile MMAs (one warp): D = K8 Q8^T over the 4 K steps of 32, committed on bar (with commit: one commit after +// several tiles covers them all). +__device__ __noinline__ void selTileMma( + uint32_t d, uint32_t aK, uint32_t aQ, uint32_t bar, uint32_t idesc, bool commit = true) +{ + tcFenceAfter(); + if (electOne()) + { +#pragma unroll + for (int ks = 0; ks < 4; ++ks) + { + mmaF8(d, aK + 32 * ks, aQ + 32 * ks, ks > 0 ? 1u : 0u, idesc); + } + if (commit) + { + commit1(bar); + } + } + __syncwarp(); +} + +#if GLM52_SEL_BATCH_MMA +// One row-shard batch, after pagesIn's proxy fence and CTA synchronization. +// All tiles use disjoint TMEM columns; keep each tile's original four-K-step chain. +// One elected lane issues every tile and the existing single completion commit. +__device__ __noinline__ void selBatchMma(uint32_t d, uint32_t aK, uint32_t aQ, uint32_t bar, int nt) +{ + tcFenceAfter(); + if (electOne()) + { +#pragma unroll 1 + for (int i = 0; i < nt; ++i) + { +#pragma unroll + for (int ks = 0; ks < 4; ++ks) + { + mmaF8(d + 32u * static_cast(i), aK + 16384u * static_cast(i) + 32 * ks, + aQ + 32 * ks, ks > 0 ? 1u : 0u, kIdescF8N32); + } + } + if (nt > 0) + { + commit1(bar); + } + } + __syncwarp(); +} +#endif + +// Leader top-k helpers, row t = warp t, lane l over the CTA's local keys l + 32 m. Logit (row t, local key k) of the +// CTA: tile k / 128's first 4 KiB, [t][k % 128], from shared address a0 + kSelK (t < 0, the row shard: the row's array +// at a0 + kSelLog), as an order key (0 = none). +__device__ __forceinline__ uint32_t selKeyAt(uint32_t a0, int t, int k) +{ + float const f = __uint_as_float( + lds32(t < 0 ? a0 + kSelLog + 4 * k : a0 + kSelK + (k >> 7) * 16384 + (t * 128 + (k & 127)) * 4)); + return f == -INFINITY ? 0u : orderKey(f); +} + +// Over the nBins bins of the shared histogram h (a multiple of kThreads; bin b counts keys whose digit is b): the bin +// holding the need-th largest key (counted from the top bin down) into scal[8], the keys in the bins above it into +// scal[9]. scal[0, 8): the warps' sums. Every thread calls it; the bins must sum to at least need. +__device__ __forceinline__ void idxBinFromTop(uint32_t const* h, int nBins, uint32_t need, uint32_t* scal) +{ + int const tid = threadIdx.x; + int const lane = tid & 31; + int const warp = tid >> 5; + int const per = nBins / kThreads; + int const hi = nBins - per * tid; // this thread's bins: [hi - per, hi), the top chunk at thread 0 + uint32_t s = 0; + for (int j = 0; j < per; ++j) + { + s += h[hi - 1 - j]; + } + uint32_t incl = s; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) + { + uint32_t const v = __shfl_up_sync(0xffffffffu, incl, off); + incl += lane >= off ? v : 0u; + } + if (lane == 31) + { + scal[warp] = incl; + } + __syncthreads(); + uint32_t before = incl - s; + for (int w = 0; w < warp; ++w) + { + before += scal[w]; + } + if (before < need && before + s >= need) + { + for (int j = 0; j < per; ++j) + { + uint32_t const c = h[hi - 1 - j]; + if (before + c >= need) + { + scal[8] = static_cast(hi - 1 - j); + scal[9] = before; + break; + } + before += c; + } + } + __syncthreads(); +} + +// Every CTA, row t = warp t, lane l over the CTA's local keys l + 32 m (in shared memory). The 10th largest of +// the lanes' four largest keys (ten keys of the CTA are >= it, so it is at most the CTA's 10th-largest key; it is that +// key unless a lane holds five of the top ten) into the arena's word of (CTA, row). From the words, per row: T(t) = +// their minimum, leaving out up to 7 CTAs whose word is not positive (ReLU logits: a CTA can hold fewer than ten keys +// above 0): the other >= 205 CTAs' 2050 keys are >= T(t), so T(t) <= the row's 2048th largest; E(t) = the geometric +// mean of the positive words (the mean of their bits), an estimate of the 2048th without a certificate (within ~1 % on +// the c1 rows), when at least half the CTAs have one; and T2(t) = max(T(t), E(t) - kTightGap). The CTA's keys >= T2(t) +// go into row t's list A, its keys in [T(t), T2(t)) into list B: the selector takes list A alone when it holds need0 +// keys (then the row's 2048 are among them), A and B otherwise. Keys equal to T2 (list A) and to T (list B) are taken +// kTieFree freely, more from the list's pool of kTopK (equal keys are interchangeable, and the pool covers any row's +// need). One 64-bit atomic per row takes the CTA's slots in both lists and counts the CTA in ((1 << 24) | A count, then +// B count and the CTA's keys in list A's histogram << 18), then the (key, position) pairs as single 64-bit stores (the +// selector polls them; key 0 = not written yet). List A's keys in [T2, T2 + kHistSpan) also go into the row's histogram +// (a relaxed add per key). sMin: shared scratch; T(t) and T2(t) left at sMin + 768 and + 896 (+ 4 t). +constexpr uint32_t kBndZero = 0x007fffffu; // the word of a +0.0 bound (= kEmpty): orderKey(-inf), never a bound +constexpr int kIdxTCta = kIdxSelBase; // the CTA that takes T from the words (its logits end among the first) +constexpr uint32_t kTieFree = 16; +constexpr int kUnionWords = kCtas * 64; // per row: every CTA lane's two largest keys +constexpr int kUnionVecs = kUnionWords / 4; // as 16-byte vectors +constexpr int kUnionPer = (kUnionVecs + kThreads - 1) / kThreads; // a thread's vectors: tid + 256 u +// Per row after the union: every CTA's summary of its 64 words (two 16-byte vectors, idxUnionBound). +constexpr int kUnionSumWords = kCtas * 8; +static_assert((kUnionWords + kUnionSumWords) * 4 <= (kCandCap - kCandList) * 8, "the union's words past list A"); + +// Inserts key x into a lane's two largest (m1 >= m2; 0 = none). Equal keys are distinct entries. +__device__ __forceinline__ void top2Insert(uint32_t& m1, uint32_t& m2, uint32_t x) +{ + if (x > m1) + { + m2 = m1; + m1 = x; + } + else if (x > m2) + { + m2 = x; + } +} + +// One 256-bin pass of the union's keys (kUnionWords words at shared address sU): keys k with k - lo <= lim (the window +// [lo, lo + lim], lim < 256 << sh; k < lo wraps past lim since lo >= 1 and lim <= 0xffffffff - lo; 0 is no key) to bin +// (k - lo) >> sh, the others to dummy bins, into a histogram with a copy per lane (bin b of lane l at word 32 b + l, +// the dummies past bin 255: a warp's adds never share a bank, and none branches: a conditional add compiled to a branch +// with a reconvergence barrier per key), then bin t's count in thread t and the bin c holding the need-th largest +// (count(>= its lower edge) >= need): c = (bins with that property) - 1, -1 when even bin 0 holds fewer. Returns c; +// cnt: bin c's count; abv: the keys in bins above c. sHist: (256 + kUnionDummies) x 32 words; sRed: 16 words. Not +// inlined and rolled: it runs once or twice per launch from a cold instruction cache (the unrolled register version +// took 6.2 us per pass in the kernel against 0.75 us warm alone, mk-u05a). Two vectors per iteration, both loaded +// before the adds, and the adds without a memory clobber: 0.61 us per 13,568 keys warm, against ~1.0 one vector at a +// time. +constexpr int kUnionDummies = 8; +// The first pass's key window below the union's largest key (log2; order keys: 2^23 per binade). +#ifndef GLM52_UNION_WIN +#define GLM52_UNION_WIN 28 +#endif +__device__ __noinline__ int unionPass(uint32_t sU, uint32_t lo, int sh, uint32_t lim, uint32_t need, uint32_t sHist, + uint32_t sRed, uint32_t& cnt, uint32_t& abv) +{ + int const tid = threadIdx.x; + int const lane = tid & 31; + int const warp = tid >> 5; +#pragma unroll 1 + for (int i = tid; i < 256 * 32 / 4; i += kThreads) + { + sts128(sHist + 16 * i, make_uint4(0u, 0u, 0u, 0u)); + } + __syncthreads(); + uint32_t const laneOff = sHist + 4 * lane; + auto add = [&](uint32_t k, uint32_t dummy) + { + uint32_t const d = k - lo; + redsAddUnordered(laneOff + 128 * (d <= lim ? d >> sh : dummy), 1u); + }; +#pragma unroll 1 + for (int i = tid; i < kUnionVecs; i += 2 * kThreads) + { + bool const two = i + kThreads < kUnionVecs; + uint4 const q = lds128(sU + 16 * i); + uint4 r = lds128(sU + 16 * min(i + kThreads, kUnionVecs - 1)); + r = two ? r : make_uint4(0u, 0u, 0u, 0u); + uint32_t const dummy = 256u + static_cast(i & (kUnionDummies - 1)); + add(q.x, dummy); + add(q.y, dummy); + add(q.z, dummy); + add(q.w, dummy); + add(r.x, dummy ^ 1u); + add(r.y, dummy ^ 1u); + add(r.z, dummy ^ 1u); + add(r.w, dummy ^ 1u); + } + __syncthreads(); + // Bin tid's 32 copies as 8 vectors, rotated by tid so a warp's loads spread over the banks. + uint32_t s = 0; +#pragma unroll + for (int j = 0; j < 8; ++j) + { + uint4 const q = lds128(sHist + 16 * (8 * tid + ((j + tid) & 7))); + s += (q.x + q.y) + (q.z + q.w); + } + // Suffix sums from bin 255 down: within the warp, then the warps above (their 8 totals in two loads). + uint32_t suf = s; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) + { + uint32_t const x = __shfl_down_sync(0xffffffffu, suf, off); + suf += lane + off < 32 ? x : 0u; + } + if (lane == 0) + { + sts32(sRed + 4 * warp, suf); + } + __syncthreads(); + { + uint4 const a = lds128(sRed); + uint4 const b = lds128(sRed + 16); + uint32_t const tw[8] = {a.x, a.y, a.z, a.w, b.x, b.y, b.z, b.w}; +#pragma unroll + for (int w = 1; w < kWarps; ++w) + { + suf += w > warp ? tw[w] : 0u; + } + } + int const c = __syncthreads_count(suf >= need) - 1; + if (tid == c) + { + sts32(sRed + 4 * 8, s); + sts32(sRed + 4 * 9, suf - s); + } + __syncthreads(); + cnt = c >= 0 ? lds32(sRed + 4 * 8) : 0u; + abv = c >= 0 ? lds32(sRed + 4 * 9) : 0u; + return c; +} + +// unionPass's lim: the window of 256 bins of 2^sh from lo, up to cap. +__device__ __forceinline__ uint32_t unionLim(uint32_t lo, int sh, uint32_t cap) +{ + uint64_t const end = static_cast(lo) + (256ull << sh) - 1ull; + return static_cast((end < cap ? end : static_cast(cap)) - lo); +} + +// The union's count and range (its nonzero keys; kFlagUnionSide: the positive keys, order key >= 0x80000000, their +// smallest, and the largest negative key), from the CTAs' summaries of their 64 words (idxUnionBound). +struct UnionStats +{ + uint32_t tot; + uint32_t lo; + uint32_t hi; + uint32_t np; + uint32_t minPos; + uint32_t maxNeg; +}; + +// The need0-th largest of the union's keys (kUnionWords words at shared address sU, 0 = none; st: their count and +// range), bounded from below: two 256-bin passes (unionPass): over the top 2^GLM52_UNION_WIN keys of the range (the +// row's need0-th largest lies far closer to the largest key; the whole range when that holds too few), then over the +// cut bin. T = the last cut bin's lower edge: at least need0 keys are >= T, within max(1, 2^-16 of the range) of the +// need0-th largest. With fewer than need0 keys: 1 (every key of the row). hsh: list A's histogram bin width (log2) for +// ~4 of the union's keys per bin at T's density (14 when the first cut bin held < 16). sHist: (256 + kUnionDummies) x +// 32 words; sRed: 16 words. Every thread calls it. +__device__ __noinline__ uint32_t idxUnionKth( + Ctx const& ctx, uint32_t sU, int need0, uint32_t sHist, uint32_t sRed, uint32_t& hsh, UnionStats st) +{ + constexpr bool kSide = (kSpecializedStaticFlags & kFlagUnionSide) != 0; + hsh = 14u; + if (st.tot < static_cast(need0)) + { + return 1u; + } + uint32_t const need = static_cast(need0); + // kFlagUnionSide: pass 1 over the side of zero that holds the need-th largest key: [minPos, hi] with need when + // there are need positive keys, else [lo, maxNeg] with need - np (every positive key is above it; keys above + // maxNeg go to the dummy bins in both passes). + uint32_t top = st.hi; + uint32_t bot = st.lo; + uint32_t needS = need; + uint32_t cap = 0xffffffffu; + if constexpr (kSide) + { + if (st.np >= need) + { + bot = st.minPos; + } + else if (st.np > 0) + { + top = st.maxNeg; + needS = need - st.np; + cap = st.maxNeg; + } + } + uint32_t base = top - bot > (1u << GLM52_UNION_WIN) ? top - (1u << GLM52_UNION_WIN) : bot; + // (top - base) >> sh < 256. __clz's result is unsigned in this toolchain: without the cast 24 - __clz(.) wraps + // below a range of 128 and max(0, .) compares unsigned, leaving sh negative. + int sh = max(0, 24 - static_cast(__clz(top - base))); + uint32_t cnt; + uint32_t abv; + int b = unionPass(sU, base, sh, unionLim(base, sh, cap), needS, sHist, sRed, cnt, abv); + if (b < 0) + { + // The top 2^26 keys of the range hold fewer than need0: the whole range (one pass, coarser). + base = st.lo; + sh = max(0, 24 - static_cast(__clz(st.hi - st.lo))); + needS = need; + cap = 0xffffffffu; + b = max(0, unionPass(sU, base, sh, unionLim(base, sh, cap), need, sHist, sRed, cnt, abv)); + } + if (cnt >= 16u) + { + hsh = static_cast(min(16, max(8, sh + 2 - (31 - static_cast(__clz(cnt)))))); + } + base += static_cast(b) << sh; + subStamp(ctx, 133); + if (sh > 0) + { + // Pass 2 counts the cut bin [base, base + 2^sh) only: for sh < 8 its 256 unit bins reach past the bin into keys + // already in abv. + int const sh2 = max(0, sh - 8); + uint32_t const binTop = base > 0xffffffffu - ((1u << sh) - 1u) ? 0xffffffffu : base + ((1u << sh) - 1u); + int const b2 + = unionPass(sU, base, sh2, unionLim(base, sh2, min(cap, binTop)), needS - abv, sHist, sRed, cnt, abv); + base += static_cast(max(b2, 0)) << sh2; + } + subStamp(ctx, 134); + return max(base, 1u); +} + +// The leader's bound for the rows this CTA's warps handle (GLM52_TOPK_UNION). Every lane of every CTA puts the two +// largest of the keys it holds (64 per CTA and row) into the row's union, kUnionWords words past list A (key + 1: 0 = +// not written; the row's candidates use list A alone). CTA kIdxTCta polls a row's union as 16-byte vectors into +// registers, zeroes the words for the next launch of this copy and takes T, a lower bound of the union's need0-th +// largest key within one histogram bin (idxUnionKth): at least need0 of the row's keys, the union's, are >= T. T = T2 +// (one list, no estimate) is published to every CTA. With two keys per lane, T sits within a few keys of the row's +// need0-th largest unless a lane holds three of the row's top keys (with one, T was 30-50 % below it at 32k-65k +// keys: 3.6-3.8k candidates, and list A's fine bins missed the cut). (The minimum of the CTAs' bounds and the estimate +// from the positive ones assumed positive logits spread over the CTAs: served rows are negative, and one CTA holds up +// to ~66 of a row's top 2048. The union's exact need0-th largest from three radix passes over warp-aggregated adds +// (__match_any_sync) took 17 us from the last CTA's words to T, one histogram pass 6.6 us: 723.6 -> 685.3 us per slice +// at 102k, mk-t08.) +__device__ __noinline__ void idxUnionBound(Ctx const& ctx, uint32_t a0, int nPer, int cta, int t, int tL, bool rs, + bool active, uint64_t* cand, uint32_t sMin, uint32_t* bnd, int L) +{ + int const warp = threadIdx.x >> 5; + int const lane = threadIdx.x & 31; + if (active) + { + uint32_t m1 = 0; + uint32_t m2 = 0; + if (rs && !(GLM52_FLAGS(ctx) & (kFlagIdxTilePairs | kFlagBoundLoop))) + { + // The row shard's tile epilogues kept them: the two largest of the 8 warps' pairs of this lane. +#pragma unroll + for (int w = 0; w < kWarps; ++w) + { + top2Insert(m1, m2, lds32(sMin + kSelLaneMax + 4 * (32 * w + lane))); + top2Insert(m1, m2, lds32(sMin + kSelLaneMax2 + 4 * (32 * w + lane))); + } + } + else + { +#pragma unroll 4 + for (int m = 0; m < nPer; ++m) + { + top2Insert(m1, m2, selKeyAt(a0, tL, lane + 32 * m)); + } + } + subStamp(ctx, 64); + // key + 1: 0 = not written (orderKey(+FLT_MAX) + 1 does not wrap). + uint32_t* const uni = reinterpret_cast(cand + static_cast(t) * kCandCap + kCandList); + st32(uni + 64 * cta + lane, m1 + 1u); + st32(uni + 64 * cta + 32 + lane, m2 + 1u); + // The CTA's summary of its 64 words (m1 >= m2, 0 = none) for the T CTA's count and range (a pass over the union + // took ~1 us there): the largest and smallest key, the smallest positive and largest negative key, the + // positive keys and all keys. Every word is nonzero once written (0xffffffff: none; the others + 1). + bool const p1 = m1 >= 0x80000000u; + bool const p2 = m2 >= 0x80000000u; + uint32_t const hiC = __reduce_max_sync(0xffffffffu, m1); + uint32_t const loC = __reduce_min_sync(0xffffffffu, m2 != 0u ? m2 : m1 != 0u ? m1 : 0xffffffffu); + uint32_t const minPosC = __reduce_min_sync(0xffffffffu, p2 ? m2 : p1 ? m1 : 0xffffffffu); + uint32_t const maxNegC = __reduce_max_sync(0xffffffffu, !p1 ? m1 : !p2 ? m2 : 0u); + uint32_t const npC = __reduce_add_sync(0xffffffffu, (m1 >> 31) + (m2 >> 31)); + uint32_t const cC = __reduce_add_sync(0xffffffffu, (m1 != 0u ? 1u : 0u) + (m2 != 0u ? 1u : 0u)); + if (lane == 0) + { + uint32_t* const sum = uni + kUnionWords + 8 * cta; + st128(sum, + make_uint4(hiC + 1u, loC == 0xffffffffu ? loC : loC + 1u, + minPosC == 0xffffffffu ? minPosC : minPosC + 1u, maxNegC + 1u)); + st128(sum + 4, make_uint4(npC + 1u, cC + 1u, 1u, 1u)); + } + } + subStamp(ctx, 65); + uint32_t* const tw = bnd + kCtas * kTok; + // kFlagUnionLean: CTA 127 takes T while it holds fewer key pages than CTAs 128-211 (with kFlagPageBalance, always: + // kIdxBalDeficit fewer): its logits end early, so its union poll starts before the last CTAs' stores (CTA 204, then + // a 14-page CTA, was one of the last). + constexpr bool kLean = (kSpecializedStaticFlags & kFlagUnionLean) != 0; + int const nPagesT = (L + kIdxPageKeys - 1) / kIdxPageKeys; + int const tCta + = kLean && idxPagesOf(kIdxQUnits - 1, nPagesT) < idxPagesOf(kIdxQUnits, nPagesT) ? kIdxQUnits - 1 : kIdxTCta; + if (cta == tCta) + { + // The histogram over Q8 and the key scales and its scratch after it (dead after the MMAs; the CTA's logits are + // past them). + uint32_t const sU = a0 + kSelK; + uint32_t const sHist = a0 + kSelQ8; + uint32_t const sRed = sHist + 4 * (256 + kUnionDummies) * 32; + static_assert(kSelK + 4 * kUnionWords <= kSelQ8, "the union in the K region"); + static_assert(kSelQ8 + 4 * (256 + kUnionDummies) * 32 + 4 * 48 <= kSelLog, "the T CTA's histogram and scratch"); +#pragma unroll 1 + for (int r = 0; r < kTok; ++r) + { + if (rs && r != t) + { + continue; + } + // The union's vectors into registers, all in flight at once; each round re-polls every vector of the + // thread not complete yet (one round trip for them all: one vector at a time took a round trip each), and + // the warp leaves the loop together (warpAll: a split warp takes the slow path of every warp collective + // after it). + uint32_t* const uni = reinterpret_cast(cand + static_cast(r) * kCandCap + kCandList); + // Thread c < kCtas: CTA c's summary too (vectors kUnionPer and kUnionPer + 1). + uint32_t* const sum = uni + kUnionWords + 8 * threadIdx.x; + bool const hasSum = threadIdx.x < kCtas; + uint4 v[kUnionPer + 2]; +#pragma unroll + for (int u = 0; u < kUnionPer; ++u) + { + int const i = threadIdx.x + u * kThreads; + v[u] = i < kUnionVecs ? ldPoll128(uni + 4 * i) : make_uint4(1u, 1u, 1u, 1u); + } + v[kUnionPer] = hasSum ? ldPoll128(sum) : make_uint4(1u, 1u, 1u, 1u); + v[kUnionPer + 1] = hasSum ? ldPoll128(sum + 4) : make_uint4(1u, 1u, 1u, 1u); + auto pending = [&]() + { + uint32_t p = 0; +#pragma unroll + for (int u = 0; u < kUnionPer + 2; ++u) + { + p |= (v[u].x == 0u) | (v[u].y == 0u) | (v[u].z == 0u) | (v[u].w == 0u) ? 1u << u : 0u; + } + return p; + }; + uint32_t p = pending(); + uint64_t t0 = 0; + while (!warpAll(p == 0u)) + { + spinCheck(ctx, t0, 181, threadIdx.x, r); +#pragma unroll + for (int u = 0; u < kUnionPer; ++u) + { + if ((p >> u) & 1u) + { + v[u] = ldPoll128(uni + 4 * (threadIdx.x + u * kThreads)); + } + } + if ((p >> kUnionPer) & 1u) + { + v[kUnionPer] = ldPoll128(sum); + } + if ((p >> (kUnionPer + 1)) & 1u) + { + v[kUnionPer + 1] = ldPoll128(sum + 4); + } + p = pending(); + } + // Into shared memory at the start of the K region (dead after the MMAs: the CTA's logits are past kSelLog), + // the slots back to zero for the next launch of this copy. +#pragma unroll + for (int u = 0; u < kUnionPer; ++u) + { + int const i = threadIdx.x + u * kThreads; + if (i < kUnionVecs) + { + sts128(sU + 16 * i, make_uint4(v[u].x - 1u, v[u].y - 1u, v[u].z - 1u, v[u].w - 1u)); + if constexpr (!kLean) + { + st128(uni + 4 * i, make_uint4(0u, 0u, 0u, 0u)); + } + } + } + if (!kLean && hasSum) + { + st128(sum, make_uint4(0u, 0u, 0u, 0u)); + st128(sum + 4, make_uint4(0u, 0u, 0u, 0u)); + } + // The count and range: the summaries' (8 warps' partials through sRed, then every thread sums them). + UnionStats st; + { + uint4 const a = v[kUnionPer]; + uint4 const b = v[kUnionPer + 1]; + uint32_t const hiS = hasSum ? a.x - 1u : 0u; + uint32_t const loS = hasSum && a.y != 0xffffffffu ? a.y - 1u : 0xffffffffu; + uint32_t const minPosS = hasSum && a.z != 0xffffffffu ? a.z - 1u : 0xffffffffu; + uint32_t const maxNegS = hasSum ? a.w - 1u : 0u; + uint32_t const npS = hasSum ? b.x - 1u : 0u; + uint32_t const cS = hasSum ? b.y - 1u : 0u; + uint32_t const warpV[6] = {__reduce_add_sync(0xffffffffu, cS), __reduce_min_sync(0xffffffffu, loS), + __reduce_max_sync(0xffffffffu, hiS), __reduce_add_sync(0xffffffffu, npS), + __reduce_min_sync(0xffffffffu, minPosS), __reduce_max_sync(0xffffffffu, maxNegS)}; + if (lane < 6) + { + uint32_t x = warpV[0]; +#pragma unroll + for (int j = 1; j < 6; ++j) + { + x = lane == j ? warpV[j] : x; + } + sts32(sRed + 4 * (8 * lane + warp), x); + } + __syncthreads(); + // Stat j's 8 warp values at sRed + 32 j. + auto sum8 = [&](int j) + { + uint4 const x = lds128(sRed + 32 * j); + uint4 const y = lds128(sRed + 32 * j + 16); + return ((x.x + x.y) + (x.z + x.w)) + ((y.x + y.y) + (y.z + y.w)); + }; + auto min8 = [&](int j) + { + uint4 const x = lds128(sRed + 32 * j); + uint4 const y = lds128(sRed + 32 * j + 16); + return min(min(min(x.x, x.y), min(x.z, x.w)), min(min(y.x, y.y), min(y.z, y.w))); + }; + auto max8 = [&](int j) + { + uint4 const x = lds128(sRed + 32 * j); + uint4 const y = lds128(sRed + 32 * j + 16); + return max(max(max(x.x, x.y), max(x.z, x.w)), max(max(y.x, y.y), max(y.z, y.w))); + }; + st = UnionStats{sum8(0), min8(1), max8(2), sum8(3), min8(4), max8(5)}; + } + __syncthreads(); + subStamp(ctx, 132); + // List A's 64 histogram bins from T get ~4 of the union's keys each at T's density (hsh): T sits within + // a few keys of the row's need0-th largest, so the cut lands in the first bins and holds few keys (a fixed + // 2^14 put up to ~60 keys in it on served rows, ranked one by one). + uint32_t hsh; + uint32_t const T = idxUnionKth(ctx, sU, min(kTopK, L - kTok + r + 1), sHist, sRed, hsh, st); + if (threadIdx.x == 0) + { + uint32_t const enc = T == kEmpty ? kBndZero : T; + stU64(tw + 2 * r, (static_cast(hsh) << 32) | enc); + } + if constexpr (kLean) + { + // The slots back to zero for the next launch of this copy, after T's publish. +#pragma unroll + for (int u = 0; u < kUnionPer; ++u) + { + int const i = threadIdx.x + u * kThreads; + if (i < kUnionVecs) + { + st128(uni + 4 * i, make_uint4(0u, 0u, 0u, 0u)); + } + } + if (hasSum) + { + st128(sum, make_uint4(0u, 0u, 0u, 0u)); + st128(sum + 4, make_uint4(0u, 0u, 0u, 0u)); + } + } + __syncthreads(); // the histogram again for the next row + } + subStamp(ctx, 77); + } + if (warp == 0) + { + bool const want = rs ? lane == t : lane < kTok; + uint64_t v = want ? ldPollU64(tw + 2 * lane) : 0ull; + uint64_t t0 = 0; + while (!warpAll(!want || static_cast(v) != kEmpty)) + { + spinCheck(ctx, t0, 167, lane, 0); + if (want && static_cast(v) == kEmpty) + { + v = ldPollU64(tw + 2 * lane); + } + } + if (want) + { + uint32_t const lo = static_cast(v); + uint32_t const T = lo == kBndZero ? kEmpty : lo; + sts32(sMin + 768 + 4 * lane, T); + sts32(sMin + 896 + 4 * lane, T); + sts32(sMin + 800 + 4 * lane, 1u); // the selector's E: none + sts32(sMin + 864 + 4 * lane, static_cast(v >> 32)); // list A's histogram bin width (log2) + } + } +} + +__device__ __noinline__ void idxBoundCands( + Ctx const& ctx, uint32_t a0, int nTiles, int cta, uint32_t* gc, uint64_t* cand, uint32_t sMin, uint32_t* bnd, int L) +{ + int const warp = threadIdx.x >> 5; + int const lane = threadIdx.x & 31; + // The row shard: warp 0 alone handles the rank's row (the tiles hold its logits as row 0). + bool const rs = (GLM52_FLAGS(ctx) & kFlagIdxAllRows) == 0; + int const t = rs ? ctx.rank >> 1 : warp; // the row (words, lists, counters) + int const tL = rs ? -1 : warp; // its logits' row in the tiles (-1: the row shard's array) + bool const active = !rs || warp == 0; + int const nPer = nTiles * 4; + // The loops are rolled (by four, the keys read from shared memory each time): this code runs once per leader launch + // from cold instruction caches, ~25-55 ns per 128-byte line, which made a fully unrolled 32-key version 3x slower. +#if GLM52_TOPK_UNION + idxUnionBound(ctx, a0, nPer, cta, t, tL, rs, active, cand, sMin, bnd, L); +#else + if (!(GLM52_FLAGS(ctx) & kFlagTopkTCta)) + { + // Each lane's largest key; the warp's 32 maxima in descending order (bitonic): the 10th is a certified bound + // (ten lanes hold a key >= it, so ten of the CTA's keys are), the 9th the estimate's sample (~ the CTA's + // 10th-largest key). One 64-bit word per (row, CTA) (rows' words contiguous): 9th << 32 | 10th. Every CTA then + // takes T, T2 and E of its row from the 212 words itself (one hop; the active warp, seven words per lane). (The + // lanes' top four and ten warp rounds took ~1.5 us, and a T CTA a second hop.) + if (active) + { + uint32_t m1 = 0; + if (rs && !(GLM52_FLAGS(ctx) & (kFlagIdxTilePairs | kFlagBoundLoop))) + { + // The row shard's tile epilogues kept them: the maximum of the 8 warps' words of this lane. +#pragma unroll + for (int w = 0; w < kWarps; ++w) + { + m1 = max(m1, lds32(sMin + kSelLaneMax + 4 * (32 * w + lane))); + } + } + else + { +#pragma unroll 4 + for (int m = 0; m < nPer; ++m) + { + m1 = max(m1, selKeyAt(a0, tL, lane + 32 * m)); + } + } + subStamp(ctx, 64); +#pragma unroll + for (int k = 2; k <= 32; k <<= 1) + { +#pragma unroll + for (int j = k >> 1; j > 0; j >>= 1) + { + uint32_t const o = __shfl_xor_sync(0xffffffffu, m1, j); + m1 = ((lane & k) == 0) == ((lane & j) == 0) ? max(m1, o) : min(m1, o); + } + } + uint32_t const tenth = __shfl_sync(0xffffffffu, m1, 9); + uint32_t const ninth = __shfl_sync(0xffffffffu, m1, 8); + uint64_t* const rowW = reinterpret_cast(bnd) + t * kCtas; + if (lane == 0) + { + stU64(rowW + cta, (static_cast(ninth) << 32) | (tenth == kEmpty ? kBndZero : tenth)); + } + subStamp(ctx, 65); + uint64_t w[7]; +#pragma unroll + for (int i = 0; i < 7; ++i) + { + w[i] = lane + 32 * i < kCtas ? ldPollU64(rowW + lane + 32 * i) : 0ull; + } + uint64_t t0 = 0; + while (true) + { + bool miss = false; +#pragma unroll + for (int i = 0; i < 7; ++i) + { + miss |= static_cast(w[i]) == kEmpty; + } + if (warpAll(!miss)) + { + break; + } + spinCheck(ctx, t0, 166, lane, 0); +#pragma unroll + for (int i = 0; i < 7; ++i) + { + if (static_cast(w[i]) == kEmpty) + { + w[i] = ldPollU64(rowW + lane + 32 * i); + } + } + } + // T: the minimum of the 10ths, leaving out up to 7 CTAs whose 10th is not positive; E: the geometric mean + // of the positive 9ths (the mean of their bits), when at least half the CTAs have one. + uint32_t mnAll = ~0u; + uint32_t mnPos = ~0u; + uint32_t nNon = 0; + uint32_t sumE = 0; + uint32_t nPosE = 0; +#pragma unroll + for (int i = 0; i < 7; ++i) + { + bool const valid = lane + 32 * i < kCtas; + uint32_t const lw = static_cast(w[i]); + uint32_t const tv = lw == kBndZero ? kEmpty : lw; + uint32_t const ev = static_cast(w[i] >> 32); + bool const posT = valid && tv > kEmpty; + bool const posE = valid && ev > kEmpty; + mnAll = valid ? min(mnAll, tv) : mnAll; + mnPos = posT ? min(mnPos, tv) : mnPos; + nNon += valid && !posT ? 1u : 0u; + sumE += posE ? (ev - kEmpty) >> 8 : 0u; + nPosE += posE ? 1u : 0u; + } + mnAll = __reduce_min_sync(0xffffffffu, mnAll); + mnPos = __reduce_min_sync(0xffffffffu, mnPos); + nNon = __reduce_add_sync(0xffffffffu, nNon); + sumE = __reduce_add_sync(0xffffffffu, sumE); + nPosE = __reduce_add_sync(0xffffffffu, nPosE); + uint32_t const Tr = max(nNon <= 7 && mnPos != ~0u ? mnPos : mnAll, 1u); + uint32_t const est = 2 * nPosE >= static_cast(kCtas) && !(GLM52_FLAGS(ctx) & kFlagTopkOneList) + ? kEmpty + ((sumE / nPosE) << 8) + : 0u; + uint32_t const T2r = est > Tr + kTightGap ? est - kTightGap : Tr; + if (lane == 0) + { + sts32(sMin + 768 + 4 * t, Tr); + sts32(sMin + 896 + 4 * t, T2r); + sts32(sMin + 800 + 4 * t, est > Tr && est != kEmpty ? est : 1u); // the selector's E (1: none) + } + } + } + else + { + uint32_t m1 = 0; + uint32_t m2 = 0; + uint32_t m3 = 0; + uint32_t m4 = 0; +#pragma unroll 4 + for (int m = 0; m < (active ? nPer : 0); ++m) + { + uint32_t const v = selKeyAt(a0, tL, lane + 32 * m); + uint32_t const r1 = min(m1, v); + uint32_t const r2 = min(m2, r1); + m4 = max(m4, min(m3, r2)); + m3 = max(m3, r2); + m2 = max(m2, r1); + m1 = max(m1, v); + } + subStamp(ctx, 64); + // Ten rounds of the warp maximum over the lanes' heads (a lane's next key follows its head). + uint32_t tenth = 0; +#pragma unroll 1 + for (int round = 0; round < (active ? 10 : 0); ++round) + { + uint32_t const wmax = __reduce_max_sync(0xffffffffu, m1); + uint32_t const who = __ballot_sync(0xffffffffu, m1 == wmax); + if (lane == __ffs(who) - 1) + { + m1 = m2; + m2 = m3; + m3 = m4; + m4 = 0; + } + tenth = wmax; + } + if (active && lane == 0) + { + st32(bnd + cta * kTok + t, tenth == kEmpty ? kBndZero : tenth); + } + subStamp(ctx, 65); + // CTA kIdxTCta polls the CTAs' words (thread c < kCtas: CTA c's 8), takes T, T2 and E per row (each from the + // warp minima, counts and sums of the words, then over the warps) and publishes them right after the words: per + // row one 64-bit word T | T2 << 32, then the 8 E words (for the selectors). Every CTA's warp 0 polls the 8 + // rows' words (all warps polling the same lines, three words each, slowed the hand-off by ~0.8 us). One CTA + // polling 212 x 8 words instead of all of them: every CTA taking T itself (one hop) came out even (mk-v73a). + uint32_t* const tw = bnd + kCtas * kTok; + if (cta == kIdxTCta) + { + uint32_t w[kTok]; + { + int const c = threadIdx.x; + uint4 b0 = make_uint4(~0u, ~0u, ~0u, ~0u); + uint4 b1 = b0; + if (rs) + { + // Row t only: its word of every CTA, at its place among b0 / b1 (the other rows stay ~0u). + uint32_t v = c < kCtas ? ldPoll32(bnd + c * kTok + t) : ~0u; + uint64_t t0 = 0; + while (!warpAll(v != kEmpty)) + { + spinCheck(ctx, t0, 166, c, 0); + v = c < kCtas && v == kEmpty ? ldPoll32(bnd + c * kTok + t) : v; + } + uint32_t* const bw = t < 4 ? &b0.x : &b1.x; + bw[t & 3] = v; + } + else + { + if (c < kCtas) + { + b0 = ldPoll128(bnd + c * kTok); + b1 = ldPoll128(bnd + c * kTok + 4); + } + uint64_t t0 = 0; + while (!warpAll(full4(b0) && full4(b1))) + { + spinCheck(ctx, t0, 166, c, 0); + if (c < kCtas && !full4(b0)) + { + b0 = ldPoll128(bnd + c * kTok); + } + if (c < kCtas && !full4(b1)) + { + b1 = ldPoll128(bnd + c * kTok + 4); + } + } + } + w[0] = b0.x; + w[1] = b0.y; + w[2] = b0.z; + w[3] = b0.w; + w[4] = b1.x; + w[5] = b1.y; + w[6] = b1.z; + w[7] = b1.w; + } + subStamp(ctx, 78); + // Per (warp, row) at sMin + 1024 (the selector's scratch, not in use yet): the minimum of all words and of + // the positive ones, the count of the others and the sum of the positive ones' bits >> 8 (< 2^31 over 212). + uint32_t const sRed = sMin + 1024; +#pragma unroll + for (int r = 0; r < kTok; ++r) + { + if (rs && r != t) + { + continue; + } + uint32_t const v = w[r] == kBndZero ? kEmpty : w[r]; + bool const pos = v > kEmpty && v != ~0u; + uint32_t const mnAll = __reduce_min_sync(0xffffffffu, v); + uint32_t const mnPos = __reduce_min_sync(0xffffffffu, pos ? v : ~0u); + uint32_t const nNon = __reduce_add_sync(0xffffffffu, threadIdx.x < kCtas && !pos ? 1u : 0u); + uint32_t const sum = __reduce_add_sync(0xffffffffu, pos ? (v - kEmpty) >> 8 : 0u); + if (lane == 0) + { + sts128(sRed + 16 * (kTok * warp + r), make_uint4(mnAll, mnPos, nNon, sum)); + } + } + __syncthreads(); + if (lane == 0 && (!rs || warp == 0)) + { + uint32_t mnAll = ~0u; + uint32_t mnPos = ~0u; + uint32_t nNon = 0; + uint32_t sum = 0; +#pragma unroll + for (int x = 0; x < kWarps; ++x) + { + uint4 const v = lds128(sRed + 16 * (kTok * x + t)); + mnAll = min(mnAll, v.x); + mnPos = min(mnPos, v.y); + nNon += v.z; + sum += v.w; + } + uint32_t const Tr = max(nNon <= 7 && mnPos != ~0u ? mnPos : mnAll, 1u); + uint32_t const nPos = static_cast(kCtas) - nNon; + uint32_t const est = 2 * nPos >= static_cast(kCtas) && !(GLM52_FLAGS(ctx) & kFlagTopkOneList) + ? kEmpty + ((sum / nPos) << 8) + : 0u; + uint32_t const T2r = est > Tr + kTightGap ? est - kTightGap : Tr; + stU64(tw + 2 * t, + (static_cast(T2r == kEmpty ? kBndZero : T2r) << 32) | (Tr == kEmpty ? kBndZero : Tr)); + st32(tw + 2 * kTok + t, est > Tr && est != kEmpty ? est : 1u); // 1: no estimate (below every bound) + } + subStamp(ctx, 77); + } + if (warp == 0) + { + bool const want = rs ? lane == t : lane < kTok; + uint64_t v = want ? ldPollU64(tw + 2 * lane) : 0ull; + uint64_t t0 = 0; + while (!warpAll(!want || static_cast(v) != kEmpty)) + { + spinCheck(ctx, t0, 167, lane, 0); + if (want && static_cast(v) == kEmpty) + { + v = ldPollU64(tw + 2 * lane); + } + } + if (want) + { + uint32_t const lo = static_cast(v); + uint32_t const hi = static_cast(v >> 32); + sts32(sMin + 768 + 4 * lane, lo == kBndZero ? kEmpty : lo); + sts32(sMin + 896 + 4 * lane, hi == kBndZero ? kEmpty : hi); + } + } + } +#endif + __syncthreads(); + // kFlagCandWarps (row shard): every warp counts and writes the candidates of its share of the lanes' keys. + bool const split = (kSpecializedStaticFlags & kFlagCandWarps) != 0 && rs; + if (!active && !split) + { + return; + } + int const mBeg = split ? warp * nPer / kWarps : 0; + int const mEnd = split ? (warp + 1) * nPer / kWarps : nPer; + uint32_t const T = lds32(sMin + 768 + 4 * t); + uint32_t const T2 = lds32(sMin + 896 + 4 * t); +#if GLM52_TOPK_UNION + uint32_t const hsh = lds32(sMin + 864 + 4 * t); // the bin width the T CTA chose (log2) +#else + uint32_t const hsh = kHistShift; +#endif + uint32_t const hspan = static_cast(kHistBins) << hsh; + subStamp(ctx, 44); + // The lane's keys of each class (A: > T2, == T2; B: in (T, T2), == T when T2 > T), their exclusive prefixes over + // the lanes and the warp's totals (two classes per 32-bit word, sums <= 768): each lane then writes its own + // candidates (slots per list: the keys above its threshold lane by lane, then the first ties lane by lane), with no + // warp collective per key (a ballot pair per key, the keys of a lane ~3% candidates, took ~3 us). The count keeps + // the indices m of the lane's last eight keys >= T (8 bits each): with at most eight (a lane has ~0.7 on average), + // the writes visit only those. (A second pass over all keys and their stores took ~1.4 us; with four kept, ~15 % of + // the CTAs still had a lane in that pass, and the last CTA sets the selectors' start.) + bool const two = T2 > T; + bool const hist = !(GLM52_FLAGS(ctx) & kFlagTopkNoHist); + uint32_t* const hs = gc + kCIdxHist + t * kHistBins; + uint32_t cntA = 0; + uint32_t cntB = 0; + uint32_t cntH = 0; + uint64_t rec = 0; + // The keys for the histogram are only counted here (their adds go with their candidates' stores below: in this + // loop they cost +1.35 us per slice even with the selector's window on the critical path, mk-v144a). +#if GLM52_CAND_FILTER_ROLL +#pragma unroll 1 +#else +#pragma unroll 4 +#endif + for (int m = mBeg; m < mEnd; ++m) + { + uint32_t const v = selKeyAt(a0, tL, lane + 32 * m); + cntA += v > T2 ? 1u : v == T2 ? 1u << 16 : 0u; + cntB += v > T && v < T2 ? 1u : two && v == T ? 1u << 16 : 0u; + rec = v >= T ? (rec << 8) | static_cast(m) : rec; + cntH += hist && v > T2 && v - T2 < hspan ? 1u : 0u; + } + uint32_t preA = cntA; + uint32_t preB = cntB; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) + { + uint32_t const va = __shfl_up_sync(0xffffffffu, preA, off); + uint32_t const vb = __shfl_up_sync(0xffffffffu, preB, off); + preA += lane >= off ? va : 0u; + preB += lane >= off ? vb : 0u; + } + uint32_t totA = __shfl_sync(0xffffffffu, preA, 31); + uint32_t totB = __shfl_sync(0xffffffffu, preB, 31); + preA -= cntA; + preB -= cntB; + uint32_t cntHw = hist ? __reduce_add_sync(0xffffffffu, cntH) : 0u; + uint32_t const scr = sMin + 1024; // kFlagCandWarps: the warps' totals [0, 24), the CTA's results [24, 30) + if (split) + { + // The warps' totals; the CTA's (and the earlier warps' share of them) after one barrier. + if (lane == 0) + { + sts32(scr + 4 * warp, totA); + sts32(scr + 4 * (8 + warp), totB); + sts32(scr + 4 * (16 + warp), cntHw); + } + __syncthreads(); + uint32_t pA = 0; + uint32_t pB = 0; + totA = 0; + totB = 0; + cntHw = 0; +#pragma unroll + for (int w = 0; w < kWarps; ++w) + { + uint32_t const a = lds32(scr + 4 * w); + uint32_t const b = lds32(scr + 4 * (8 + w)); + pA += w < warp ? a : 0u; + pB += w < warp ? b : 0u; + totA += a; + totB += b; + cntHw += lds32(scr + 4 * (16 + w)); + } + preA += pA; + preB += pB; + } + uint32_t const gtA = totA & 0xffffu; + uint32_t const eqA = totA >> 16; + uint32_t const gtB = totB & 0xffffu; + uint32_t const eqB = totB >> 16; + uint32_t tieA = min(eqA, kTieFree); + uint32_t tieB = min(eqB, kTieFree); + // One lane per CTA (the row shard's warp 0, or every warp of its row without it) takes the extra ties, the + // histogram's tie add and the slot atomic. + bool const leadW = !split || warp == 0; + if (leadW && (eqA > kTieFree || eqB > kTieFree)) + { + uint32_t extraA = 0; + uint32_t extraB = 0; + if (lane == 0 && eqA > kTieFree) + { + uint32_t const used = atomicAdd(gc + kCIdxTies + t, eqA - kTieFree); + extraA = used < static_cast(kTopK) ? min(eqA - kTieFree, kTopK - used) : 0u; + } + if (lane == 0 && eqB > kTieFree) + { + uint32_t const used = atomicAdd(gc + kCIdxTiesB + t, eqB - kTieFree); + extraB = used < static_cast(kTopK) ? min(eqB - kTieFree, kTopK - used) : 0u; + } + tieA += __shfl_sync(0xffffffffu, extraA, 0); + tieB += __shfl_sync(0xffffffffu, extraB, 0); + } + // The CTA's keys in the histogram: the ones counted above (added with their stores) and its keys equal to T2 taken + // (bin 0). Their count rides on the slot atomic: the selector waits until the bins' sum equals the row's (every add + // landed). + uint32_t const totH = hist ? cntHw + tieA : 0u; + if (hist && leadW && lane == 0 && tieA != 0) + { + redRelaxedAdd(hs, tieA); + } + subStamp(ctx, 67); + uint32_t baseA = 0; + uint32_t baseB = 0; + if (leadW && lane == 0) + { + unsigned long long const add + = (static_cast((totH << 18) | (gtB + tieB)) << 32) | ((1u << 24) | (gtA + tieA)); + unsigned long long const old = atomicAdd(reinterpret_cast(gc + kCIdxCand + 2 * t), add); + baseA = static_cast(old) & 0xffffffu; + baseB = static_cast(old >> 32) & 0x3ffffu; + if (split) + { + sts32(scr + 4 * 24, baseA); + sts32(scr + 4 * 25, baseB); + sts32(scr + 4 * 26, tieA); + sts32(scr + 4 * 27, tieB); + } + } + if (split) + { + __syncthreads(); + baseA = lds32(scr + 4 * 24); + baseB = lds32(scr + 4 * 25); + tieA = lds32(scr + 4 * 26); + tieB = lds32(scr + 4 * 27); + } + else + { + baseA = __shfl_sync(0xffffffffu, baseA, 0); + baseB = __shfl_sync(0xffffffffu, baseB, 0); + } + subStamp(ctx, 68); + if (cntA + cntB == 0) + { + return; + } + uint64_t* const rowA = cand + static_cast(t) * kCandCap; + uint32_t hiA = baseA + (preA & 0xffffu); + uint32_t tA = preA >> 16; + uint32_t hiB = baseB + (preB & 0xffffu); + uint32_t tB = preB >> 16; + auto put = [&](int m) + { + uint32_t const v = selKeyAt(a0, tL, lane + 32 * m); + uint32_t slot = ~0u; + if (v > T2) + { + slot = hiA++; + if (hist && v - T2 < hspan) + { + redRelaxedAdd(hs + ((v - T2) >> hsh), 1u); + } + } + else if (v == T2) + { + slot = tA < tieA ? baseA + gtA + tA : ~0u; + ++tA; + } + else if (v > T) + { + slot = kCandList + hiB++; + } + else + { + slot = tB < tieB ? kCandList + baseB + gtB + tB : ~0u; + ++tB; + } + // Each list holds every key of the row (kCandList). + if (slot < (v >= T2 ? kCandList : kCandCap)) + { + // Local key k: tile k / 128, page 2 (k / 128) + (k % 128) / 64 of the CTA's, row k % 64. + int const k = lane + 32 * m; + int const page = 2 * (k >> 7) + ((k & 127) >> 6); + uint32_t const pos = static_cast( + idxPageOf(cta, page, (L + kIdxPageKeys - 1) / kIdxPageKeys) * kIdxPageKeys + (k & 63)); + stU64(rowA + slot, (static_cast(pos) << 32) | v); + } + }; + int const nRec = static_cast((cntA & 0xffffu) + (cntA >> 16) + (cntB & 0xffffu) + (cntB >> 16)); + if (nRec <= 8) + { +#pragma unroll 1 + for (int i = 0; i < nRec; ++i) + { + put(static_cast((rec >> (8 * i)) & 0xffu)); + } + } + else + { +#if GLM52_CAND_FILTER_ROLL +#pragma unroll 1 +#else +#pragma unroll 4 +#endif + for (int m = mBeg; m < mEnd; ++m) + { + if (selKeyAt(a0, tL, lane + 32 * m) >= T) + { + put(m); + } + } + } +} + +// A row's top need0 from its n0 > kSelSmall candidates in lst0 ((key, position) pairs, keys >= lo, the largest hiKey; +// not modified; lst1: kSelSmall pairs of scratch; hist: 2048 words; sSel: 32 words). Each round scans lst0 restricted +// to the window +// [lo, lo + range] (all of it at first): a 2048-bin histogram of key - lo (bins as wide as the window needs), the keys +// above the crossing bin to the output (slots by a block-wide scan of the threads' counts), the crossing bin as the +// next window. Once the window holds at most kSelSmall keys they are gathered into lst1 (in that round's writes) and +// ranked against each other (larger key first, then the earlier index); with single-key bins the crossing key's ties +// are taken directly. Outputs: the positions into the shared int32 [need0] at outS (written out coalesced by the +// caller; per-thread slots in global memory were scattered stores). (Adding once per distinct bin of a warp with +// match.any made the histogram pass 2x slower.) +__device__ __noinline__ void idxSelect(Ctx const& ctx, uint2 const* lst0, uint2* lst1, uint32_t* hist, uint32_t* sSel, + int n0, int need0, uint32_t lo, uint32_t hiKey, uint32_t outS) +{ + int const tid = threadIdx.x; + uint32_t const l0 = smemU32(lst0); + uint32_t const l1 = smemU32(lst1); + uint32_t const hs = smemU32(hist); + uint32_t range = hiKey > lo ? hiKey - lo : 0u; + int need = need0; + int emitted = 0; + int nWin = n0; + int nSmall = -1; +#pragma unroll 1 + while (nWin > kSelSmall) + { + int const s = range < 2048u ? 0 : 21 - __clz(static_cast(range)); +#pragma unroll 1 + for (int b = tid; b < 2048 / 4; b += kThreads) + { + reinterpret_cast(hist)[b] = make_uint4(0u, 0u, 0u, 0u); + } + __syncthreads(); +#pragma unroll 4 + for (int i = tid; i < n0; i += kThreads) + { + uint32_t const d = lds32(l0 + 8 * i) - lo; + if (d <= range) + { + redsAdd(hs + 4 * (d >> s), 1u); + } + } + __syncthreads(); + int bstar; + uint32_t above; + uint32_t inBin; + blockCrossBin(hist, static_cast(need), sSel, bstar, above, inBin); + if (emitted == 0) + { + subStamp(ctx, 62); + } + int const ties = need - static_cast(above); + bool const gather = s != 0 && inBin <= static_cast(kSelSmall); + uint32_t nHi = 0; + uint32_t nEq = 0; +#pragma unroll 4 + for (int i = tid; i < n0; i += kThreads) + { + uint32_t const d = lds32(l0 + 8 * i) - lo; + if (d <= range) + { + int const b = static_cast(d >> s); + nHi += b > bstar ? 1u : 0u; + nEq += b == bstar ? 1u : 0u; + } + } + uint32_t baseH; + uint32_t baseE; + blockScan2(nHi, nEq, smemU32(sSel + 16), baseH, baseE); + uint32_t const o = outS + 4 * emitted; +#pragma unroll 4 + for (int i = tid; i < n0; i += kThreads) + { + uint2 const e = lds64(l0 + 8 * i); + uint32_t const d = e.x - lo; + if (d <= range) + { + int const b = static_cast(d >> s); + if (b > bstar) + { + sts32(o + 4 * baseH++, e.y); + } + else if (b == bstar) + { + if (s == 0) + { + if (static_cast(baseE) < ties) + { + sts32(o + 4 * (above + baseE), e.y); + } + } + else if (gather) + { + sts64(l1 + 8 * baseE, e); + } + ++baseE; + } + } + } + __syncthreads(); + if (emitted == 0) + { + subStamp(ctx, 63); + } + emitted += static_cast(above); + need = ties; + if (s == 0) + { + nWin = 0; + break; + } + lo += static_cast(bstar) << s; + range = (1u << s) - 1u; + nWin = static_cast(inBin); + if (gather) + { + nSmall = nWin; + break; + } + } + // n0 > kSelSmall: the rounds end gathered into lst1, or with single-key bins (nothing left to rank). + uint32_t const src = l1; + int const n = nSmall >= 0 ? nSmall : 0; +#pragma unroll 1 + for (int j = tid; j < n; j += kThreads) + { + uint2 const e = lds64(src + 8 * j); + int rank = 0; +#pragma unroll 4 + for (int i = 0; i < n; ++i) + { + uint32_t const k = lds32(src + 8 * i); + rank += (k > e.x || (k == e.x && i < j)) ? 1 : 0; + } + if (rank < need) + { + sts32(outS + 4 * (emitted + rank), e.y); + } + } +} + +static_assert( + kIdxSelBase + kTok == kCtas && kIdxSelBase >= kIdxQUnits, "selector CTAs: the last 8, past the IdxQ ones"); +static_assert(kIdxQUnits * kIdxQUnitRows == kIdxHeads * kIdxDim, "IdxQ units"); +constexpr int kSelOut = kCandSmem * 8 + kSelSmall * 8 + 2048 * 4; // the selection's output, int32 [2048] +static_assert(32 * kThreads * 8 + 2 * 32 * kThreads * 4 <= kSelOut && 32 * kThreads > kSelSmall, + "selector: pairs, then packed keys and positions; idxSelect's n0"); +static_assert(kSelOut + kTopK * 4 <= kSelBytes, "selection buffers"); +static_assert(1024 + 64 + 8 * kSelWin <= kSmemRedBytes, "idxSelectBin: the bin's keys"); + +// The selector of a row with nc <= 32 x 256 candidates (keys >= lo at shared address sKey, zero past nc to a multiple +// of four; their positions at sPos): X = the largest value with at least need0 keys >= X, by +// a search over [lo, the largest key], or over the window win from list A's histogram (win.z != 0) (a step: each +// thread's count of keys >= a probe, one block-wide sum; the first probes est and est +- kGallop, kGallop doubling, +// until both sides are known, when est is in the range, then bisection) until the window holds at most kSelWin keys (~8 +// steps instead of ~25), then by ranking the window's keys against each other; the keys > X and need0 - count(> X) of +// the keys == X (any) are the row's selection, staged in shared memory and written to topkOut coalesced (the -1 fill +// past the row's valid keys included). Thread tid holds the keys of groups tid + 256 j (four keys each) in registers. +// The code is kept small: it runs once per leader launch on 8 SMs from cold instruction caches, where straight-line +// passes over the 32 registers (13.5 KB of code) took 11 us against 7 warm. So the counts of keys > X and == X reuse +// the search's count (probes X + 1 and X), and the scatter is a rolled loop. sSel: shared scratch, 64 words, then the +// staged output (int32 [2048]) from sSel + 512. +__device__ __noinline__ void idxSelectRow(Ctx const& ctx, int nc, int need0, uint32_t lo, uint32_t est, uint4 win, + int* topkOut, int* arenaOut, int* mcOut, uint32_t sSel, uint32_t sKey, uint32_t sPos) +{ + int const tid = threadIdx.x; + int const lane = tid & 31; + int const warp = tid >> 5; + int const nq = (nc + 3) >> 2; + int const ng = (nq + kThreads - 1) / kThreads; // the thread's groups of four that can hold keys (the rest are zero) + uint32_t mx = 0; + uint32_t k[32]; +#pragma unroll + for (int j = 0; j < 8; ++j) + { + int const q = tid + kThreads * j; + uint4 const g = q < nq ? lds128(sKey + 16 * q) : make_uint4(0u, 0u, 0u, 0u); + k[4 * j] = g.x; + k[4 * j + 1] = g.y; + k[4 * j + 2] = g.z; + k[4 * j + 3] = g.w; + mx = max(mx, max(max(g.x, g.y), max(g.z, g.w))); + } + mx = __reduce_max_sync(0xffffffffu, mx); + if (lane == 0) + { + sts32(sSel + 4 * (16 + warp), mx); + } + __syncthreads(); +#pragma unroll + for (int x = 0; x < kWarps; ++x) + { + mx = max(mx, lds32(sSel + 4 * (16 + x))); + } + subStamp(ctx, 61); + // Search invariant: count(>= a) = ca >= need0 > cb = count(> b), and the answer is in [a, b]. Stage 0 bisects + // (probe mid), stage 3 ranks the window's ca - cb <= kSelWin keys (gathered into the output staging area), stage 1 + // takes the thread's count of keys >= X + 1 (= > X), stage 2 its count of keys >= X; X + 1 does not wrap (order + // keys of finite logits stay below 0xff800000). + uint32_t a = lo; + uint32_t b = max(mx, lo); + uint32_t ca = static_cast(nc); + uint32_t cb = 0; + if (win.z != 0) + { + // The first window from list A's histogram (the row shard): count(>= win.x) = win.z >= need0 > win.w = + // count(> win.y); win.y = 0: up to the largest key. + a = win.x; + b = win.y != 0 ? win.y : b; + ca = win.z; + cb = win.w; + } + int stage = a < b ? (ca - cb > static_cast(kSelWin) ? 0 : 3) : 1; + // g: 3 = est is the next probe, 1 / 2 = galloping up / down from it, 0 = bisection. + int g = est > a && est <= b ? 3 : 0; + uint32_t gd = kGallop; + uint32_t probe = stage == 0 ? (g != 0 ? est : a + (b - a) / 2 + 1) : a + 1; + int parity = 0; + int steps = 0; // bisection steps, stored once (stamped runs; a global add per step stalled thread 0) + uint32_t nHi = 0; + uint32_t nEq = 0; + uint32_t const sWin = sSel + 512; + uint32_t const sWinN = sSel + 4 * 40; +#pragma unroll 1 + while (true) + { + if (stage == 3) + { + subStamp(ctx, 66); + // The window's keys gather at sWin, zero past them to a multiple of four (zero is below every key). + if (tid == 0) + { + sts32(sWinN, 0u); + } +#pragma unroll 1 + for (int i = tid; i < kSelWin + 4; i += kThreads) + { + sts32(sWin + 4 * i, 0u); + } + __syncthreads(); +#pragma unroll 1 + for (int q = tid; q < nq; q += kThreads) + { + uint4 const g = lds128(sKey + 16 * q); + uint32_t const w[4] = {g.x, g.y, g.z, g.w}; +#pragma unroll + for (int e = 0; e < 4; ++e) + { + if (w[e] >= a && w[e] <= b) + { + sts32(sWin + 4 * atomsAdd(sWinN, 1u), w[e]); + } + } + } + __syncthreads(); + int const nW = static_cast(lds32(sWinN)); + // The largest window key v with cb + (window keys >= v) >= need0. + uint32_t best = 0; +#pragma unroll 1 + for (int j = tid; j < nW; j += kThreads) + { + uint32_t const v = lds32(sWin + 4 * j); + uint32_t ge = 0; +#pragma unroll 2 + for (int i = 0; i < nW; i += 4) + { + uint4 const w = lds128(sWin + 4 * i); + ge += (w.x >= v ? 1u : 0u) + (w.y >= v ? 1u : 0u) + (w.z >= v ? 1u : 0u) + (w.w >= v ? 1u : 0u); + } + best = cb + ge >= static_cast(need0) ? max(best, v) : best; + } + best = __reduce_max_sync(0xffffffffu, best); + if (lane == 0) + { + sts32(sSel + 4 * (8 * parity + warp), best); + } + __syncthreads(); + best = 0; +#pragma unroll + for (int x = 0; x < kWarps; ++x) + { + best = max(best, lds32(sSel + 4 * (8 * parity + x))); + } + parity ^= 1; + a = best; + stage = 1; + probe = a + 1; + continue; + } + uint32_t c0 = 0; + uint32_t c1 = 0; + uint32_t c2 = 0; + uint32_t c3 = 0; +#pragma unroll + for (int j = 0; j < 32; j += 4) + { + if (j < 4 * ng) + { + c0 += k[j] >= probe ? 1u : 0u; + c1 += k[j + 1] >= probe ? 1u : 0u; + c2 += k[j + 2] >= probe ? 1u : 0u; + c3 += k[j + 3] >= probe ? 1u : 0u; + } + } + uint32_t const c = (c0 + c1) + (c2 + c3); + if (stage == 2) + { + nEq = c - nHi; + break; + } + if (stage == 1) + { + nHi = c; + stage = 2; + probe = a; + continue; + } + uint32_t const cw = __reduce_add_sync(0xffffffffu, c); + if (lane == 0) + { + sts32(sSel + 4 * (8 * parity + warp), cw); + } + __syncthreads(); + uint32_t tot = 0; +#pragma unroll + for (int x = 0; x < kWarps; ++x) + { + tot += lds32(sSel + 4 * (8 * parity + x)); + } + parity ^= 1; + ++steps; + if (steps <= 5) + { + subStamp(ctx, 90 + steps); // stamped runs: the first five steps' ends (sub-stamps 91-95) + } + bool const up = tot >= static_cast(need0); + if (up) + { + a = probe; + ca = tot; + } + else + { + b = probe - 1; + cb = tot; + } + g = g == 3 ? (up ? 1 : 2) : (g == 1 && !up) || (g == 2 && up) ? 0 : g; + stage = a < b ? (ca - cb > static_cast(kSelWin) ? 0 : 3) : 1; + if (stage == 0 && g != 0 && b - a > gd) + { + probe = g == 1 ? a + gd : b + 1 - gd; + gd <<= 1; + } + else + { + g = 0; + probe = stage == 0 ? a + (b - a) / 2 + 1 : a + 1; + } + } + uint32_t const X = a; + subStamp(ctx, 62); + if (GLM52_STAMPS(ctx) != nullptr && tid == 0) + { + GLM52_STAMPS(ctx)[kMaxRoles * 4 + 4 + 90] = static_cast(steps); + } + uint32_t baseH; + uint32_t baseE; + blockScan2(nHi, nEq, sSel + 4 * 24, baseH, baseE); + uint32_t totHi = 0; +#pragma unroll + for (int x = 0; x < kWarps; ++x) + { + totHi += lds32(sSel + 4 * (24 + x)); + } + uint32_t const ties = static_cast(need0) > totHi ? static_cast(need0) - totHi : 0u; + uint32_t const sOut = sSel + 512; + // The thread's keys in the order counted above: groups tid + 256 j, four keys and their positions per 16-byte load + // (keys past nc are zero, below X). +#pragma unroll 1 + for (int q = tid; q < nq; q += kThreads) + { + uint4 const kk = lds128(sKey + 16 * q); + uint4 const pp = lds128(sPos + 16 * q); + uint32_t const key[4] = {kk.x, kk.y, kk.z, kk.w}; + uint32_t const pos[4] = {pp.x, pp.y, pp.z, pp.w}; +#pragma unroll + for (int e = 0; e < 4; ++e) + { + if (key[e] > X) + { + sts32(sOut + 4 * baseH++, pos[e]); + } + else if (key[e] == X) + { + if (baseE < ties) + { + sts32(sOut + 4 * (totHi + baseE), pos[e]); + } + ++baseE; + } + } + } + subStamp(ctx, 63); + __syncthreads(); + int const filled = min(nc, need0); +#pragma unroll + for (int v = 0; v < kTopK / 4 / kThreads; ++v) + { + int const i = 4 * (tid + kThreads * v); + uint4 o = lds128(sOut + 4 * i); + o.x = i < filled ? o.x : ~0u; + o.y = i + 1 < filled ? o.y : ~0u; + o.z = i + 2 < filled ? o.z : ~0u; + o.w = i + 3 < filled ? o.w : ~0u; + if (mcOut != nullptr) + { + st128(mcOut + i, o); // the top-k exchange first: the other ranks wait on it + } + if (topkOut != nullptr) + { + st128(topkOut + i, o); + st128(arenaOut + i, o); + } + } +} + +// The selector of a row with more than 32 x 256 candidates ((key, position) pairs at shared address a0 = s0): its +// largest key, then the histogram rounds of idxSelect; the result out of shared memory. +__device__ __noinline__ void idxSelectLarge(Ctx const& ctx, int nc, int need0, uint32_t lo, uint8_t* s0, uint32_t a0, + uint32_t* sSel, int* topkOut, int* arenaOut, int* mcOut) +{ + int const tid = threadIdx.x; + int const lane = tid & 31; + int const warp = tid >> 5; + uint32_t mx = 0; + for (int i = tid; i < nc; i += kThreads) + { + mx = max(mx, lds32(a0 + 8 * i)); + } + mx = __reduce_max_sync(0xffffffffu, mx); + uint32_t const sMx = smemU32(sSel) + 4 * 48; + if (lane == 0) + { + sts32(sMx + 4 * warp, mx); + } + __syncthreads(); +#pragma unroll + for (int x = 0; x < kWarps; ++x) + { + mx = max(mx, lds32(sMx + 4 * x)); + } + uint32_t const outS = a0 + kSelOut; + idxSelect(ctx, reinterpret_cast(s0), reinterpret_cast(s0 + kCandSmem * 8), + reinterpret_cast(s0 + kCandSmem * 8 + kSelSmall * 8), sSel, nc, need0, lo, mx, outS); + __syncthreads(); + int const filled = min(nc, need0); +#pragma unroll 4 + for (int i = tid; i < kTopK; i += kThreads) + { + int const v = i < filled ? static_cast(lds32(outS + 4 * i)) : -1; + if (mcOut != nullptr) + { + mcOut[i] = v; + } + if (topkOut != nullptr) + { + topkOut[i] = v; + arenaOut[i] = v; + } + } +} + +// The selector when list A's histogram puts the row's need0-th key in a bin [a, b] of at most kSelWin keys +// (count(> b) = cb, count(>= a) = ca; list A's nc (key, position) pairs at shared address a0, copied from list, or with +// fromList read from list by the pass itself, the next four pairs per lane in flight while it handles four): one +// pass, four pairs per lane at a time, writes the positions of the keys > b (the row's cb surest) straight into the +// row's output out (and out2 unless null: the exchange row, or the top-k buffer and the arena), slots from one +// warp-aggregated shared counter per four pairs (keys > b | the bin's << 16), and gathers the bin's keys; a pair the +// copy read before it landed (key 0) is polled from list. Warp 0 then ranks the bin (larger key first, then the +// earlier entry) and writes the need0 - cb largest after them, while the other warps clear list for the next leader +// launch (every slot has landed). No search, no packing. (Staging the row in shared memory for 16-byte stores at the +// end was 4.2 us slower per slice, mk-v125a: the attention then waits for the whole row.) +__device__ __noinline__ void idxSelectBin(Ctx const& ctx, int nc, int need0, uint4 win, uint32_t a0, uint32_t sSel, + int* out, int* out2, uint64_t* list, bool fromList, uint2 const* pre) +{ + int const tid = threadIdx.x; + int const lane = tid & 31; + int const warp = tid >> 5; + uint32_t const a = win.x; + uint32_t const b = win.y; + uint32_t const cb = win.w; + uint32_t const sWinK = sSel + 64; + uint32_t const sWinP = sWinK + 4 * kSelWin; + if (tid == 0) + { + sts32(sSel, 0u); + } + __syncthreads(); + subStamp(ctx, 61); + unsigned const below = (1u << lane) - 1u; + auto load = [&](int i0, uint2(&kp)[4]) + { +#pragma unroll + for (int k = 0; k < 4; ++k) + { + int const i = i0 + lane + 32 * k; + if (i >= nc) + { + kp[k] = make_uint2(0u, 0u); + } + else if (fromList && (GLM52_FLAGS(ctx) & kFlagBinVolatile)) + { + uint64_t const v = ldPollU64(list + i); + kp[k] = make_uint2(static_cast(v), static_cast(v >> 32)); + } + else if (fromList) + { + // Plain L2 loads (ld.global.cg): free to overlap the processing of the four before (volatile loads + // kept the pass at ~0.35 us per four pairs); a pair read before it landed is polled below. + kp[k] = __ldcg(reinterpret_cast(list + i)); + } + else + { + kp[k] = lds64(a0 + 8 * i); + } + } + }; + uint2 kp[4]; + uint2 nxt[4] = {make_uint2(0u, 0u), make_uint2(0u, 0u), make_uint2(0u, 0u), make_uint2(0u, 0u)}; + if (pre != nullptr) + { + // The pass's first four pairs per lane, loaded by the caller before the window (in flight through its polls). +#pragma unroll + for (int k = 0; k < 4; ++k) + { + kp[k] = pre[k]; + } + } + else + { + load(128 * warp, kp); + } +#pragma unroll 1 + for (int i0 = 128 * warp; i0 < nc; i0 += 128 * kWarps) + { + if (i0 + 128 * kWarps < nc) + { + load(i0 + 128 * kWarps, nxt); + } +#pragma unroll + for (int k = 0; k < 4; ++k) + { + int const i = i0 + lane + 32 * k; + uint64_t t0 = 0; + while (i < nc && kp[k].x == 0u) + { + uint64_t const v = ldPollU64(list + i); + kp[k] = make_uint2(static_cast(v), static_cast(v >> 32)); + if (kp[k].x == 0u) + { + spinCheck(ctx, t0, 173, i, 0); + } + } + } + unsigned ms[4]; + unsigned mb[4]; + uint32_t n = 0; +#pragma unroll + for (int k = 0; k < 4; ++k) + { + ms[k] = __ballot_sync(0xffffffffu, kp[k].x > b); + mb[k] = __ballot_sync(0xffffffffu, kp[k].x >= a && kp[k].x <= b); + n += static_cast(__popc(ms[k])) | (static_cast(__popc(mb[k])) << 16); + } + uint32_t base = 0; + if (lane == 0 && n != 0) + { + base = atomsAdd(sSel, n); + } + base = __shfl_sync(0xffffffffu, base, 0); + uint32_t bs = base & 0xffffu; + uint32_t bb = base >> 16; +#pragma unroll + for (int k = 0; k < 4; ++k) + { + uint32_t const s = bs + static_cast(__popc(ms[k] & below)); + uint32_t const w = bb + static_cast(__popc(mb[k] & below)); + if (kp[k].x > b && s < cb) + { + st32(out + s, kp[k].y); + if (out2 != nullptr) + { + st32(out2 + s, kp[k].y); + } + } + else if (kp[k].x >= a && kp[k].x <= b && w < static_cast(kSelWin)) + { + sts32(sWinK + 4 * w, kp[k].x); + sts32(sWinP + 4 * w, kp[k].y); + } + bs += static_cast(__popc(ms[k])); + bb += static_cast(__popc(mb[k])); + } +#pragma unroll + for (int k = 0; k < 4; ++k) + { + kp[k] = nxt[k]; + } + } + __syncthreads(); + subStamp(ctx, 62); + uint32_t const cnt = lds32(sSel); + uint32_t const nSure = cnt & 0xffffu; + uint32_t const nBin = cnt >> 16; + if (nSure != cb || nBin != win.z - cb) + { + fail(ctx, 177, static_cast(nSure), static_cast(nBin)); // the histogram and the list disagree + } + if (warp == 0) + { + int const want = need0 - static_cast(cb); +#pragma unroll 1 + for (int j = lane; j < static_cast(nBin); j += 32) + { + uint32_t const v = lds32(sWinK + 4 * j); + int r = 0; +#pragma unroll 1 + for (int x = 0; x < static_cast(nBin); ++x) + { + uint32_t const u = lds32(sWinK + 4 * x); + r += u > v || (u == v && x < j) ? 1 : 0; + } + if (r < want) + { + uint32_t const pos = lds32(sWinP + 4 * j); + st32(out + cb + r, pos); + if (out2 != nullptr) + { + st32(out2 + cb + r, pos); + } + } + } + } + else + { + // The list back to zero for the next leader launch (two pairs per store). +#pragma unroll 1 + for (int i = 2 * (tid - 32); i < nc; i += 2 * (kThreads - 32)) + { + st128(list + i, make_uint4(0u, 0u, 0u, 0u)); + } + } + subStamp(ctx, 63); + // Past the row's valid keys (short rows only): -1. +#pragma unroll 1 + for (int i = need0 + tid; i < kTopK; i += kThreads) + { + st32(out + i, ~0u); + if (out2 != nullptr) + { + st32(out2 + i, ~0u); + } + } +} + +// Optional row-shard bin pass. CTA 204 publishes either fallback (state 1) or a release-ordered descriptor (state 2). +// CTAs 204..211 own disjoint contiguous ranges of 128 pairs; each pair is polled before use and before its clear. +// A single packed global allocator assigns disjoint sure-output and boundary slots. The CTA completion releases +// order all boundary stores before CTA 204's acquire; only CTA 204 ranks the boundary and counts the row done. +// The new counter tail is cleared with the next copy, after its previous readers finished, just like the histogram. +__device__ __noinline__ void idxSelectBinParallel(Ctx const& ctx, int need0, uint64_t* list, int* out) +{ + int const tid = threadIdx.x; + int const lane = tid & 31; + int const warp = tid >> 5; + int const part = static_cast(blockIdx.x) - kIdxSelBase; + uint32_t* const gc = ctx.counters; + uint32_t const sSel = ctx.smemBase + kSmemRed + 1024; + if (tid == 0) + { + uint64_t t0 = 0; + uint32_t state; + while ((state = ldAcquire(gc + kCIdxParState)) == 0u) + { + spinCheck(ctx, t0, 178, part, 0); + } + sts32(sSel, state); + } + __syncthreads(); + if (lds32(sSel) == 1u) + { + return; // the original selector owns every candidate and output; helpers have read neither + } + int const nc = static_cast(ldPoll32(gc + kCIdxParN)); + uint4 const win = ldPoll128(gc + kCIdxParWin); + uint32_t const a = win.x; + uint32_t const b = win.y; + uint32_t const cb = win.w; + int const chunks = (nc + 127) / 128; + int const begin = 128 * (chunks * part / kTok); + int const end = min(nc, 128 * (chunks * (part + 1) / kTok)); + unsigned const below = (1u << lane) - 1u; + subStamp(ctx, 61); +#pragma unroll 1 + for (int i0 = begin + 128 * warp; i0 < end; i0 += 128 * kWarps) + { + uint2 kp[4]; +#pragma unroll + for (int k = 0; k < 4; ++k) + { + int const i = i0 + lane + 32 * k; + if (i >= end) + { + kp[k] = make_uint2(0u, 0u); + } + else if (GLM52_FLAGS(ctx) & kFlagBinVolatile) + { + uint64_t const v = ldPollU64(list + i); + kp[k] = make_uint2(static_cast(v), static_cast(v >> 32)); + } + else + { + kp[k] = __ldcg(reinterpret_cast(list + i)); + } + } +#pragma unroll + for (int k = 0; k < 4; ++k) + { + int const i = i0 + lane + 32 * k; + uint64_t t0 = 0; + while (i < end && kp[k].x == 0u) + { + uint64_t const v = ldPollU64(list + i); + kp[k] = make_uint2(static_cast(v), static_cast(v >> 32)); + if (kp[k].x == 0u) + { + spinCheck(ctx, t0, 173, i, part); + } + } + } + unsigned ms[4]; + unsigned mb[4]; + uint32_t n = 0; +#pragma unroll + for (int k = 0; k < 4; ++k) + { + ms[k] = __ballot_sync(0xffffffffu, kp[k].x > b); + mb[k] = __ballot_sync(0xffffffffu, kp[k].x >= a && kp[k].x <= b); + n += static_cast(__popc(ms[k])) | (static_cast(__popc(mb[k])) << 16); + } + uint32_t base = 0; + if (lane == 0 && n != 0u) + { + // cb < need0 <= 2048 and the boundary has <=64 entries: neither packed field can carry into the other. + base = atomicAdd(gc + kCIdxParCount, n); + } + base = __shfl_sync(0xffffffffu, base, 0); + uint32_t bs = base & 0xffffu; + uint32_t bb = base >> 16; +#pragma unroll + for (int k = 0; k < 4; ++k) + { + int const i = i0 + lane + 32 * k; + uint32_t const s = bs + static_cast(__popc(ms[k] & below)); + uint32_t const w = bb + static_cast(__popc(mb[k] & below)); + if (kp[k].x > b && s < cb) + { + st32(out + s, kp[k].y); // streaming multicast, so attention may consume completed slots now + } + else if (kp[k].x >= a && kp[k].x <= b && w < static_cast(kSelWin)) + { + st32(gc + kCIdxParKey + w, kp[k].x); + st32(gc + kCIdxParPos + w, kp[k].y); + } + if (i < end) + { + stU64(list + i, 0ull); // this unique owner has observed the producer's single-copy 64-bit store + } + bs += static_cast(__popc(ms[k])); + bb += static_cast(__popc(mb[k])); + } + } + signalCounter(ctx, kCIdxParDone); + if (part != 0) + { + return; + } + waitCounter(ctx, kCIdxParDone, kTok, 179); + subStamp(ctx, 62); + uint32_t const count = ldPoll32(gc + kCIdxParCount); + uint32_t const nSure = count & 0xffffu; + uint32_t const nBin = count >> 16; + if (nSure != cb || nBin != win.z - cb || nBin > static_cast(kSelWin)) + { + fail(ctx, 180, static_cast(nSure), static_cast(nBin)); + } + uint32_t const sWinK = sSel + 64; + uint32_t const sWinP = sWinK + 4 * kSelWin; + if (tid < static_cast(nBin)) + { + sts32(sWinK + 4 * tid, ldPoll32(gc + kCIdxParKey + tid)); + sts32(sWinP + 4 * tid, ldPoll32(gc + kCIdxParPos + tid)); + } + __syncthreads(); + if (warp == 0) + { + int const want = need0 - static_cast(cb); +#pragma unroll 1 + for (int j = lane; j < static_cast(nBin); j += 32) + { + uint32_t const v = lds32(sWinK + 4 * j); + int r = 0; +#pragma unroll 1 + for (int x = 0; x < static_cast(nBin); ++x) + { + uint32_t const u = lds32(sWinK + 4 * x); + r += u > v || (u == v && x < j) ? 1 : 0; + } + if (r < want) + { + st32(out + cb + r, lds32(sWinP + 4 * j)); + } + } + } +#pragma unroll 1 + for (int i = need0 + tid; i < kTopK; i += kThreads) + { + st32(out + i, ~0u); + } + subStamp(ctx, 63); +} + +// kFlagSelFast: the parallel bin pass without its descriptor and completion counters (each a release, ~1 us). Every +// selector CTA took the window itself; the boundary keys go out as single 64-bit (key, position) stores into slots +// zeroed with the counters, which CTA kIdxSelBase polls (it knows their number from the window). +__device__ __noinline__ void idxSelectBinParallelFast( + Ctx const& ctx, int need0, uint64_t* list, int* out, int const nc, uint4 const win) +{ + int const tid = threadIdx.x; + int const lane = tid & 31; + int const warp = tid >> 5; + int const part = static_cast(blockIdx.x) - kIdxSelBase; + uint32_t* const gc = ctx.counters; + uint32_t const sSel = ctx.smemBase + kSmemRed + 1024; + uint32_t const a = win.x; + uint32_t const b = win.y; + uint32_t const cb = win.w; + int const chunks = (nc + 127) / 128; + int const begin = 128 * (chunks * part / kTok); + int const end = min(nc, 128 * (chunks * (part + 1) / kTok)); + unsigned const below = (1u << lane) - 1u; + subStamp(ctx, 61); +#pragma unroll 1 + for (int i0 = begin + 128 * warp; i0 < end; i0 += 128 * kWarps) + { + uint2 kp[4]; +#pragma unroll + for (int k = 0; k < 4; ++k) + { + int const i = i0 + lane + 32 * k; + if (i >= end) + { + kp[k] = make_uint2(0u, 0u); + } + else if (GLM52_FLAGS(ctx) & kFlagBinVolatile) + { + uint64_t const v = ldPollU64(list + i); + kp[k] = make_uint2(static_cast(v), static_cast(v >> 32)); + } + else + { + kp[k] = __ldcg(reinterpret_cast(list + i)); + } + } +#pragma unroll + for (int k = 0; k < 4; ++k) + { + int const i = i0 + lane + 32 * k; + uint64_t t0 = 0; + while (i < end && kp[k].x == 0u) + { + uint64_t const v = ldPollU64(list + i); + kp[k] = make_uint2(static_cast(v), static_cast(v >> 32)); + if (kp[k].x == 0u) + { + spinCheck(ctx, t0, 173, i, part); + } + } + } + unsigned ms[4]; + unsigned mb[4]; + uint32_t n = 0; +#pragma unroll + for (int k = 0; k < 4; ++k) + { + ms[k] = __ballot_sync(0xffffffffu, kp[k].x > b); + mb[k] = __ballot_sync(0xffffffffu, kp[k].x >= a && kp[k].x <= b); + n += static_cast(__popc(ms[k])) | (static_cast(__popc(mb[k])) << 16); + } + uint32_t base = 0; + if (lane == 0 && n != 0u) + { + // cb < need0 <= 2048 and the boundary has <=64 entries: neither packed field can carry into the other. + base = atomicAdd(gc + kCIdxParCount, n); + } + base = __shfl_sync(0xffffffffu, base, 0); + uint32_t bs = base & 0xffffu; + uint32_t bb = base >> 16; +#pragma unroll + for (int k = 0; k < 4; ++k) + { + int const i = i0 + lane + 32 * k; + uint32_t const s = bs + static_cast(__popc(ms[k] & below)); + uint32_t const w = bb + static_cast(__popc(mb[k] & below)); + if (kp[k].x > b && s < cb) + { + st32(out + s, kp[k].y); // streaming multicast, so attention may consume completed slots now + } + else if (kp[k].x >= a && kp[k].x <= b && w < static_cast(kSelWin)) + { + stU64(reinterpret_cast(gc + kCIdxParKey) + w, + (static_cast(kp[k].y) << 32) | kp[k].x); + } + if (i < end) + { + stU64(list + i, 0ull); // this unique owner has observed the producer's single-copy 64-bit store + } + bs += static_cast(__popc(ms[k])); + bb += static_cast(__popc(mb[k])); + } + } + if (part != 0) + { + return; + } + // The window's boundary keys (as many as the histogram counted), each one 64-bit store (key != 0): polled. + uint32_t const nBin = win.z - cb; + if (nBin > static_cast(kSelWin)) + { + fail(ctx, 180, static_cast(cb), static_cast(nBin)); + } + subStamp(ctx, 62); + uint32_t const sWinK = sSel + 64; + uint32_t const sWinP = sWinK + 4 * kSelWin; + if (tid < static_cast(nBin)) + { + uint64_t const* const slot = reinterpret_cast(gc + kCIdxParKey) + tid; + uint64_t v = ldPollU64(slot); + uint64_t t0 = 0; + while (static_cast(v) == 0u) + { + spinCheck(ctx, t0, 179, tid, 0); + v = ldPollU64(slot); + } + sts32(sWinK + 4 * tid, static_cast(v)); + sts32(sWinP + 4 * tid, static_cast(v >> 32)); + } + __syncthreads(); + if (warp == 0) + { + int const want = need0 - static_cast(cb); +#pragma unroll 1 + for (int j = lane; j < static_cast(nBin); j += 32) + { + uint32_t const v = lds32(sWinK + 4 * j); + int r = 0; +#pragma unroll 1 + for (int x = 0; x < static_cast(nBin); ++x) + { + uint32_t const u = lds32(sWinK + 4 * x); + r += u > v || (u == v && x < j) ? 1 : 0; + } + if (r < want) + { + st32(out + cb + r, lds32(sWinP + 4 * j)); + } + } + } +#pragma unroll 1 + for (int i = need0 + tid; i < kTopK; i += kThreads) + { + st32(out + i, ~0u); + } + subStamp(ctx, 63); +} + +// The selector when a list it takes holds more than kCandSmemList candidates (more than the shared-memory selection +// copies: an estimate E far from the row's 2048th key, or keys spread unevenly over the CTAs): an exact selection over +// the nA + nB candidates in global memory (list A, then list B when taken; nB = 0 otherwise). The first pass polls +// every slot until its candidate landed (key 0: not yet) and counts the keys' bits 31-20 into a shared histogram at s0; +// two more passes count bits 19-8 and 7-0 of the keys in the bin found so far: X = the row's need0-th largest key. The +// keys > X and need0 - count(> X) of the keys == X (equal keys are interchangeable) are the row's selection, out in any +// order (the attention takes them as a set; every thread handles the same slots in every pass, so it zeroes its own for +// the next leader launch as it reads them last); entries past min(nc, need0) get -1 (up to outEnd). Without select (the +// row shard's odd rank) the lists are only polled and zeroed. +__device__ __noinline__ void idxSelectGlobal(Ctx const& ctx, uint64_t* listA, int nA, uint64_t* listB, int nB, + int need0, bool select, uint8_t* s0, int* topkOut, int* arenaOut, int* mcOut, int outEnd = kTopK) +{ + int const tid = threadIdx.x; + int const nc = nA + nB; + uint32_t* const hist = reinterpret_cast(s0); // 4096 bins + uint32_t* const scal = hist + 4096; + auto at = [&](int i) { return i < nA ? listA + i : listB + (i - nA); }; + bool const all = nc <= need0; // every candidate is selected + for (int i = tid; i < 4096; i += kThreads) + { + hist[i] = 0u; + } + if (tid == 0) + { + scal[10] = 0u; + scal[11] = 0u; + } + __syncthreads(); + // Pass 1: every slot landed; bits 31-20. +#pragma unroll 1 + for (int base = tid; base < nc; base += 8 * kThreads) + { + uint64_t v[8]; +#pragma unroll + for (int u = 0; u < 8; ++u) + { + int const i = base + u * kThreads; + v[u] = i < nc ? ldPollU64(at(i)) : 0ull; + } +#pragma unroll + for (int u = 0; u < 8; ++u) + { + int const i = base + u * kThreads; + if (i < nc) + { + uint64_t t0 = 0; + while (static_cast(v[u]) == 0u) + { + spinCheck(ctx, t0, 177, i, nc); + v[u] = ldPollU64(at(i)); + } + if (select && !all) + { + atomicAdd(&hist[static_cast(v[u]) >> 20], 1u); + } + } + } + } + __syncthreads(); + uint32_t X = 0; + uint32_t gt = 0; + if (select && !all) + { + uint32_t const need = static_cast(need0); + idxBinFromTop(hist, 4096, need, scal); + uint32_t const b1 = scal[8]; + uint32_t const above1 = scal[9]; + for (int i = tid; i < 4096; i += kThreads) + { + hist[i] = 0u; + } + __syncthreads(); + // Pass 2: bits 19-8 of the keys in bin b1. +#pragma unroll 4 + for (int i = tid; i < nc; i += kThreads) + { + uint32_t const key = static_cast(__ldcg(reinterpret_cast(at(i)))); + if ((key >> 20) == b1) + { + atomicAdd(&hist[(key >> 8) & 0xfffu], 1u); + } + } + __syncthreads(); + idxBinFromTop(hist, 4096, need - above1, scal); + uint32_t const b2 = scal[8]; + uint32_t const above2 = scal[9]; + uint32_t const hi24 = (b1 << 12) | b2; + for (int i = tid; i < 256; i += kThreads) + { + hist[i] = 0u; + } + __syncthreads(); + // Pass 3: bits 7-0 of the keys in bin hi24. +#pragma unroll 4 + for (int i = tid; i < nc; i += kThreads) + { + uint32_t const key = static_cast(__ldcg(reinterpret_cast(at(i)))); + if ((key >> 8) == hi24) + { + atomicAdd(&hist[key & 0xffu], 1u); + } + } + __syncthreads(); + idxBinFromTop(hist, 256, need - above1 - above2, scal); + X = (hi24 << 8) | scal[8]; + gt = above1 + above2 + scal[9]; + } + // Output: the keys > X (slots from 0), then need0 - gt keys == X; every slot zeroed after its last read. + uint32_t const ties = select && !all ? static_cast(need0) - gt : 0u; +#pragma unroll 4 + for (int i = tid; i < nc; i += kThreads) + { + uint64_t* const p = at(i); + uint64_t const v = __ldcg(reinterpret_cast(p)); + if (select) + { + uint32_t const key = static_cast(v); + int slot = -1; + if (all || key > X) + { + slot = static_cast(atomicAdd(&scal[10], 1u)); + } + else if (key == X) + { + uint32_t const t = atomicAdd(&scal[11], 1u); + slot = t < ties ? static_cast(gt + t) : -1; + } + if (slot >= 0) + { + int const pos = static_cast(v >> 32); + if (mcOut != nullptr) + { + mcOut[slot] = pos; + } + if (topkOut != nullptr) + { + topkOut[slot] = pos; + arenaOut[slot] = pos; + } + } + } + stU64(p, 0ull); + } + if (select) + { + int const filled = min(nc, need0); +#pragma unroll 4 + for (int i = filled + tid; i < outEnd; i += kThreads) + { + if (mcOut != nullptr) + { + mcOut[i] = -1; + } + if (topkOut != nullptr) + { + topkOut[i] = -1; + arenaOut[i] = -1; + } + } + } +} + +// The selector's last part: the row counted done, then list B, when it was not taken, zeroed slot by slot once each +// candidate landed (a zero stored before the candidate lands would leave a stale key). +__device__ __forceinline__ void idxTopKDone(Ctx const& ctx, uint32_t* gc, uint64_t* listB, int nB, bool useB, int row) +{ + int const tid = threadIdx.x; + __syncthreads(); + subStamp(ctx, 47); + if (tid == 0) + { + redReleaseAdd(gc + kCIdxDone, 1u); + } + if (!useB) + { + int const n = min(nB, kCandList); +#pragma unroll 1 + for (int i = tid; i < n; i += kThreads) + { + uint64_t t1 = 0; + while (static_cast(ldPollU64(listB + i)) == 0u) + { + spinCheck(ctx, t1, 174, row, i); + } + stU64(listB + i, 0ull); + } + } +} + +// The top-2048 of every row from the logits in the CTA's tiles (stageIdxSel's last part): T(t), T2(t) and the CTA's +// candidates (idxBoundCands: one 64-bit (key, position) store each into zeroed lists A and B, so key 0 = not written +// yet). CTA kIdxSelBase + t, once every CTA has made its candidates' stores (a written count, released), copies the +// row's list A, and list B too when A holds fewer than need0 keys (A, a zero pair when A's count is odd, then B), into +// shared memory at a0 (bulk copies on barK, phase parity barParity), polls the slots it read as zero (not landed yet) +// from global memory, zeroes them for the next leader launch, packs the keys and positions for the selection, picks +// the row's 2048 (slots past the row's valid keys get -1; the search's first probe is E(t)), counts the row done and +// then zeroes list B when it did not take it (polling each slot first: a zero stored before the candidate lands would +// leave a stale key). (Copying as soon as every CTA held its slots, before their stores, left 2-5 us of polling.) +__device__ __noinline__ void idxTopK(Ctx const& ctx, uint32_t a0, uint8_t* s0, int nTiles, int L, uint64_t* cand, + int* topkBase, uint32_t barK, uint32_t barParity) +{ + int const tid = threadIdx.x; + int const cta = blockIdx.x; + uint32_t* gc = ctx.counters; + uint32_t const sScr = ctx.smemBase + kSmemRed; + idxBoundCands(ctx, a0, nTiles, cta, gc, cand, sScr, arena(ctx, kArIdxBnd), L); + // The CTA's candidate stores, then the written count as a release (after the join: it covers every thread's + // stores), which the selector's acquire pairs with before it bulk-copies the lists: a relaxed count (~1 us less) + // left the copy racing the stores, and a torn record (new key, old position) would pass its key-0 re-poll. + __syncthreads(); + if (tid == 0) + { + redReleaseAdd(gc + kCIdxWritten, 1u); + } + subStamp(ctx, 45); + // The row shard: CTA kIdxSelBase selects the rank's row (the top-k exchange takes the even rank's to every rank). + bool const rs = (GLM52_FLAGS(ctx) & kFlagIdxAllRows) == 0; + bool const parallel = rs && (ctx.rank & 1) == 0 && (GLM52_FLAGS(ctx) & kFlagIdxParallel); + if (rs ? cta < kIdxSelBase || (cta != kIdxSelBase && !parallel) : cta < kIdxSelBase) + { + return; + } + int const row = rs ? ctx.rank >> 1 : cta - kIdxSelBase; + int const need0 = min(kTopK, L - kTok + row + 1); + // The row into the top-k buffer and into this launch's arena (cleared to kEmpty: the leader's attention polls those + // words). The row shard: into the top-k exchange instead, from the even rank of the pair alone (two selections of + // a row list it in different orders); the attention copies every row from there into the top-k buffer. The odd + // rank only clears its lists. + int* topkOut = rs ? nullptr : topkBase + row * kTopK; + int* arenaOut = rs ? nullptr : arena(ctx, kArIdxTopk) + row * kTopK; + int* mcOut = rs && (ctx.rank & 1) == 0 + ? reinterpret_cast(ptr(ctx, kPTkMc) + static_cast(ctx.parity) * kTkBytes) + row * kTopK + : nullptr; + bool const select = topkOut != nullptr || mcOut != nullptr; + uint64_t* const listA = cand + static_cast(row) * kCandCap; + uint64_t* const listB = listA + kCandList; + constexpr bool kSelFast = (kSpecializedStaticFlags & kFlagSelFast) != 0; + if (parallel && cta != kIdxSelBase && !kSelFast) + { + idxSelectBinParallel(ctx, need0, listA, mcOut); + return; + } + // As idxBoundCands took them; E(row) from the arena (published with them). + uint32_t const T = lds32(sScr + 768 + 4 * row); + uint32_t const T2 = lds32(sScr + 896 + 4 * row); + bool const hist = select && !(GLM52_FLAGS(ctx) & kFlagTopkNoHist); + uint32_t const sWin = sScr + 848; +#if GLM52_TOPK_UNION + // The written count, the counts and the window in one poll (warp 1): every CTA has issued its candidates' stores, + // the slot atomics counted every CTA (bits 24-31: the counts are final), and the bins' sum equals the binned count + // that rode on those atomics. (One after the other they took three round trips. Starting before the written count + // left 2-5 us of polling for slots not landed yet.) + uint32_t const hsh = lds32(sScr + 864 + 4 * row); + uint32_t const hspan = static_cast(kHistBins) << hsh; + if ((tid >> 5) == 1) + { + int const lane = tid & 31; + uint32_t const* const hs = gc + kCIdxHist + row * kHistBins; + uint32_t wr = ldAcquire(gc + kCIdxWritten); + uint64_t nn = ldPollU64(gc + kCIdxCand + 2 * row); + uint32_t h0 = hist ? ldPoll32(hs + 2 * lane) : 0u; + uint32_t h1 = hist ? ldPoll32(hs + 2 * lane + 1) : 0u; + uint64_t t1 = 0; + while (true) + { + bool const all = wr >= static_cast(kCtas) + && ((static_cast(nn) >> 24) & 0xffu) == static_cast(kCtas); + uint32_t const sum = __reduce_add_sync(0xffffffffu, h0 + h1); + if (warpAll(all && (!hist || sum == static_cast(nn >> 50)))) + { + break; + } + spinCheck(ctx, t1, 168, row, lane); + wr = ldAcquire(gc + kCIdxWritten); + nn = ldPollU64(gc + kCIdxCand + 2 * row); + if (hist) + { + h0 = ldPoll32(hs + 2 * lane); + h1 = ldPoll32(hs + 2 * lane + 1); + } + } + uint32_t const cA = static_cast(nn) & 0xffffffu; + uint32_t const binned = static_cast(nn >> 50); + if (lane == 0) + { + sts32(sScr + 832, cA); + sts32(sScr + 836, static_cast(nn >> 32) & 0x3ffffu); + sts32(sScr + 844, binned); // list A's keys in the histogram + } + if (hist) + { + // The window: the bin holding the row's need0-th key (count(>= its lower edge) >= need0 > count(> its upper + // edge)), or the keys above the bins when they are that many; none when list A is short (list B taken). + uint4 w = make_uint4(0u, 0u, 0u, 0u); + if (cA >= static_cast(need0)) + { + uint32_t const above = cA - binned; + uint32_t suf = h0 + h1; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) + { + uint32_t const x = __shfl_down_sync(0xffffffffu, suf, off); + suf += lane + off < 32 ? x : 0u; + } + uint32_t const s0 = above + suf; + uint32_t const s1 = s0 - h0; + uint32_t const need = static_cast(need0); + unsigned const hit0 = __ballot_sync(0xffffffffu, s0 >= need); + unsigned const hit1 = __ballot_sync(0xffffffffu, s1 >= need); + if (above >= need) + { + w = make_uint4(T2 + hspan, 0u, above, 0u); // up to the largest key + } + else + { + int const c = max(2 * (31 - __clz(hit0)), hit1 != 0 ? 2 * (31 - __clz(hit1)) + 1 : 0); + uint32_t const sc = __shfl_sync(0xffffffffu, (c & 1) ? s1 : s0, c >> 1); + uint32_t const hc = __shfl_sync(0xffffffffu, (c & 1) ? h1 : h0, c >> 1); + uint32_t const edge = T2 + (static_cast(c) << hsh); + w = make_uint4(edge, edge + (1u << hsh) - 1u, sc, sc - hc); + } + } + if (lane == 0) + { + sts128(sWin, w); + } + } + } + if (tid == 0) + { + sts32(sScr + 840, lds32(sScr + 800 + 4 * row)); + st32(gc + kCIdxT + row, ~T); + } +#else + if (tid == 0) + { + // E's load first (published with T: in by now), its round trip under the written count's wait. + // E: from the T CTA's word, or as this CTA took it with T (never kEmpty then). + uint8_t const* const eWord = arena(ctx, kArIdxBnd) + (kCtas * kTok + 2 * kTok + row) * 4; + uint32_t const e = (GLM52_FLAGS(ctx) & kFlagTopkTCta) ? ldPoll32(eWord) : lds32(sScr + 800 + 4 * row); + uint64_t t0 = 0; + while (ldAcquire(gc + kCIdxWritten) < static_cast(kCtas)) + { + spinCheck(ctx, t0, 168, row, 0); + } + // The slot atomics are done in L2 before the counted stores: one 64-bit read of both lists' counts. + uint64_t const nn = ldPollU64(gc + kCIdxCand + 2 * row); + sts32(sScr + 832, static_cast(nn) & 0xffffffu); + sts32(sScr + 836, static_cast(nn >> 32) & 0x3ffffu); + sts32(sScr + 844, static_cast(nn >> 50)); // list A's keys in the histogram + sts32(sScr + 840, e != kEmpty ? e : poll1(ctx, eWord, 175)); + st32(gc + kCIdxT + row, ~T); + } +#endif + // Every thread's shared-memory accesses before the bulk copy rewrites the region (the CTA's own logits), and its + // own candidates' stores before the async-proxy read. + fenceProxyAsyncSmem(); + __syncthreads(); + subStamp(ctx, 46); + int const nA = static_cast(lds32(sScr + 832)); + int const nB = static_cast(lds32(sScr + 836)); + uint32_t const E = lds32(sScr + 840); + bool const useB = nA < need0; + int const pad = useB ? (nA & 1) : 0; + int const nc = useB ? nA + pad + nB : nA; + uint32_t const lo = useB ? T : T2; + if (nA > kCandList || (useB && nB > kCandList)) + { + fail(ctx, 169, row, nA + (useB ? nB << 16 : 0)); + } + if (nA > kCandSmemList || (useB && nB > kCandSmemList)) + { + // More candidates than the shared-memory selection copies: the exact selection in global memory, by the + // original selector alone (the helpers took the same counts and leave). + if (parallel && cta != kIdxSelBase) + { + return; + } + if (parallel && !kSelFast && tid == 0) + { + redReleaseAdd(gc + kCIdxParState, 1u); + } + idxSelectGlobal(ctx, listA, nA, listB, useB ? nB : 0, need0, select, s0, topkOut, arenaOut, mcOut); + idxTopKDone(ctx, gc, listB, nB, useB, row); + return; + } + auto issueCopy = [&]() + { + if (tid == 0) + { + asm volatile("fence.proxy.async.global;" ::: "memory"); + uint32_t const bytesA = static_cast(useB ? nA + pad : (nA + 1) & ~1) * 8u; + uint32_t const bytesB = useB ? static_cast((nB + 1) & ~1) * 8u : 0u; + mbarExpectTx(barK, bytesA + bytesB); + if (bytesA != 0) + { + bulkCopy(a0, listA, bytesA, barK); + } + if (bytesB != 0) + { + bulkCopy(a0 + bytesA, listB, bytesB, barK); + } + } + }; + // With the histogram and list A alone, no copy unless the window is not a small bin: the bin pass reads list A + // itself (the copy took ~2 us after the written count, mk-v134a L10). + bool const direct = select && !(GLM52_FLAGS(ctx) & kFlagTopkNoHist) && !useB && !(GLM52_FLAGS(ctx) & kFlagBinCopy); + if (parallel && !direct && tid == 0 && !kSelFast) + { + redReleaseAdd(gc + kCIdxParState, 1u); // helpers leave without touching the original selector's lists + } + if (kSelFast && parallel && !direct && cta != kIdxSelBase) + { + return; // the original selector owns every candidate and output; the helpers took the same decision + } + if (!direct) + { + issueCopy(); + } + // The direct pass's first four pairs per lane go out now, in flight while warp 1 takes the window. + uint2 pre[4]; +#pragma unroll + for (int k = 0; k < 4; ++k) + { + int const i = 128 * (tid >> 5) + (tid & 31) + 32 * k; + pre[k] = direct && !parallel && i < nA ? __ldcg(reinterpret_cast(listA + i)) : make_uint2(0u, 0u); + } + // While the copy lands (warp 1): the search's first window from list A's histogram, once the bins' sum equals the + // count that rode on the slot atomics (every CTA's adds landed): the bin holding the row's need0-th key (count(>= + // its lower edge) >= need0 > count(> its upper edge)), or the keys above the bins when they are that many. + if (!GLM52_TOPK_UNION && hist && (tid >> 5) == 1) + { + int const lane = tid & 31; + uint4 w = make_uint4(0u, 0u, 0u, 0u); // ca = 0: no window (the search starts from the whole list) + if (!useB) + { + uint32_t const binned = lds32(sScr + 844); + uint32_t const* const hs = gc + kCIdxHist + row * kHistBins; + uint32_t h0 = ldPoll32(hs + 2 * lane); + uint32_t h1 = ldPoll32(hs + 2 * lane + 1); + uint64_t t1 = 0; + while (__reduce_add_sync(0xffffffffu, h0 + h1) != binned) + { + spinCheck(ctx, t1, 176, lane, static_cast(binned)); + h0 = ldPoll32(hs + 2 * lane); + h1 = ldPoll32(hs + 2 * lane + 1); + } + // List A's keys >= the lower edge of bin 2 lane (s0) and of bin 2 lane + 1 (s1). + uint32_t const above = static_cast(nA) - binned; + uint32_t suf = h0 + h1; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) + { + uint32_t const v = __shfl_down_sync(0xffffffffu, suf, off); + suf += lane + off < 32 ? v : 0u; + } + uint32_t const s0 = above + suf; + uint32_t const s1 = s0 - h0; + uint32_t const need = static_cast(need0); + unsigned const hit0 = __ballot_sync(0xffffffffu, s0 >= need); + unsigned const hit1 = __ballot_sync(0xffffffffu, s1 >= need); + if (above >= need) + { + w = make_uint4(T2 + kHistSpan, 0u, above, 0u); // up to the largest key + } + else + { + // The largest such bin (lane 0's s0 = nA >= need0). + int const c = max(2 * (31 - __clz(hit0)), hit1 != 0 ? 2 * (31 - __clz(hit1)) + 1 : 0); + uint32_t const sc = __shfl_sync(0xffffffffu, (c & 1) ? s1 : s0, c >> 1); + uint32_t const hc = __shfl_sync(0xffffffffu, (c & 1) ? h1 : h0, c >> 1); + uint32_t const edge = T2 + (static_cast(c) << kHistShift); + w = make_uint4(edge, edge + (1u << kHistShift) - 1u, sc, sc - hc); + } + } + if (lane == 0) + { + sts128(sWin, w); + } + } + bool binDirect = false; + if (direct) + { + __syncthreads(); // warp 1's window to every thread + uint4 const w = lds128(sWin); + binDirect = w.y != 0 && w.z - w.w <= static_cast(kSelWin) && !(GLM52_FLAGS(ctx) & kFlagTopkNoBin); + if (binDirect) + { + subStamp(ctx, 79); + if (parallel && kSelFast) + { + idxSelectBinParallelFast(ctx, need0, listA, mcOut, nc, w); + if (cta != kIdxSelBase) + { + return; + } + } + else if (parallel) + { + if (tid == 0) + { + st32(gc + kCIdxParN, static_cast(nc)); + st128(gc + kCIdxParWin, w); + redReleaseAdd(gc + kCIdxParState, 2u); + } + idxSelectBinParallel(ctx, need0, listA, mcOut); + } + else + { + idxSelectBin(ctx, nc, need0, w, a0, sScr + 1024, rs ? mcOut : topkOut, rs ? nullptr : arenaOut, listA, + true, (GLM52_FLAGS(ctx) & kFlagBinNoPreload) ? nullptr : pre); + } + } + else + { + if (kSelFast && parallel && cta != kIdxSelBase) + { + return; + } + if (parallel && tid == 0 && !kSelFast) + { + redReleaseAdd(gc + kCIdxParState, 1u); + } + issueCopy(); + } + } + if (!binDirect) + { + waitBar(ctx, barK, barParity, 171); + subStamp(ctx, 79); + if (hist) + { + __syncthreads(); // warp 1's window to every thread + } + uint4 const win = hist ? lds128(sWin) : make_uint4(0u, 0u, 0u, 0u); + bool const bin + = win.y != 0 && win.z - win.w <= static_cast(kSelWin) && !(GLM52_FLAGS(ctx) & kFlagTopkNoBin); + if (bin) + { + // A bin window (list A only): one pass, into the exchange (the row shard) or the top-k buffer and the + // arena; it polls the slots the copy read before they landed, then clears the list. + idxSelectBin(ctx, nc, need0, win, a0, sScr + 1024, rs ? mcOut : topkOut, rs ? nullptr : arenaOut, listA, + false, nullptr); + } + else + { + // Slots the copy read before their candidate landed (key 0; thread tid's slots tid + 256 k, k < 64, the pad + // excepted): polled from global memory, up to 8 loads in flight per round trip (one at a time took ~2 us + // more); per lane, the barrier below rejoins the warps. + auto src = [&](int i) { return i < nA ? listA + i : listB + (i - nA - pad); }; + uint64_t zero = 0; +#pragma unroll 1 + for (int k = 0; k < 64 && tid + kThreads * k < nc; ++k) + { + int const i = tid + kThreads * k; + zero |= lds32(a0 + 8 * i) == 0u && !(pad && i == nA) ? 1ull << k : 0ull; + } + uint64_t t0 = 0; +#pragma unroll 1 + while (zero != 0) + { + uint64_t v[8]; + int kk[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) + { + uint64_t const rest = j == 0 ? zero : zero & ~((2ull << kk[j - 1]) - 1); + kk[j] = rest != 0 ? __ffsll(static_cast(rest)) - 1 : -1; + v[j] = kk[j] >= 0 ? ldPollU64(src(tid + kThreads * kk[j])) : 0ull; + } +#pragma unroll + for (int j = 0; j < 8; ++j) + { + if (kk[j] >= 0 && static_cast(v[j]) != 0u) + { + sts64(a0 + 8 * (tid + kThreads * kk[j]), + make_uint2(static_cast(v[j]), static_cast(v[j] >> 32))); + zero &= ~(1ull << kk[j]); + } + } + if (zero != 0) + { + spinCheck(ctx, t0, 173, row, tid); + } + } + __syncthreads(); + // The lists taken are in shared memory now: back to zero for the next leader launch, the stores draining + // while the selection computes (zeroed after it, they delayed this CTA's next units by ~1.2 us; at every + // CTA's exit, the next launch by ~2.7 us; after the selection in mk-v98a: 0.7 us slower per slice). +#pragma unroll 1 + for (int i = 2 * tid; i < nA; i += 2 * kThreads) + { + st128(listA + i, make_uint4(0u, 0u, 0u, 0u)); + } + if (useB) + { +#pragma unroll 1 + for (int i = 2 * tid; i < nB; i += 2 * kThreads) + { + st128(listB + i, make_uint4(0u, 0u, 0u, 0u)); + } + } + if (select && nc <= 32 * kThreads) + { + // Keys and positions packed four per 16 bytes (keys past nc zero) after the pairs. + uint32_t const sKey = a0 + 32 * kThreads * 8; + uint32_t const sPos = sKey + 32 * kThreads * 4; +#pragma unroll 1 + for (int q = tid; q < (nc + 3) >> 2; q += kThreads) + { + int const i = 4 * q; + uint4 const p0 = lds128(a0 + 32 * q); + uint4 const p1 = lds128(a0 + 32 * q + 16); + sts128(sKey + 16 * q, + make_uint4(p0.x, i + 1 < nc ? p0.z : 0u, i + 2 < nc ? p1.x : 0u, i + 3 < nc ? p1.z : 0u)); + sts128(sPos + 16 * q, make_uint4(p0.y, p0.w, p1.y, p1.w)); + } + // Each thread reads back only the groups it packed. + idxSelectRow(ctx, nc, need0, lo, E, win, topkOut, arenaOut, mcOut, sScr + 1024, sKey, sPos); + } + else if (select) + { + idxSelectLarge(ctx, nc, need0, lo, s0, a0, reinterpret_cast(ctx.smem + kSmemRed + 2048), + topkOut, arenaOut, mcOut); + } + } + } + idxTopKDone(ctx, gc, listB, nB, useB, row); +} + +#if GLM52_TOPK_RSH +// ---------------------------------------------------------------------------------------------------------------- +// RS-H: the row shard's exact top-2048 above kTopK keys from histograms of the row's keys (no bound T, candidate lists +// or selectors). Rank 2 t scores row t over all keys (the row shard) and publishes it: +// - each CTA counts its valid keys per bin (rsBin) in shared memory in its tile epilogues; +// - right after its logits and one scan it adds its counts of the bins from the one holding its kRsLocalK-th largest +// key up (B) to the row's global histogram H (one relaxed red.add per nonzero bin; H: 6144 u32 in the union words' +// place past list A, zero when unused) and each warp publishes its count added (and B if in its bins) in its word +// of Pw (past H); then it sorts its keys of the bins >= B by bin (descending) in shared memory; +// - the row CTA (kIdxSelBase) polls H and Pw until every word of the CTAs with keys is in and H's bins sum to their +// counts (relaxed adds only raise the sum: every add landed; no fence), and takes the cut bin b* (the largest bin +// with count(>= b*) >= need), the count above it and its own: exact when b* >= every CTA's B (H holds every key of +// those bins); else (kRsMore) every CTA adds its other keys too and the row CTA takes the cut on the whole +// histogram. It publishes a descriptor D (the counters' kCIdxParWin; after kRsMore, kCIdxParKey; nonzero words); +// - each CTA reads its counts above b* and in b* from its sorted bins (two shared loads), takes its slots with one +// 64-bit atomic (the row's kCIdxCand pair: above | in-bin << 32), stores the positions of its keys above b* +// straight into the multicast top-k row, and its keys in b* there too when b* is taken whole or holds equal keys +// (+0.0 / -0.0), else as (key, position) pairs into list A; +// - the row CTA ranks the pairs (larger key first, then the smaller position: torch.topk's choice among equal keys) +// and stores the first need - C_above at [C_above, need); more than kRsPairs pairs (a cut in a tail bin, heavy ties) +// go through the exact global selection. [need, kTopK) gets -1. It zeroes H and Pw. +// Every slot of [0, need) is written once (the bins partition the keys by order key, so the slot counts match). The +// code between a CTA's logits and its stores runs once per launch from a cold instruction cache (~0.4 us per KiB): +// the work before the adds and after D is kept to a few rolled loops. +// ---------------------------------------------------------------------------------------------------------------- +static_assert( + (kCandCap - kCandList) * 2 >= kRsBins + kCtas * kWarps, "RS-H's histogram and words in the union's place"); +constexpr int kRsLh = kSelQ8 + 32 * 128 + 512; // the CTA's per-bin counts (u32 [kRsBins]) past the row's Q and blocks +static_assert(4 * kIdxMaxPagesRs <= 512 && kRsLh + 4 * kRsBins <= kSelKs, "RS-H's counts in the Q region"); +constexpr int kRsMaxKeys = 128 * (kIdxMaxPagesRs / 2); // a CTA's local keys +constexpr int kRsKey = 0; // the CTA's sorted keys (u32), in the K region (dead after the +constexpr int kRsPos = kRsKey + 4 * kRsMaxKeys; // logits), then their positions +static_assert(kRsPos + 4 * kRsMaxKeys <= kSelQ8, "RS-H's sorted keys in the K region"); + +// The bin of order key key (!= 0), without branches (it runs once per key in the tile epilogues): the offset into the +// sign's window clamped to [-1, kRsW] (-1: below it, kRsW: above), from bin 1 (negative) or kRsW + 5 (positive); then +// -0.0 and +0.0. +__device__ __forceinline__ uint32_t rsBin(uint32_t key) +{ + bool const pos = key >= 0x80000000u; + int const d = min(max(static_cast(key >> 16) - static_cast(pos ? kRsP0 : kRsN0), -1), kRsW); + uint32_t const b = static_cast((pos ? kRsW + 5 : 1) + d); + return key == 0x80000000u ? kRsBinPZ : key == 0x7fffffffu ? kRsBinNZ : b; +} + +// The smallest order key of bin b (keys >= it lie in the bins >= b). +__device__ __forceinline__ uint32_t rsLo(uint32_t b) +{ + return b == 0u ? 1u + : b <= kRsW + 1u ? (kRsN0 + b - 1u) << 16 + : b == kRsBinNZ ? 0x7fffffffu + : b == kRsBinPZ ? 0x80000000u + : b == kRsW + 4u ? 0x80000001u + : (kRsP0 + b - kRsW - 5u) << 16; +} + +// Bin b's word in the global histogram (thread b / kRsPer's polls hold it). +__device__ __forceinline__ uint32_t rsWord(uint32_t b) +{ + uint32_t const t = b / kRsPer; + uint32_t const r = b - t * kRsPer; + return 4u * (t + kThreads * (r >> 2)) + (r & 3u); +} + +// The tile epilogue's count of one logit in the CTA's per-bin counts at shared address sLh (-inf: a key past the row's +// last, not counted). +__device__ __forceinline__ void rsCount(uint32_t sLh, float lgt) +{ + if (lgt != -INFINITY) + { + redsAddUnordered(sLh + 4u * rsBin(orderKey(lgt)), 1u); + } +} + +// After the CTA's logits (at a0 + kSelLog) and per-bin counts (at a0 + kRsLh; thread t's bins [kRsPer t, + kRsPer)): +// its counts of the bins >= B (the bin of its kRsLocalK-th largest key, 0 when it has fewer) into H after one scan +// (thread t adds at its own poll words), each warp's word Pw[cta][warp] = (its count added + 1) | (B, if in its bins) +// << 16; then per bin the CTA's keys in the bins above it (in place, rsSort's starts). Returns B. sScr: 64 bytes. +__device__ __noinline__ uint32_t rsFlush(Ctx const& ctx, uint32_t a0, uint32_t* H, uint32_t* Pw, uint32_t sScr) +{ + int const tid = threadIdx.x; + int const lane = tid & 31; + int const warp = tid >> 5; + uint32_t const sLh = a0 + kRsLh + 4u * kRsPer * static_cast(tid); + uint32_t s = 0; +#pragma unroll 1 + for (int j = 0; j < kRsPer / 4; ++j) + { + uint4 const q = lds128(sLh + 16 * j); + s += (q.x + q.y) + (q.z + q.w); + } + // The keys in the bins above this thread's: its warp's higher lanes, then the warps above. + uint32_t incl = s; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) + { + uint32_t const x = __shfl_down_sync(0xffffffffu, incl, off); + incl += lane + off < 32 ? x : 0u; + } + if (lane == 0) + { + sts32(sScr + 4 * warp, incl); + } + __syncthreads(); + uint32_t above = incl - s; +#pragma unroll 1 + for (int w = warp + 1; w < kWarps; ++w) + { + above += lds32(sScr + 4 * w); + } + // From this thread's top bin down: a bin is added while fewer than kRsLocalK keys lie above it. + uint32_t acc = above; + uint32_t added = 0; + uint32_t bK = 0; +#pragma unroll 1 + for (int j = kRsPer / 4 - 1; j >= 0; --j) + { + uint4 const q = lds128(sLh + 16 * j); + uint32_t const c[4] = {q.x, q.y, q.z, q.w}; +#pragma unroll + for (int i = 3; i >= 0; --i) + { + if (c[i] != 0u && acc < kRsLocalK) + { + asm volatile( + "red.relaxed.gpu.global.add.u32 [%0], %1;" ::"l"(H + 4 * (tid + kThreads * j) + i), "r"(c[i])); + added += c[i]; + bK = acc + c[i] >= kRsLocalK ? static_cast(kRsPer * tid + 4 * j + i) : bK; + } + acc += c[i]; + } + } + added = __reduce_add_sync(0xffffffffu, added); + bK = __reduce_max_sync(0xffffffffu, bK); + if (lane == 0) + { + st32(Pw + kWarps * blockIdx.x + warp, (added + 1u) | (bK << 16)); + sts32(sScr + 32 + 4 * warp, bK); + } + subStamp(ctx, 65); + // Per bin the CTA's keys in the bins above it, in place. + acc = above; +#pragma unroll 1 + for (int j = kRsPer / 4 - 1; j >= 0; --j) + { + uint4 const q = lds128(sLh + 16 * j); + uint4 st; + st.w = acc; + acc += q.w; + st.z = acc; + acc += q.z; + st.y = acc; + acc += q.y; + st.x = acc; + acc += q.x; + sts128(sLh + 16 * j, st); + } + __syncthreads(); + uint32_t bC = 0; +#pragma unroll 1 + for (int w = 0; w < kWarps; ++w) + { + bC = max(bC, lds32(sScr + 32 + 4 * w)); + } + return bC; +} + +// The CTA's keys of the bins >= bC (or < bC with less) sorted by bin, descending, after rsFlush: keys at a0 + kRsKey, +// positions at a0 + kRsPos (each bin's start then becomes the end of its keys). With H, each key also goes into H +// (kRsMore: the keys rsFlush left out, one red.add each). +__device__ __noinline__ void rsSort( + Ctx const& ctx, uint32_t a0, int nPer, int nPagesL, uint32_t bC, bool less, uint32_t* H) +{ + int const nKeys = 32 * nPer; + uint32_t const lo = rsLo(bC); // the keys of the bins >= bC: one compare per key + // The scan only records the thread's last eight selected keys (iteration i: key tid + 256 i); they are placed after + // it (placing them inside put a divergent atomic and its stores into most iterations). + uint32_t n = 0; + uint64_t rec = 0; +#pragma unroll 1 + for (int i = 0; threadIdx.x + kThreads * i < nKeys; ++i) + { + uint32_t const v = selKeyAt(a0, -1, threadIdx.x + kThreads * i); + bool const sel = less ? v != 0u && v < lo : v >= lo; + n += sel ? 1u : 0u; + rec = sel ? (rec << 8) | static_cast(i) : rec; + } + auto place = [&](int k) + { + uint32_t const v = selKeyAt(a0, -1, k); + if (less ? v != 0u && v < lo : v >= lo) + { + uint32_t const b = rsBin(v); + if (H != nullptr) + { + redRelaxedAdd(H + rsWord(b), 1u); + } + uint32_t const idx = atomsAdd(a0 + kRsLh + 4 * b, 1u); + sts32(a0 + kRsKey + 4 * idx, v); + sts32(a0 + kRsPos + 4 * idx, + static_cast(idxPageOf(blockIdx.x, k >> 6, nPagesL) * kIdxPageKeys + (k & 63))); + } + }; + if (n <= 8u) + { +#pragma unroll 1 + for (uint32_t j = 0; j < n; ++j) + { + place(static_cast(threadIdx.x) + kThreads * static_cast((rec >> (8 * j)) & 0xffu)); + } + } + else + { +#pragma unroll 1 + for (int k = threadIdx.x; k < nKeys; k += kThreads) + { + place(k); + } + } + __syncthreads(); +} + +// The row CTA: polls H (thread t: its bins [kRsPer t, + kRsPer), six 16-byte vectors) and, with Pw, the words of the +// CTAs holding keys (thread c: CTA c's eight) until every such word is in and H's bins sum to their counts (without +// Pw: to n), then the cut for need: (b*, the keys above b*, the keys in b*, the largest B of the CTAs). Every round +// runs the whole cut on what it read, so its code is warm when the last add lands. sScr: 272 bytes. +__device__ __noinline__ uint4 rsRowCut( + Ctx const& ctx, uint32_t const* H, uint32_t const* Pw, int nPagesL, uint32_t n, uint32_t need, uint32_t sScr) +{ + int const tid = threadIdx.x; + int const lane = tid & 31; + int const warp = tid >> 5; + bool const mine = Pw != nullptr && tid < kCtas && idxPagesOf(tid, nPagesL) > 0; + uint4 p0 = mine ? ldPoll128(Pw + kWarps * tid) : make_uint4(1u, 1u, 1u, 1u); + uint4 p1 = mine ? ldPoll128(Pw + kWarps * tid + 4) : make_uint4(1u, 1u, 1u, 1u); + uint4 v[kRsPer / 4]; +#pragma unroll + for (int j = 0; j < kRsPer / 4; ++j) + { + v[j] = ldPoll128(H + 4 * (tid + kThreads * j)); + } + uint64_t t0 = 0; + uint32_t par = 0; +#pragma unroll 1 + while (true) + { + uint32_t h[kRsPer]; +#pragma unroll + for (int j = 0; j < kRsPer / 4; ++j) + { + h[4 * j] = v[j].x; + h[4 * j + 1] = v[j].y; + h[4 * j + 2] = v[j].z; + h[4 * j + 3] = v[j].w; + } + uint32_t s = 0; +#pragma unroll + for (int i = 0; i < kRsPer; ++i) + { + s += h[i]; + } + // Inclusive suffix sums over the warp's lanes (the higher lanes hold the larger bins), then the warps above; + // the CTAs' words missing, their counts and their largest B. + uint32_t incl = s; +#pragma unroll + for (int off = 1; off < 32; off <<= 1) + { + uint32_t const x = __shfl_down_sync(0xffffffffu, incl, off); + incl += lane + off < 32 ? x : 0u; + } + uint32_t const pv[8] = {p0.x, p0.y, p0.z, p0.w, p1.x, p1.y, p1.z, p1.w}; + bool have = true; + uint32_t kc = 0; + uint32_t bc = 0; +#pragma unroll + for (int w = 0; w < kWarps; ++w) + { + have &= pv[w] != 0u; + kc += (pv[w] & 0xffffu) - 1u; + bc = max(bc, pv[w] >> 16); + } + uint32_t const missW = __reduce_add_sync(0xffffffffu, have ? 0u : 1u); + uint32_t const kW = __reduce_add_sync(0xffffffffu, have ? kc : 0u); + uint32_t const bW = __reduce_max_sync(0xffffffffu, have ? bc : 0u); + if (lane == 0) + { + sts128(sScr + 128 * par + 16 * warp, make_uint4(incl, missW, kW, bW)); + } + __syncthreads(); + uint32_t higher = 0; + uint32_t total = 0; + uint32_t miss = 0; + uint32_t expect = 0; + uint32_t bMax = 0; +#pragma unroll + for (int w = 0; w < kWarps; ++w) + { + uint4 const x = lds128(sScr + 128 * par + 16 * w); + higher += w > warp ? x.x : 0u; + total += x.x; + miss += x.y; + expect += x.z; + bMax = max(bMax, x.w); + } + expect = Pw != nullptr ? expect : n; + // This thread's bins from its top: the one holding the need-th largest key, if any. + uint32_t acc = higher + incl - s; + uint32_t cut = 0xffffffffu; + uint32_t above = 0; + uint32_t inBin = 0; +#pragma unroll + for (int i = kRsPer - 1; i >= 0; --i) + { + bool const hit = acc < need && acc + h[i] >= need; + cut = hit ? static_cast(kRsPer * tid + i) : cut; + above = hit ? acc : above; + inBin = hit ? h[i] : inBin; + acc += h[i]; + } + if (miss == 0u && total == expect) + { + if (cut != 0xffffffffu) + { + sts128(sScr + 256, make_uint4(cut, above, inBin, 0u)); + } + __syncthreads(); + uint4 const c = lds128(sScr + 256); + return make_uint4(c.x, c.y, c.z, bMax); + } + if (miss == 0u && total > expect) + { + fail(ctx, 191, static_cast(total), + static_cast(expect)); // a stale histogram (not zeroed after use) + } + spinCheck(ctx, t0, 190, static_cast(total), static_cast(expect)); + if (!have) + { + p0 = ldPoll128(Pw + kWarps * tid); + p1 = ldPoll128(Pw + kWarps * tid + 4); + } +#pragma unroll + for (int j = 0; j < kRsPer / 4; ++j) + { + v[j] = ldPoll128(H + 4 * (tid + kThreads * j)); + } + par ^= 1u; + } +} + +// The CTA's emission for descriptor d (thread 0's; b* + 1; 1; C_above + 1 | need << 12; n_bin | mode << 24): its +// counts above b* and in b* from its sorted bins, one 64-bit slot atomic, then every thread stores its share: the +// keys in b* first (the row CTA ranks them), then the keys above. Returns false (no emission) for kRsMore. +// sScr: 32 bytes. +__device__ __noinline__ bool rsEmit( + Ctx const& ctx, uint32_t a0, uint4 d, uint32_t sScr, int* tk, uint64_t* list, unsigned long long* slots) +{ + int const tid = threadIdx.x; + if (tid == 0) + { + uint4 cnt = make_uint4(0u, 0u, 0u, 0u); // slot bases above / in b*, the keys above / in b* + if ((d.w >> 24) != kRsMore) + { + cnt.z = lds32(a0 + kRsLh + 4 * d.x); // bin b* + 1's end: the keys above b* + cnt.w = lds32(a0 + kRsLh + 4 * d.x - 4) - cnt.z; + if (cnt.z + cnt.w != 0u) + { + unsigned long long const old = atomicAdd(slots, (static_cast(cnt.w) << 32) | cnt.z); + cnt.x = static_cast(old); + cnt.y = static_cast(old >> 32); + } + } + sts128(sScr, cnt); + sts128(sScr + 16, d); + } + __syncthreads(); + subStamp(ctx, 68); + uint4 const cnt = lds128(sScr); + uint4 const dd = lds128(sScr + 16); + if ((dd.w >> 24) == kRsMore) + { + return false; + } + uint32_t const cAbove = (dd.z & 0xfffu) - 1u; + uint32_t const want = (dd.z >> 12) - cAbove; + bool const direct = (dd.w >> 24) == kRsDirect; +#pragma unroll 1 + for (uint32_t i = tid; i < cnt.w; i += kThreads) + { + uint32_t const x = cnt.z + i; + uint32_t const pos = lds32(a0 + kRsPos + 4 * x); + uint32_t const j = cnt.y + i; + if (!direct) + { + stU64(list + j, (static_cast(pos) << 32) | lds32(a0 + kRsKey + 4 * x)); + } + else if (j < want) + { + st32(tk + cAbove + j, pos); + } + } +#pragma unroll 1 + for (uint32_t i = tid; i < cnt.z; i += kThreads) + { + st32(tk + cnt.x + i, lds32(a0 + kRsPos + 4 * i)); + } + return true; +} + +// The row CTA's part after its emission: the cut bin's pairs ranked (larger key first, then the smaller position) into +// [C_above, need) of tk, or (kRsGlobal) selected exactly in global memory; list's slots back to zero. sScr: 4 KiB. +__device__ __noinline__ void rsBoundary(Ctx const& ctx, uint4 d, uint64_t* list, int* tk, uint32_t sScr, uint8_t* s0) +{ + int const tid = threadIdx.x; + uint32_t const cAbove = (d.z & 0xfffu) - 1u; + uint32_t const want = (d.z >> 12) - cAbove; + int const nBin = static_cast(d.w & 0xffffffu); + if ((d.w >> 24) == kRsGlobal) + { + // Its histogram at s0 is the sorted keys' place (a0 + kRsKey), zeroed before its first barrier: every + // thread's rsEmit reads of them first (rsEmit returns without a join; d is the CTA's own). + __syncthreads(); + idxSelectGlobal(ctx, list, nBin, nullptr, 0, static_cast(want), true, s0, nullptr, nullptr, tk + cAbove, + static_cast(want)); + return; + } + // Pairs tid and tid + 256 (key != 0 once stored), polled together; the warp leaves together. + uint64_t p[2]; +#pragma unroll + for (int u = 0; u < 2; ++u) + { + p[u] = tid + kThreads * u < nBin ? ldPollU64(list + tid + kThreads * u) : 1ull; + } + uint64_t t0 = 0; + while (!warpAll(static_cast(p[0]) != 0u && static_cast(p[1]) != 0u)) + { + spinCheck(ctx, t0, 193, tid, nBin); +#pragma unroll + for (int u = 0; u < 2; ++u) + { + if (static_cast(p[u]) == 0u) + { + p[u] = ldPollU64(list + tid + kThreads * u); + } + } + } + // Pair i as one 64-bit value, key << 32 | ~position: a larger key, or the same key and a smaller position, is the + // larger value (torch.topk's choice among equal keys; the positions are distinct). Pairs [nBin, nPad) are 0, below + // every pair (its key is nonzero). +#pragma unroll + for (int u = 0; u < 2; ++u) + { + if (tid + kThreads * u < nBin) + { + sts64(sScr + 8 * (tid + kThreads * u), + make_uint2(~static_cast(p[u] >> 32), static_cast(p[u]))); + } + } + int const nPad = (nBin + 7) & ~7; + if (tid < nPad - nBin) + { + sts64(sScr + 8 * (nBin + tid), make_uint2(0u, 0u)); + } + __syncthreads(); + subStamp(ctx, 62); + // Pair i's rank (the pairs above it), by thread i % 256: eight pairs per step (four 16-byte loads in flight) into + // four independent counts (one chain of dependent adds took ~9 cycles per pair). Warps without a pair skip it. +#pragma unroll 1 + for (int u = 0; u < 2; ++u) + { + int const i = tid + kThreads * u; + if ((i & ~31) < nBin) + { + uint2 const mm = i < nBin ? lds64(sScr + 8 * i) : make_uint2(0u, 0u); + uint64_t const m = static_cast(mm.y) << 32 | mm.x; + uint32_t r[4] = {0u, 0u, 0u, 0u}; +#pragma unroll 1 + for (int x = 0; x < nPad; x += 8) + { +#pragma unroll + for (int k = 0; k < 4; ++k) + { + // Two values as 64-bit words straight into register pairs (one 64-bit compare each, no moves). + uint64_t q0; + uint64_t q1; + asm volatile("ld.shared.v2.u64 {%0, %1}, [%2];" : "=l"(q0), "=l"(q1) : "r"(sScr + 8 * x + 16 * k)); + r[k] += (q0 > m ? 1u : 0u) + (q1 > m ? 1u : 0u); + } + } + uint32_t const rank = (r[0] + r[1]) + (r[2] + r[3]); + if (i < nBin && rank < want) + { + st32(tk + cAbove + rank, ~mm.x); + } + } + } + subStamp(ctx, 63); +#pragma unroll + for (int u = 0; u < 2; ++u) + { + if (tid + kThreads * u < nBin) + { + stU64(list + tid + kThreads * u, 0ull); + } + } +} + +// The top-k part of stageIdxSel under RS-H, after the CTA's logits (its row array at a0 + kSelLog, its per-bin counts +// at a0 + kRsLh): the CTA adds its top bins to H and sorts its top keys, the row CTA takes the cut and publishes D, +// every CTA emits, the row CTA resolves the cut bin and zeroes H and Pw. L: the context; tr: the row; nPer: the CTA's +// local keys / 32; list: the row's list A (H and Pw past it); tk: the row of the multicast top-k exchange. Every thread +// of the even rank's CTAs with key pages (and of the row CTA) calls it. +__device__ __noinline__ void rsTopK( + Ctx const& ctx, uint32_t a0, uint8_t* s0, int nPer, int L, int tr, uint64_t* list, int* tk) +{ + int const tid = threadIdx.x; + uint32_t* const gc = ctx.counters; + uint32_t const sScr = ctx.smemBase + kSmemRed; + uint32_t* const H = reinterpret_cast(list + kCandList); + uint32_t* const Pw = H + kRsBins; + unsigned long long* const slots = reinterpret_cast(gc + kCIdxCand + 2 * tr); + int const nPagesL = (L + kIdxPageKeys - 1) / kIdxPageKeys; + bool const row = blockIdx.x == kIdxSelBase; + uint32_t const need = min(static_cast(kTopK), static_cast(L - kTok + tr + 1)); + uint32_t const bC = rsFlush(ctx, a0, H, Pw, sScr); + if (!row) + { + rsSort(ctx, a0, nPer, nPagesL, bC, false, nullptr); // the row CTA sorts once D is out + } + subStamp(ctx, 64); + uint4 d = make_uint4(0u, 0u, 0u, 0u); +#pragma unroll 1 + for (int phase = 0; phase < 2; ++phase) + { + // Phase 0: D in kCIdxParWin; after kRsMore (every CTA added the rest of its keys) D in kCIdxParKey. + uint32_t* const dAddr = gc + (phase == 0 ? kCIdxParWin : kCIdxParKey); + if (row) + { + // Its scratch past rsFlush's B words (sScr + 32: slower warps may still read them, with no join after). + uint4 const c = rsRowCut( + ctx, H, phase == 0 ? Pw : nullptr, nPagesL, static_cast(L - kTok + tr + 1), need, sScr + 64); + subStamp(ctx, phase == 0 ? 132 : 134); + if (tid == 0) + { + uint32_t const mode = c.x < c.w ? kRsMore + : need - c.y == c.z || c.x == kRsBinNZ || c.x == kRsBinPZ ? kRsDirect + : c.z <= kRsPairs ? kRsRank + : kRsGlobal; + d = make_uint4(c.x + 1u, 1u, (c.y + 1u) | (need << 12), c.z | (mode << 24)); + st128(dAddr, d); + } + subStamp(ctx, 133); + if (phase == 0) + { + rsSort(ctx, a0, nPer, nPagesL, bC, false, nullptr); + } + } + else if (tid < 32) + { + d = ldPoll128(dAddr); + uint64_t t0 = 0; + while (!warpAll(d.x != 0u && d.y != 0u && d.z != 0u && d.w != 0u)) + { + spinCheck(ctx, t0, 192, phase, 0); + d = ldPoll128(dAddr); + } + subStamp(ctx, 44); + } + if (rsEmit(ctx, a0, d, sScr + 512, tk, list, slots)) + { + break; + } + rsSort(ctx, a0, nPer, nPagesL, bC, true, H); // kRsMore: the CTA's other keys into H and its sorted keys + } + subStamp(ctx, 45); + if (row) + { + d = lds128(sScr + 512 + 16); +#pragma unroll 1 + for (int i = static_cast(need) + tid; i < kTopK; i += kThreads) + { + st32(tk + i, ~0u); + } + if ((d.w >> 24) != kRsDirect) + { + rsBoundary(ctx, d, list, tk, sScr + 1024, s0); + } + // H and Pw back to zero for the next leader launch of this copy (every add landed: its sum was the expected). +#pragma unroll + for (int j = 0; j < kRsPer / 4; ++j) + { + st128(H + 4 * (tid + kThreads * j), make_uint4(0u, 0u, 0u, 0u)); + } + for (int i = tid; i < kCtas * kWarps / 4; i += kThreads) + { + st128(Pw + 4 * i, make_uint4(0u, 0u, 0u, 0u)); + } + subStamp(ctx, 47); + } +} +#endif + +// Leader: the indexer logits of this CTA's key pages (pages c, c + 212, ... of the request's ceil(L / 64)) for the 8 +// rows, then the exact top-2048 of every row with all the other CTAs. Logit of key s for row t (s <= L - 8 + t): +// scale(s) * sum_h w(t, h) relu(Q8(t, h) . K8(s)), w = weights_proj x q_scale x 128^-0.5 x 32^-0.5 (stock numerics). +// Top-k: a lower bound T(t) of row t's 2048th-largest logit from every CTA's ten largest (each CTA polls the others' +// bounds), every CTA's keys >= T(t) into row t's candidate list, and CTA kIdxSelBase + t selects the row's 2048 from +// the list. +__device__ __noinline__ void stageIdxSel(Ctx const& ctx, uint32_t wOff, int const* kvLenPtr) +{ + int const tid = threadIdx.x; + int const warp = tid >> 5; + int const lane = tid & 31; + int const cta = blockIdx.x; + uint8_t* s0 = ctx.smem + wOff; + uint32_t const a0 = ctx.smemBase + wOff; + float* sKs = reinterpret_cast(s0 + kSelKs); + float* sW = reinterpret_cast(s0 + kSelW); + uint32_t const barK = ctx.smemBase + kSmemSmall + kSmallBars + kBarIdxK * 8; + uint32_t const barS0 = barK + 8; + uint32_t const barS1 = barK + 16; + uint32_t const barK2 = ctx.smemBase + kSmemSmall + kSmallBars + kBarIdxK2 * 8; + uint32_t* sTmem = reinterpret_cast(ctx.smem + kSmemSmall + kSmallScalars + 4 * 120); + int const L = __ldg(kvLenPtr); + if (idxAllKeys(L)) + { + return; + } + int const nMine = idxSelMyPages(L); + int const nPagesL = (L + kIdxPageKeys - 1) / kIdxPageKeys; + // The row shard: this rank's row tr (ranks 2 tr and 2 tr + 1 both score it; the even one publishes its selection). + bool const rs = (GLM52_FLAGS(ctx) & kFlagIdxAllRows) == 0; + int const tr = ctx.rank >> 1; +#if GLM52_TOPK_RSH + static_assert( + (kSpecializedStaticFlags & (kFlagIdxAllRows | kFlagIdxTilePairs)) == 0, "RS-H: the row shard's rounds"); + // RS-H: the even rank's CTAs score, count and emit row tr; the odd rank's selection would be unused, and a CTA + // without key pages (the IdxQ CTAs below ~37k keys) has nothing to score or emit (the row CTA still takes the cut). + if ((ctx.rank & 1) != 0 || (nMine == 0 && cta != kIdxSelBase)) + { + return; + } + // The CTA's per-bin counts start at zero (the epilogues add to them; pagesIn's barrier comes first). +#pragma unroll + for (int j = 0; j < kRsPer / 4; ++j) + { + sts128(a0 + kRsLh + 16 * (tid + kThreads * j), make_uint4(0u, 0u, 0u, 0u)); + } +#endif + // The row shard takes its pages in rounds of kIdxMaxPages (the logits into the row's array), the 8 rows in one. + bool const rounds = rs && !(GLM52_FLAGS(ctx) & kFlagIdxTilePairs); + if (nMine > (rounds ? kIdxMaxPagesRs : kIdxMaxPages)) + { + fail(ctx, 160, nMine, L); + } + int const nTiles = (nMine + 1) / 2; + uint64_t* cand = ptr(ctx, kPIdxCand) + static_cast(ctx.parity) * kTok * kCandCap; + int* const topkBase = ptr(ctx, kPTopkOut); + // 1. The key pages and their scales (the first round's). (Issued by the CTA's IdxQ unit once its MMAs had read its + // weights, they held up that unit's epilogue: warp 0 takes ~1-2 us to issue 22 requests; IdxQ ended 2 us later, + // mk-v98a.) + // The row shard: in batches of kSelBatchPages, batch k in slots [8 (k & 1), + 8) on kBarIdxK / kBarIdxK2, two in + // flight (a batch's MMAs go out once its 8 pages are in, and batch k + 2 once batch k's MMAs are done: rounds of 16 + // pages, the next round's pages after the round's MMAs, took 3.4-3.6 us per round at 900k against ~2.7 at the HBM + // cap, and the first 8 pages' epilogues now run while the next 8 load). + // The row shard: the block indices of the CTA's later pages into a table (past Q's 32 rows), loaded while Q is on + // its way: a batch's issue then reads them from shared memory (each issue's own block-index loads, one L2 round + // trip, sat on warp 0's path between a batch's MMAs and the next batch's requests). + int* const sBlk = reinterpret_cast(s0 + kSelQ8 + 32 * 128); + static_assert(32 * 128 + 4 * kIdxMaxPagesRs <= 256 * 128, "IdxSel block table"); + if (rounds) + { + idxSelIssuePages(ctx, a0, 0, min(nMine, kSelBatchPages), nPagesL, 0, barK); + idxSelIssuePages(ctx, a0, kSelBatchPages, min(max(nMine - kSelBatchPages, 0), kSelBatchPages), nPagesL, + kSelBatchPages, barK2); + for (int j = 2 * kSelBatchPages + tid; j < nMine; j += kThreads) + { + sBlk[j] = __ldg(ptr(ctx, kPIdxBlocks) + idxPageOf(cta, j, nPagesL)); + } + } + else + { + idxSelIssuePages(ctx, a0, 0, min(nMine, kIdxMaxPages), nPagesL); + } + // The logits epilogue once on a dummy tile while Q is on its way (its first real tile took ~2 us against ~0.7 for + // the next ones): TMEM columns no MMA of this stage has written yet, the results into the Q8 region (rewritten + // below after the barrier); the CTAs that ran an IdxQ unit did it in that unit's weight wait. (Dry runs of the + // top-k steps cost 6-15 us here, more than their real runs: dropped.) + { + uint8_t* scratch = s0 + kSelQ8; + if (cta >= kIdxQUnits && !rs) + { + tcFenceAfter(); + selEpilogue(*sTmem + 256u + (static_cast(32 * (warp & 3)) << 16) + + 128u * static_cast(warp >> 2), + sW, reinterpret_cast(scratch), 32 * (warp & 3) + lane, warp >> 2, 0.f, 0, nullptr, 0); + } + __syncthreads(); + } + // 2. Q in E4M3 (row R = 32 t + h of [256][128 B], SWIZZLE_128B) and the head weights: 16-byte vector c = tid + 256 + // i of Q (coalesced) to row c / 8, chunk (c % 8) ^ (row % 8); weight R = tid. The row shard: row tr's 32 heads only + // (rows 0-31, one vector per thread; weights 0-31). + if (rs) + { + // Row tr's BF16 Q (thread: head R = tid / 8, dims [16 (tid % 8), +16)) and the head's raw weight (thread 8 R); + // the head's E4M3 scale from its 128 values (its 8 threads), then the E4M3 chunk and the weight, as IdxQ did. + int const R = tid >> 3; + uint8_t const* qsrc = arena(ctx, kArIdxQb) + (tr * 32 * kIdxDim + 16 * tid) * 2; + uint8_t const* wsrc = arena(ctx, kArIdxW) + 4 * (32 * tr + R); + bool const wThread = (tid & 7) == 0; + uint4 v0 = ldPoll128(qsrc); + uint4 v1 = ldPoll128(qsrc + 16); + uint32_t wv = wThread ? ldPoll32(wsrc) : 0u; + uint64_t t0 = 0; + while (!warpAll((!wThread || wv != kEmpty) && full4(v0) && full4(v1))) + { + spinCheck(ctx, t0, 161, tid, 0); + if (wThread && wv == kEmpty) + { + wv = ldPoll32(wsrc); + } + if (!full4(v0)) + { + v0 = ldPoll128(qsrc); + } + if (!full4(v1)) + { + v1 = ldPoll128(qsrc + 16); + } + } + uint32_t const qw[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + float m = 0.f; +#pragma unroll + for (int j = 0; j < 8; ++j) + { + m = fmaxf(m, fmaxf(fabsf(bf16Lo(qw[j])), fabsf(bf16Hi(qw[j])))); + } + m = fmaxf(m, __shfl_xor_sync(0xffffffffu, m, 1)); + m = fmaxf(m, __shfl_xor_sync(0xffffffffu, m, 2)); + m = fmaxf(m, __shfl_xor_sync(0xffffffffu, m, 4)); + float const scale = ue8m0Scale(m); + float const inv = rcpApproxFtz(scale); + uint32_t q8[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + uint32_t const a = toE4m3x2(__fmul_rn(bf16Lo(qw[2 * j]), inv), __fmul_rn(bf16Hi(qw[2 * j]), inv)); + uint32_t const b = toE4m3x2(__fmul_rn(bf16Lo(qw[2 * j + 1]), inv), __fmul_rn(bf16Hi(qw[2 * j + 1]), inv)); + q8[j] = a | (b << 16); + } + *reinterpret_cast(s0 + kSelQ8 + R * 128 + 16 * ((tid & 7) ^ (R & 7))) + = make_uint4(q8[0], q8[1], q8[2], q8[3]); + if (wThread) + { + sW[R] = __fmul_rn(__fmul_rn(__uint_as_float(wv), scale), kIdxWeightScale); + } + } + else + { + uint8_t const* qsrc = arena(ctx, kArIdxQ8); + uint8_t const* wsrc = arena(ctx, kArIdxW) + 4 * tid; + uint4 v[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + v[i] = ldPoll128(qsrc + 16 * (tid + kThreads * i)); + } + uint32_t wv = ldPoll32(wsrc); + uint64_t t0 = 0; + while (true) + { + bool ok = wv != kEmpty; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + ok &= full4(v[i]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, 161, tid, 0); + if (wv == kEmpty) + { + wv = ldPoll32(wsrc); + } +#pragma unroll + for (int i = 0; i < 8; ++i) + { + if (!full4(v[i])) + { + v[i] = ldPoll128(qsrc + 16 * (tid + kThreads * i)); + } + } + } +#pragma unroll + for (int i = 0; i < 8; ++i) + { + int const c = tid + kThreads * i; + int const R = c >> 3; + *reinterpret_cast(s0 + kSelQ8 + R * 128 + 16 * ((c & 7) ^ (R & 7))) = v[i]; + } + sW[tid] = __uint_as_float(wv); + } + stampInput(ctx); + // 3. The CTA's pages [p0, p0 + np) in (slots [slot0, + np), on bar with parity ph); this step's keys among them + // from the arena (the cache rows may not have been written when the pages were read). + auto pagesIn = [&](int p0, int np, int slot0, uint32_t bar, uint32_t ph) + { + if (np > 0) + { + waitBar(ctx, bar, ph, 162); + } + if (tid < kTok * 8) + { + int const i = tid >> 3; + int const ch = tid & 7; + int const pf = L - kTok + i; + int const page = pf / kIdxPageKeys; + int const j = idxPageSlot(cta, page, nPagesL) - p0; + if (j >= 0 && j < np) + { + int const r = pf % kIdxPageKeys; + uint4 const v = poll4(ctx, arena(ctx, kArIdxKNew) + i * kIdxDim + 16 * ch, 163); + *reinterpret_cast(s0 + kSelK + (slot0 + j) * 8192 + r * 128 + 16 * (ch ^ (r & 7))) = v; + if (ch == 0) + { + sKs[(slot0 + j) * kIdxPageKeys + r] + = __uint_as_float(poll1(ctx, arena(ctx, kArIdxKNewS) + 4 * i, 164)); + } + } + } + fenceProxyAsyncSmem(); + tcFenceBefore(); + __syncthreads(); + tcFenceAfter(); + }; + if (!rounds) + { + pagesIn(0, min(nMine, kIdxMaxPages), 0, barK, 0u); + subStamp(ctx, 48); + } + if (!GLM52_LEAN && ctx.p[kPIdxDebug2] != 0 && cta == 0 && !rs) + { + uint8_t* dbg = ptr(ctx, kPIdxDebug2); + for (int v = tid; v < (64 + 256) * 8; v += kThreads) + { + int const r = v >> 3; + int const c = v & 7; + uint8_t const* src = r < 64 ? s0 + kSelK + r * 128 : s0 + kSelQ8 + (r - 64) * 128; + *reinterpret_cast(dbg + r * 128 + 16 * c) + = *reinterpret_cast(src + 16 * (c ^ ((r < 64 ? r : r - 64) & 7))); + } + } + // 4. Per tile of two pages (128 keys): D[key][32 t + h] = K8 Q8^T into TMEM buffer (tile & 1) (two tiles in + // flight), then the epilogue: warp w takes keys 32 (w % 4) + lane and rows 4 (w / 4) .. + 3 (one 128-column TMEM + // read); the tile's logits, FP32 [8][128] (masked: -inf), into its first 4 KiB (its MMAs are done). + uint32_t const tmem = *sTmem; + bool const dbg = !GLM52_LEAN && ctx.p[kPIdxDebug] != 0; + bool const dbg2 = !GLM52_LEAN && ctx.p[kPIdxDebug2] != 0 && cta == 0 && !rs; + int64_t const dbgLd = ctx.p[kPIdxDebugLd]; + if (rounds) + { + // The row shard: N = 32, so every tile of a batch has 32 TMEM columns of its own (tile t at 32 t, batch k's + // tiles [4 (k & 1), + 4)): the batch's MMAs at once (one commit, kBarIdxK + 1 / + 2 by k & 1), then the + // epilogues two tiles at a time, warps 0-3 the even tiles and warps 4-7 the odd ones, the logits into the row's + // array (local key k at kSelLog + 4 k). (Two buffers of 256 columns, as for the 8 rows, took ~0.65 us per tile: + // each tile's MMAs waited for the epilogue two tiles back, and warps 4-7 idled.) + int const q = warp & 3; + int const kt = 32 * q + lane; + float* const sLog = reinterpret_cast(s0 + kSelLog); +#if GLM52_TOPK_RSH + // RS-H: every valid key counted per bin in shared memory (rsTopK; zeroed at the stage's start). + uint32_t const rsLh = a0 + kRsLh; +#else + // The thread's two largest keys as order keys (selKeyAt's: -inf = 0), for the bound's union: local key 128 i + + // kt is lane kt % 32 = lane's (idxUnionBound takes the two largest of the 8 warps' pairs per lane). + uint32_t lm1 = 0; + uint32_t lm2 = 0; +#endif + int const nBatches = (nMine + kSelBatchPages - 1) / kSelBatchPages; + // Batch k: its pages in (all threads), then its MMAs (warp 1, one commit on barS0 / barS1 by k & 1). + auto batchIn = [&](int k) + { + int const p0 = kSelBatchPages * k; + int const np = min(nMine - p0, kSelBatchPages); + int const nt = (np + 1) / 2; + int const sb = k & 1; + int const t0 = kSelBatchPages / 2 * sb; + pagesIn(p0, np, kSelBatchPages * sb, sb ? barK2 : barK, static_cast((k >> 1) & 1)); + if (warp == 1) + { +#if GLM52_SEL_BATCH_MMA + selBatchMma(tmem + 32u * static_cast(t0), a0 + kSelK + t0 * 16384, a0 + kSelQ8, + sb ? barS1 : barS0, nt); +#else + for (int i = 0; i < nt; ++i) + { + selTileMma(tmem + 32u * static_cast(t0 + i), a0 + kSelK + (t0 + i) * 16384, a0 + kSelQ8, + sb ? barS1 : barS0, kIdescF8N32, i + 1 == nt); + } +#endif + } + }; + if (nBatches > 0) + { + batchIn(0); + subStamp(ctx, 48); + } + // Per batch k: its scales into registers, its MMAs done -> batch k + 2's pages into its slots, then batch + // k + 1's pages and MMAs (they run during this batch's epilogues), then its epilogues (warps 0-3 the even + // tiles, 4-7 the odd ones). (Batch k + 1's MMAs after batch k's epilogues left the tensor cores idle through + // them.) +#pragma unroll 1 + for (int k = 0; k < nBatches; ++k) + { + int const p0 = kSelBatchPages * k; + int const np = min(nMine - p0, kSelBatchPages); + int const nt = (np + 1) / 2; + int const sb = k & 1; // the batch's slots [8 sb, + 8), tiles [4 sb, + 4) + int const t0 = kSelBatchPages / 2 * sb; + float ks[2]; +#pragma unroll + for (int j = 0; j < 2; ++j) + { + int const i = (warp >> 2) + 2 * j; + int const page = p0 + 2 * i + (kt >> 6); + ks[j] + = i < nt && page < nMine ? sKs[(kSelBatchPages * sb + page - p0) * kIdxPageKeys + (kt & 63)] : 0.f; + } + waitBar(ctx, sb ? barS1 : barS0, static_cast((k >> 1) & 1), 165); + tcFenceAfter(); + if (k == 0) + { + subStamp(ctx, 49); + } + else if (k < 4) + { + subStamp(ctx, 47 + 3 * k); // 50, 53, 56: batch k's MMAs done + } + if (k + 2 < nBatches) + { + int const nf = p0 + 2 * kSelBatchPages; + __syncthreads(); // every thread's scales of the batch read + idxSelIssuePages(ctx, a0, nf, min(nMine - nf, kSelBatchPages), nPagesL, kSelBatchPages * sb, + sb ? barK2 : barK, sBlk); + } + if (k + 1 < nBatches) + { + batchIn(k + 1); + if (k < 3) + { + subStamp(ctx, 51 + 3 * k); // 51, 54, 57: batch k + 1's pages in, its MMAs issued + } + } +#pragma unroll + for (int j = 0; j < 2; ++j) + { + int const i = (warp >> 2) + 2 * j; + if (i < nt) + { + int const page = p0 + 2 * i + (kt >> 6); + int const pos = idxPageOf(cta, page, nPagesL) * kIdxPageKeys + (kt & 63); + bool const inPage = page < nMine; + float const lgt = selEpilogue1( + tmem + 32u * static_cast(t0 + i) + (static_cast(32 * q) << 16), sW, + sLog + 64 * p0 + 128 * i, kt, ks[j], inPage && pos <= L - kTok + tr, + dbg && inPage && pos < L && pos < dbgLd ? ptr(ctx, kPIdxDebug) + tr * dbgLd + pos + : nullptr); +#if GLM52_TOPK_RSH + rsCount(rsLh, lgt); +#else + top2Insert(lm1, lm2, lgt == -INFINITY ? 0u : orderKey(lgt)); +#endif + } + } + if (k < 3) + { + subStamp(ctx, 52 + 3 * k); // 52, 55, 58: batch k's epilogues done (thread 0's) + } + } +#if !GLM52_TOPK_RSH + sts32(ctx.smemBase + kSmemRed + kSelLaneMax + 4 * (32 * warp + lane), lm1); + sts32(ctx.smemBase + kSmemRed + kSelLaneMax2 + 4 * (32 * warp + lane), lm2); +#endif + tcFenceBefore(); + } + else + { + uint32_t phases = 0; // bit b: the parity buffer b's barrier waits for next + auto issue = [&](int i) + { + if (warp == 1) + { + selTileMma(tmem + 256u * static_cast(i & 1), a0 + kSelK + i * 16384, a0 + kSelQ8, + barS0 + 8 * (i & 1), rs ? kIdescF8N32 : kIdescF8); + } + }; + for (int i = 0; i < 2 && i < nTiles; ++i) + { + issue(i); + } + int const q = warp & 3; + int const hh = warp >> 2; + for (int i = 0; i < nTiles; ++i) + { + waitBar(ctx, barS0 + 8 * (i & 1), (phases >> (i & 1)) & 1u, 165); + phases ^= 1u << (i & 1); + tcFenceAfter(); + if (i < 6) + { + subStamp(ctx, 49 + 2 * i); + } + int const kt = 32 * q + lane; + int const page = 2 * i + (kt >> 6); + int const pos = idxPageOf(cta, page, nPagesL) * kIdxPageKeys + (kt & 63); + bool const inPage = page < nMine; + float const ks = inPage ? sKs[page * kIdxPageKeys + (kt & 63)] : 0.f; + uint32_t const taddr + = tmem + 256u * static_cast(i & 1) + (static_cast(32 * q) << 16) + 128u * hh; + if (dbg2 && i == 0) + { + // Before the read below: no accumulator registers live across the call. + dumpTileD( + taddr, reinterpret_cast(ptr(ctx, kPIdxDebug2) + 320 * 128) + kt * 256 + 128 * hh); + } + // Every MMA of tile i is done: its K rows (read by nothing else now) take its logits. + if (rs) + { + // Row tr only (N = 32): warps 0-3, the tile's logits into the row's array (kSelLog). + if (warp < 4) + { + selEpilogue1(tmem + 256u * static_cast(i & 1) + (static_cast(32 * q) << 16), sW, + reinterpret_cast(s0 + kSelLog) + 128 * i, kt, ks, inPage && pos <= L - kTok + tr, + dbg && inPage && pos < L && pos < dbgLd ? ptr(ctx, kPIdxDebug) + tr * dbgLd + pos + : nullptr); + } + } + else + { + selEpilogue(taddr, sW, reinterpret_cast(s0 + kSelK + i * 16384), kt, hh, ks, + inPage ? pos - (L - kTok) : kTok, + dbg && inPage && pos < L && pos < dbgLd ? ptr(ctx, kPIdxDebug) + pos : nullptr, dbgLd); + } + // Every thread has read buffer (i & 1): it takes tile i + 2. + tcFenceBefore(); + __syncthreads(); + if (i < 6) + { + subStamp(ctx, 50 + 2 * i); + } + if (i + 2 < nTiles) + { + issue(i + 2); + } + } + } + __syncthreads(); + subStamp(ctx, 43); +#if GLM52_TOPK_RSH + // 5. RS-H: row tr's top-2048 from its histogram, into the top-k exchange (rank 2 tr publishes it to every rank). + (void) topkBase; + rsTopK(ctx, a0, s0, 4 * nTiles, L, tr, cand + static_cast(tr) * kCandCap, + reinterpret_cast(ptr(ctx, kPTkMc) + static_cast(ctx.parity) * kTkBytes) + tr * kTopK); +#else + // 5. Top-2048 of every row (idxTopK); kBarIdxK's parity: its page loads so far (the row shard's even batches). + int const nLoadsK = rounds ? ((nMine + kSelBatchPages - 1) / kSelBatchPages + 1) / 2 : (nMine > 0 ? 1 : 0); + idxTopK(ctx, a0, s0, nTiles, L, cand, topkBase, barK, static_cast(nLoadsK & 1)); +#endif +} + +// Position of the CTA's key t in the k order of the P.V MMA (production flash decode): within every 16 keys, k slots +// 4c .. 4c + 3 hold keys {2c, 2c + 1, 2c + 8, 2c + 9}. +__device__ __forceinline__ int permutedSlot(int t) +{ + int const u = t & 15; + int const slot16 = (u < 8) ? ((u >> 1) * 4 + (u & 1)) : (((u - 8) >> 1) * 4 + 2 + (u & 1)); + return (t & ~15) + slot16; +} + +// Sparse attention of row t over keys [128 b, +128) of its 2048: partial O, max and sum per head. +// kFlagSliceMerge: attention unit (t, blk), after its block, merges dims [32 blk, +32) of row t's 4 heads (64 threads: +// head tid / 16, two dims each) from the 16 blocks' partials (this unit is their only reader: it empties them) and +// stats (every unit of the row reads them: in the arena, which its copy's clear units empty), stageMerge's arithmetic. +// No merge stage: o_proj takes O from the 128 attention units, each consumer reads 8 KB instead of 32 KB, and nothing +// is cleared behind o_proj's stores. +__device__ __noinline__ void attnSliceMerge(Ctx const& ctx, int t, int blk) +{ + int const tid = threadIdx.x; + if (tid >= kHeads * 16) + { + return; + } + int const h = tid >> 4; + int const d = 32 * blk + 2 * (tid & 15); + uint8_t const* const sts = arena(ctx, kArPStats) + (t * kAttnBlocks * kHeads + h) * 8; + constexpr int kStatStride = kHeads * 8; + uint8_t* const base = reinterpret_cast( + ptr(ctx, kPPartO) + (static_cast(t * kAttnBlocks) * kHeads + h) * kDv + d); + constexpr int kStride = kHeads * kDv * 4; + uint2 o[kAttnBlocks]; + uint2 sb[kAttnBlocks]; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + o[b] = ldPoll64(base + b * kStride); + sb[b] = ldPoll64(sts + b * kStatStride); + } + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + ok &= full2(o[b]) & full2(sb[b]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, 108, t, h); +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + if (!full2(o[b])) + { + o[b] = ldPoll64(base + b * kStride); + } + if (!full2(sb[b])) + { + sb[b] = ldPoll64(sts + b * kStatStride); + } + } + } + float merged = -INFINITY; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + merged = fmaxf(merged, __uint_as_float(sb[b].x)); + } + float mergedSum = 0.f; + float acc0 = 0.f; + float acc1 = 0.f; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + float const mb = __uint_as_float(sb[b].x); + float const alpha = mb == -INFINITY ? 0.f : exp2f(mb - merged); + mergedSum += alpha * __uint_as_float(sb[b].y); + acc0 += alpha * __uint_as_float(o[b].x); + acc1 += alpha * __uint_as_float(o[b].y); + } + float const v0 = mergedSum > 0.f ? acc0 / (mergedSum * kPScale) * 1.f : 0.f; + float const v1 = mergedSum > 0.f ? acc1 / (mergedSum * kPScale) * 1.f : 0.f; + uint32_t const ow = lamport(packBf16(v0, v1)); + st32(arena(ctx, kArO) + ((t * kHeads + h) * kDv + d) * 2, ow); +#if GLM52_IO_MODE == 1 || GLM52_IO_MODE == 3 +#pragma unroll + for (int r = 0; r < 3; ++r) + { + st32(arena(ctx, kArORep + r * kORepBytes) + ((t * kHeads + h) * kDv + d) * 2, ow); + } +#endif +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + st64(base + b * kStride, make_uint2(kEmpty, kEmpty)); + } +} + +__device__ __noinline__ void stageAttn( + Ctx const& ctx, int unit, uint32_t wOff, int const* kvLenPtr, int const* blockTable, int const* topk) +{ + int const t = unit / kAttnBlocks; + int const blk = unit % kAttnBlocks; + int const tid = threadIdx.x; + int const warp = tid >> 5; + int const lane = tid & 31; + uint8_t* sKv = ctx.smem + wOff; + uint8_t* sQ = sKv + kAttnKeys * kSmemRowBytes; + uint8_t* sP = sQ + 8 * kSmemRowBytes; + float* sRedMax = reinterpret_cast(sP + 8 * kPRowBytes); + float* sRedSum = sRedMax + kWarps * 8; + int* sFreshAll = reinterpret_cast(sRedSum + kWarps * 8); + uint32_t const barGather = ctx.smemBase + kSmemSmall + kSmallBars + (kMaxRoles + warp) * 8; + + // The warp's 16 rows: rows of earlier steps from the pool (bulk copies), this step's rows from the arena. A + // leader's top-k is this launch's: its selectors' copy in the arena (cleared to kEmpty), polled word by word (no + // count to wait for, no reload). Meanwhile the block table goes into L1 (ld.global.nc, as the lookups below): each + // index then finds its page there instead of one more L2 round trip. + int const L = __ldg(kvLenPtr); + int const slot = blk * kAttnKeys + warp * 16 + lane; + int const numValid = min(L - (kTok - 1 - t), kTopK); + auto issueOproj = [&]() + { + if (tid == 0) + { + // The o_proj weights held back for these gathers: the SM takes in the rows first (the 128 KB request ahead + // of them held the attention's input back ~0.4 us per plain layer, mk-v149a). + int* sRoles = reinterpret_cast(ctx.smem + kSmemSmall + kSmallRoles); +#pragma unroll 1 + for (int j = 0; j < kMaxRoles && sRoles[j] != 0; ++j) + { + if (sRoles[2 * kMaxRoles + j] == -4) + { +#if GLM52_SETUP_MODE == 0 + issueUnitWeights( + ctx, sRoles[j], sRoles[kMaxRoles + j], ctx.smemBase + kSmemSmall + kSmallBars + 8 * j); +#else + issueUnitWeights(ctx, j, ctx.smemBase + kSmemSmall + kSmallBars + 8 * j); +#endif + sRoles[2 * kMaxRoles + j] = -3; + } + } + } + }; + // A block past the row's keys (contexts below 2048 + 7): no indices, gathers or Q; the partials of an empty block + // (O = 0, max -inf, sum 0). (Gathering row 0 of the pool for every slot past the keys put ~16k copies of one row on + // one L2 slice: the attention took ~10 us more per layer at 32 keys than at 32k, mk-t10.) + if (!(kSpecializedStaticFlags & kFlagSliceMerge) && blk * kAttnKeys >= numValid) + { + issueOproj(); + float* partO = ptr(ctx, kPPartO) + static_cast(t * kAttnBlocks + blk) * kHeads * kDv; + for (int i = tid; i < kHeads * kDv / 2; i += kThreads) + { + st64(partO + 2 * i, make_uint2(0u, 0u)); + } + if (tid < kHeads) + { + st64(ptr(ctx, kPPartStats) + ((t * kAttnBlocks + blk) * kHeads + tid) * 2, + make_uint2(__float_as_uint(-INFINITY), 0u)); + } + stampInput(ctx); + return; + } + int P = -1; + if (ctx.leader) + { + int const nb = min((L + kIdxPageKeys - 1) / kIdxPageKeys, static_cast(ctx.p[kPMaxBlocks])); +#pragma unroll 1 + for (int b = tid; b < nb; b += kThreads) + { + uint32_t dummy; + asm volatile("ld.global.nc.u32 %0, [%1];" : "=r"(dummy) : "l"(blockTable + b)); + } + // The row shard: every row from this GPU's copy of the top-k exchange (row t published by rank 2 t). + bool const rs = (GLM52_FLAGS(ctx) & kFlagIdxAllRows) == 0; + bool const all = idxAllKeys(L); + if (all) + { + P = slot < numValid ? slot : -1; + } + else + { + uint32_t const* src = (rs ? reinterpret_cast( + ptr(ctx, kPTkUc) + static_cast(ctx.parity) * kTkBytes) + : arena(ctx, kArIdxTopk)) + + t * kTopK + slot; + uint32_t w = lane < 16 ? ldPoll32(src) : 0u; + uint64_t t0 = 0; + while (!warpAll(lane >= 16 || w != kEmpty)) + { + spinCheck(ctx, t0, 170, t, slot); + if (lane < 16 && w == kEmpty) + { + w = ldPoll32(src); + } + } + P = static_cast(w); + } + if ((rs || all) && lane < 16) + { + // The leader's top-k buffer (the next plain layers' input): under the row shard, or with every key, no + // selector writes it. + const_cast(topk)[t * kTopK + slot] = P; + } + } + else if (lane < 16) + { + P = __ldcg(topk + t * kTopK + slot); + } + bool valid = false; + bool fresh = false; + int freshIdx = 0; + int64_t row = 0; + if (lane < 16) + { + int const block = P >> 6; + int page = -1; + if (P >= 0 && block < static_cast(ctx.p[kPMaxBlocks])) + { + page = __ldg(blockTable + block); + } + valid = slot < numValid && page >= 0; + fresh = valid && P >= L - kTok; + freshIdx = P - (L - kTok); + row = valid ? static_cast(page) * ctx.p[kPStrideRows] + (P & 63) + ctx.p[kPLayerRows] : 0; + } + uint32_t const validMask = __ballot_sync(0xffffffffu, valid); + uint32_t const freshMask = __ballot_sync(0xffffffffu, fresh); + uint8_t const* kvBase = ptr(ctx, kPKvBase); + if (lane == 0) + { + mbarExpectTx(barGather, __popc(validMask & ~freshMask) * kRowBytes); + } + __syncwarp(); + if (lane < 16 && !valid) + { + // A slot past the row's keys: its row zeroed (its P is 0, its V must be finite), not gathered. + uint4* z = reinterpret_cast(sKv + (warp * 16 + lane) * kSmemRowBytes); +#pragma unroll 4 + for (int i = 0; i < kRowBytes / 16; ++i) + { + z[i] = make_uint4(0, 0, 0, 0); + } + } + if (valid && !fresh) + { + fenceProxyAsyncSmem(); + // Read once per layer: L2 evict-first (see the indexer's key pages). + bulkCopyHint(smemU32(sKv + (warp * 16 + lane) * kSmemRowBytes), kvBase + row * kRowBytes, kRowBytes, barGather, + policyEvictFirst()); + } + subStamp(ctx, 5); + issueOproj(); + + // Q of row t (4 heads x 576 E4M3, rows 0-3 of sQ; rows 4-7 zero) and this step's rows among the warp's (from the + // arena; vector j of the warp's fresh rows is lane j mod 32's), polled in one round trip (the fresh rows are in + // long before Q). + { + constexpr int kVecs = kRowBytes / 16; + constexpr int kMaxPerLane = (kTok * kVecs + 31) / 32; + for (int i = kHeads * kVecs + tid; i < 8 * kVecs; i += kThreads) + { + *reinterpret_cast(sQ + (i / kVecs) * kSmemRowBytes + 16 * (i % kVecs)) = make_uint4(0, 0, 0, 0); + } + int* sFresh = sFreshAll + warp * 16; + if (fresh) + { + int const rank = __popc(freshMask & ((1u << lane) - 1u)); + sFresh[2 * rank] = lane; + sFresh[2 * rank + 1] = freshIdx; + } + __syncwarp(); + int const nVec = __popc(freshMask) * kVecs; + uint8_t const* kvNew = arena(ctx, kArKv); + auto srcOf = [&](int j) + { + int const r = j / kVecs; + return kvNew + sFresh[2 * r + 1] * kRowBytes + 16 * (j - kVecs * r); + }; + static_assert(kHeads * kVecs <= kThreads, "one Q vector per thread"); + uint8_t const* qSrc + = arena(ctx, kArQ8) + t * kQbRows + (tid / kVecs) * kRowBytes + 16 * (tid % kVecs); + bool const hasQ = tid < kHeads * kVecs; + uint4 qv = hasQ ? ldPoll128(qSrc) : make_uint4(0, 0, 0, 0); + uint4 v[kMaxPerLane]; +#pragma unroll + for (int i = 0; i < kMaxPerLane; ++i) + { + int const j = lane + 32 * i; + v[i] = j < nVec ? ldPoll128(srcOf(j)) : make_uint4(0, 0, 0, 0); + } + uint64_t t0 = 0; + while (true) + { + bool ok = full4(qv); +#pragma unroll + for (int i = 0; i < kMaxPerLane; ++i) + { + ok &= full4(v[i]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, 105, t, nVec); + if (!full4(qv)) + { + qv = ldPoll128(qSrc); + } +#pragma unroll + for (int i = 0; i < kMaxPerLane; ++i) + { + if (!full4(v[i])) + { + v[i] = ldPoll128(srcOf(lane + 32 * i)); + } + } + } + if (hasQ) + { + *reinterpret_cast(sQ + (tid / kVecs) * kSmemRowBytes + 16 * (tid % kVecs)) = qv; + } +#pragma unroll + for (int i = 0; i < kMaxPerLane; ++i) + { + int const j = lane + 32 * i; + if (j < nVec) + { + int const r = j / kVecs; + *reinterpret_cast(sKv + (warp * 16 + sFresh[2 * r]) * kSmemRowBytes + 16 * (j - kVecs * r)) + = v[i]; + } + } + } + subStamp(ctx, 6); + __syncthreads(); + stampInput(ctx); + subStamp(ctx, 7); + + int const g = lane / 4; + int const c = lane % 4; + uint32_t const qAddr = smemU32(sQ + (lane % 8) * kSmemRowBytes + 16 * ((lane / 8) % 2)); + waitBar(ctx, barGather, 0, 107); + subStamp(ctx, 8); + + float const scaleSoftmaxLog2 = 0.0901684f; // (1 / 16) * log2(e) + float s[4]; + { + // kFlagAttnIlp: four accumulator chains (each MMA's two halves into different ones). + float acc[4][4] = {}; + uint32_t const kAddr + = smemU32(sKv + (warp * 16 + lane % 8 + 8 * ((lane / 8) % 2)) * kSmemRowBytes + 16 * (lane / 16)); + // Two K steps per iteration (one per accumulator); the query fragments are unpacked in the loop (a fully + // unrolled loop was ~7 KiB of code, fetched cold once per layer). +#pragma unroll 2 + for (int kd = 0; kd < kRowBytes / 32; ++kd) + { + uint32_t b0, b1; + ldmatrixX2(qAddr + 32 * kd, b0, b1); + uint32_t a0, a1, a2, a3; + ldmatrixX4(kAddr + 32 * kd, a0, a1, a2, a3); + mmaF16(acc[kd % 2], e4m3x2ToHalf2(a0), e4m3x2ToHalf2(a1), e4m3x2ToHalf2(a2), e4m3x2ToHalf2(a3), + e4m3x2ToHalf2(b0), e4m3x2ToHalf2(b1)); + mmaF16(acc[2 + kd % 2], e4m3x2ToHalf2(a0 >> 16), e4m3x2ToHalf2(a1 >> 16), e4m3x2ToHalf2(a2 >> 16), + e4m3x2ToHalf2(a3 >> 16), e4m3x2ToHalf2(b0 >> 16), e4m3x2ToHalf2(b1 >> 16)); + } + bool const valid0 = (validMask >> g) & 1u; + bool const valid1 = (validMask >> (g + 8)) & 1u; + s[0] = valid0 ? ((acc[0][0] + acc[1][0]) + (acc[2][0] + acc[3][0])) * scaleSoftmaxLog2 : -INFINITY; + s[1] = valid0 ? ((acc[0][1] + acc[1][1]) + (acc[2][1] + acc[3][1])) * scaleSoftmaxLog2 : -INFINITY; + s[2] = valid1 ? ((acc[0][2] + acc[1][2]) + (acc[2][2] + acc[3][2])) * scaleSoftmaxLog2 : -INFINITY; + s[3] = valid1 ? ((acc[0][3] + acc[1][3]) + (acc[2][3] + acc[3][3])) * scaleSoftmaxLog2 : -INFINITY; + } + subStamp(ctx, 9); + float max0 = fmaxf(s[0], s[2]); + float max1 = fmaxf(s[1], s[3]); +#pragma unroll + for (int off = 4; off < 32; off *= 2) + { + max0 = fmaxf(max0, __shfl_xor_sync(0xffffffffu, max0, off)); + max1 = fmaxf(max1, __shfl_xor_sync(0xffffffffu, max1, off)); + } + if (g == 0) + { + sRedMax[warp * 8 + 2 * c] = max0; + sRedMax[warp * 8 + 2 * c + 1] = max1; + } + __syncthreads(); + float ctaMax0 = -INFINITY; + float ctaMax1 = -INFINITY; +#pragma unroll + for (int w = 0; w < kWarps; ++w) + { + ctaMax0 = fmaxf(ctaMax0, sRedMax[w * 8 + 2 * c]); + ctaMax1 = fmaxf(ctaMax1, sRedMax[w * 8 + 2 * c + 1]); + } + float const offset0 = ctaMax0 == -INFINITY ? 0.f : ctaMax0; + float const offset1 = ctaMax1 == -INFINITY ? 0.f : ctaMax1; + { + float const p00 = exp2f(s[0] - offset0); + float const p01 = exp2f(s[1] - offset1); + float const p10 = exp2f(s[2] - offset0); + float const p11 = exp2f(s[3] - offset1); + int const slot0 = permutedSlot(warp * 16 + g); + int const slot1 = permutedSlot(warp * 16 + g + 8); + sP[(2 * c) * kPRowBytes + slot0] = toE4m3(p00 * kPScale); + sP[(2 * c + 1) * kPRowBytes + slot0] = toE4m3(p01 * kPScale); + sP[(2 * c) * kPRowBytes + slot1] = toE4m3(p10 * kPScale); + sP[(2 * c + 1) * kPRowBytes + slot1] = toE4m3(p11 * kPScale); + float sum0 = p00 + p10; + float sum1 = p01 + p11; +#pragma unroll + for (int off = 4; off < 32; off *= 2) + { + sum0 += __shfl_xor_sync(0xffffffffu, sum0, off); + sum1 += __shfl_xor_sync(0xffffffffu, sum1, off); + } + if (g == 0) + { + sRedSum[warp * 8 + 2 * c] = sum0; + sRedSum[warp * 8 + 2 * c + 1] = sum1; + } + } + __syncthreads(); + subStamp(ctx, 10); + // O^T = V^T P^T: warp w owns the 16-column output blocks 4w .. 4w + 3 (production flash decode). + float* partO = ptr(ctx, kPPartO) + static_cast(t * kAttnBlocks + blk) * kHeads * kDv; + { + uint32_t const pAddr = smemU32(sP + (lane % 8) * kPRowBytes + 16 * ((lane / 8) % 2)); + uint32_t const vAddr = smemU32(sKv + lane * kSmemRowBytes + 16 * (4 * warp)); + uint32_t pb[kAttnKeys / 32][4]; +#pragma unroll + for (int kStep = 0; kStep < kAttnKeys / 32; ++kStep) + { + uint32_t b0, b1; + ldmatrixX2(pAddr + 32 * kStep, b0, b1); + pb[kStep][0] = e4m3x2ToHalf2(b0); + pb[kStep][1] = e4m3x2ToHalf2(b1); + pb[kStep][2] = e4m3x2ToHalf2(b0 >> 16); + pb[kStep][3] = e4m3x2ToHalf2(b1 >> 16); + } + // kFlagAttnIlp: two output blocks per iteration, two independent accumulator chains. +#pragma unroll 1 + for (int bi0 = 0; bi0 < 4; bi0 += 2) + { + float ob2[2][4] = {}; +#pragma unroll + for (int kStep = 0; kStep < kAttnKeys / 32; ++kStep) + { +#pragma unroll + for (int u = 0; u < 2; ++u) + { + uint32_t r0, r1, r2, r3; + ldmatrixX4Trans(vAddr + 32 * kStep * kSmemRowBytes + 16 * (bi0 + u), r0, r1, r2, r3); + uint32_t const a0 = __byte_perm(r0, r1, 0x6420); + uint32_t const a1 = __byte_perm(r0, r1, 0x7531); + uint32_t const a2 = __byte_perm(r2, r3, 0x6420); + uint32_t const a3 = __byte_perm(r2, r3, 0x7531); + mmaF16(ob2[u], e4m3x2ToHalf2(a0), e4m3x2ToHalf2(a1), e4m3x2ToHalf2(a2), e4m3x2ToHalf2(a3), + pb[kStep][0], pb[kStep][1]); + mmaF16(ob2[u], e4m3x2ToHalf2(a0 >> 16), e4m3x2ToHalf2(a1 >> 16), e4m3x2ToHalf2(a2 >> 16), + e4m3x2ToHalf2(a3 >> 16), pb[kStep][2], pb[kStep][3]); + } + } +#pragma unroll + for (int u = 0; u < 2; ++u) + { + float const* ob = ob2[u]; + int const col = 16 * (4 * warp + bi0 + u) + 2 * g; + if (c < 2) + { + st64(partO + (2 * c) * kDv + col, + make_uint2(lamport(__float_as_uint(ob[0])), lamport(__float_as_uint(ob[2])))); + st64(partO + (2 * c + 1) * kDv + col, + make_uint2(lamport(__float_as_uint(ob[1])), lamport(__float_as_uint(ob[3])))); + } + } + } + } + if (tid < kHeads) + { + float m = -INFINITY; + float sm = 0.f; +#pragma unroll + for (int w = 0; w < kWarps; ++w) + { + m = fmaxf(m, sRedMax[w * 8 + tid]); + sm += sRedSum[w * 8 + tid]; + } + float* st = (kSpecializedStaticFlags & kFlagSliceMerge) + ? reinterpret_cast(arena(ctx, kArPStats)) + ((t * kAttnBlocks + blk) * kHeads + tid) * 2 + : ptr(ctx, kPPartStats) + ((t * kAttnBlocks + blk) * kHeads + tid) * 2; + st64(st, make_uint2(lamport(__float_as_uint(m)), lamport(__float_as_uint(sm)))); + } + if constexpr ((kSpecializedStaticFlags & kFlagSliceMerge) != 0) + { + subStamp(ctx, 135); // the block done; 11: its slice merged + attnSliceMerge(ctx, t, blk); + } + subStamp(ctx, 11); +} + +#if (GLM52_STATIC_FLAGS >> 39) & 1 +// kFlagMergeDry (a separate source so that the default build stays as it was). +// Merge of the 16 partials of (row t, head h): o in BF16. This unit is the partials' only reader: it empties them. +__device__ __noinline__ void stageMerge(Ctx const& ctx, int unit) +{ + int const t = unit / kHeads; + int const h = unit % kHeads; + int const tid = threadIdx.x; + float* sStats = reinterpret_cast(ctx.smem + kSmemRed); + // Thread tid < 32 also polls stats word tid (block tid / 2: max, sum), with its partials in one round trip. + bool const hasStat = tid < 2 * kAttnBlocks; + float* st = ptr(ctx, kPPartStats) + ((t * kAttnBlocks + tid / 2) * kHeads + h) * 2 + (tid & 1); + int const d = 2 * tid; + uint8_t* base = reinterpret_cast( + ptr(ctx, kPPartO) + (static_cast(t * kAttnBlocks) * kHeads + h) * kDv + d); + constexpr int kStride = kHeads * kDv * 4; + bool const clearFirst = (GLM52_FLAGS(ctx) & kFlagMergeClearFirst) != 0; + // The partials back to empty for a later launch: after this unit's output (issued before it, the 4,096 clearing + // stores held the output up behind them; o_proj saw it 0.7-2 us after the last merge, mk-v105a). + auto clearPartials = [&]() + { + if (hasStat) + { + st32(st, kEmpty); + } +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + st64(base + b * kStride, make_uint2(kEmpty, kEmpty)); + } + }; + // The unit's code once on zeros (no stores) while the partials are still far off, so the real pass after the poll + // runs from a warm instruction cache (the same loop body serves both passes). + uint32_t sw = 0u; + uint2 o[kAttnBlocks]; +#pragma unroll 1 + for (int pass = 0; pass < 2; ++pass) + { + bool const dry = pass == 0; + if (dry) + { +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + o[b] = make_uint2(0u, 0u); + } + } + else + { + sw = hasStat ? ldPoll32(st) : 0u; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + o[b] = ldPoll64(base + b * kStride); + } + uint64_t t0 = 0; + while (true) + { + bool ok = sw != kEmpty; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + ok &= full2(o[b]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, 108, t, h); + if (sw == kEmpty) + { + sw = ldPoll32(st); + } +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + if (!full2(o[b])) + { + o[b] = ldPoll64(base + b * kStride); + } + } + } + } + if (hasStat) + { + sStats[tid] = __uint_as_float(sw); + } + if (!dry && clearFirst) + { + clearPartials(); + } + __syncthreads(); + if (!dry) + { + stampInput(ctx); + } + float merged = -INFINITY; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + merged = fmaxf(merged, sStats[2 * b]); + } + float mergedSum = 0.f; + float acc0 = 0.f; + float acc1 = 0.f; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + float const mb = sStats[2 * b]; + float const alpha = mb == -INFINITY ? 0.f : exp2f(mb - merged); + mergedSum += alpha * sStats[2 * b + 1]; + acc0 += alpha * __uint_as_float(o[b].x); + acc1 += alpha * __uint_as_float(o[b].y); + } + float const v0 = mergedSum > 0.f ? acc0 / (mergedSum * kPScale) * 1.f : 0.f; + float const v1 = mergedSum > 0.f ? acc1 / (mergedSum * kPScale) * 1.f : 0.f; + uint32_t const ow = lamport(packBf16(v0, v1)); + if (dry) + { + // The dry pass's reads of the stats end before the real pass writes them. + __syncthreads(); + continue; + } + st32(arena(ctx, kArO) + ((t * kHeads + h) * kDv + d) * 2, ow); +#if GLM52_IO_MODE == 1 || GLM52_IO_MODE == 3 +#pragma unroll + for (int r = 0; r < 3; ++r) + { + st32(arena(ctx, kArORep + r * kORepBytes) + ((t * kHeads + h) * kDv + d) * 2, ow); + } +#endif + if (!clearFirst) + { + clearPartials(); + } + } +} +#else +// Merge of the 16 partials of (row t, head h): o in BF16. This unit is the partials' only reader: it empties them. +__device__ __noinline__ void stageMerge(Ctx const& ctx, int unit) +{ + int const t = unit / kHeads; + int const h = unit % kHeads; + int const tid = threadIdx.x; + float* sStats = reinterpret_cast(ctx.smem + kSmemRed); + // Thread tid < 32 also polls stats word tid (block tid / 2: max, sum), with its partials in one round trip. + bool const hasStat = tid < 2 * kAttnBlocks; + float* st = ptr(ctx, kPPartStats) + ((t * kAttnBlocks + tid / 2) * kHeads + h) * 2 + (tid & 1); + uint32_t sw = hasStat ? ldPoll32(st) : 0u; + int const d = 2 * tid; + uint8_t* base = reinterpret_cast( + ptr(ctx, kPPartO) + (static_cast(t * kAttnBlocks) * kHeads + h) * kDv + d); + constexpr int kStride = kHeads * kDv * 4; + uint2 o[kAttnBlocks]; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + o[b] = ldPoll64(base + b * kStride); + } + { + uint64_t t0 = 0; + while (true) + { + bool ok = sw != kEmpty; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + ok &= full2(o[b]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, 108, t, h); + if (sw == kEmpty) + { + sw = ldPoll32(st); + } +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + if (!full2(o[b])) + { + o[b] = ldPoll64(base + b * kStride); + } + } + } + } + if (hasStat) + { + sStats[tid] = __uint_as_float(sw); + } + bool const clearFirst = (GLM52_FLAGS(ctx) & kFlagMergeClearFirst) != 0; + // The partials back to empty for a later launch: after this unit's output (issued before it, the 4,096 clearing + // stores held the output up behind them; o_proj saw it 0.7-2 us after the last merge, mk-v105a). + auto clearPartials = [&]() + { + if (hasStat) + { + st32(st, kEmpty); + } +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + st64(base + b * kStride, make_uint2(kEmpty, kEmpty)); + } + }; + if (clearFirst) + { + clearPartials(); + } + __syncthreads(); + stampInput(ctx); + // Threads 0-15: block tid's weight exp2(m_b - max) into shared memory, thread 0 the merged sum's reciprocal (once + // per unit instead of 16 exp2 and two divides in every thread: the path ran cold once per launch). + float* sAlpha = sStats + 2 * kAttnBlocks; + if (tid < kAttnBlocks) + { + float merged = -INFINITY; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + merged = fmaxf(merged, sStats[2 * b]); + } + float const mb = sStats[2 * tid]; + float const alpha = mb == -INFINITY ? 0.f : exp2f(mb - merged); + sAlpha[tid] = alpha; + float sum = alpha * sStats[2 * tid + 1]; +#pragma unroll + for (int off = kAttnBlocks / 2; off > 0; off >>= 1) + { + sum += __shfl_xor_sync(0x0000ffffu, sum, off); + } + if (tid == 0) + { + sAlpha[kAttnBlocks] = sum > 0.f ? 1.f / (sum * kPScale) : 0.f; + } + } + __syncthreads(); + float acc0 = 0.f; + float acc1 = 0.f; +#pragma unroll + for (int b = 0; b < kAttnBlocks; ++b) + { + float const alpha = sAlpha[b]; + acc0 += alpha * __uint_as_float(o[b].x); + acc1 += alpha * __uint_as_float(o[b].y); + } + float const rcp = sAlpha[kAttnBlocks]; + float const v0 = acc0 * rcp; + float const v1 = acc1 * rcp; +#if GLM52_IO_MODE == 1 || GLM52_IO_MODE == 3 + uint32_t const ow = lamport(packBf16(v0, v1)); + st32(arena(ctx, kArO) + ((t * kHeads + h) * kDv + d) * 2, ow); +#pragma unroll + for (int r = 0; r < 3; ++r) + { + st32(arena(ctx, kArORep + r * kORepBytes) + ((t * kHeads + h) * kDv + d) * 2, ow); + } +#else + st32(arena(ctx, kArO) + ((t * kHeads + h) * kDv + d) * 2, lamport(packBf16(v0, v1))); +#endif + if (!clearFirst) + { + clearPartials(); + } +} +#endif + +// Byte offset of hop 1's slot (token t, rank r, column c): [token][rank][6144], or with kFlagHop1Lines +// [rank][o_proj unit][token][32 columns]. +__device__ __forceinline__ size_t hop1Off(int t, int r, int c) +{ + if constexpr ((kSpecializedStaticFlags & kFlagHop1Lines) != 0) + { + return ((static_cast(r) * (kHidden / 32) + c / 32) * kTok + t) * 64 + (c % 32) * 2; + } + return (static_cast(t * kRanks + r) * kHidden + c) * 2; +} + +// o_proj: z rows [32 u, +32) = o W_OV^T (o = the 4 heads' merged outputs, K = 2048; W_OV = W_O W_UV per head), +// published into every GPU's attention all-reduce slot of this rank. +__device__ __noinline__ void stageOproj(Ctx const& ctx, int unit, uint32_t wOff, uint32_t bar) +{ + int const warp = threadIdx.x >> 5; + constexpr int kPer = kOChunks / kWarps; + if (GLM52_FLAGS(ctx) & kFlagCanaryOproj) + { + // The first word of every merge unit (token t, head h). + uint8_t const* a = arena(ctx, kArO); + waitCanaries( + ctx, 32, [&](int u) { return a + u * kDv * 2; }, 140); + } + uint4 b[kPer][2]; +#if GLM52_IO_MODE == 1 || GLM52_IO_MODE == 3 + int const orr = static_cast(blockIdx.x) & 3; + uint8_t const* const oSrc = arena(ctx, orr == 0 ? kArO : kArORep + (orr - 1) * kORepBytes); +#else + uint8_t const* const oSrc = arena(ctx, kArO); +#endif + float acc[2][4] = {{0.f, 0.f, 0.f, 0.f}, {0.f, 0.f, 0.f, 0.f}}; + pollB(ctx, b, oSrc, kHeads * kDv, warp * kPer, 111); + stampInput(ctx); + waitBar(ctx, bar, 0, 112); +#pragma unroll + for (int i = 0; i < kPer; ++i) + { +#pragma unroll + for (int t = 0; t < 2; ++t) + { + mmaChunkBf16(acc[t], ctx.smemBase + wOff + (t * kOChunks + warp * kPer + i) * kBf16Block, b[i]); + } + } + float* out = reinterpret_cast(ctx.smem + kSmemRed + 8192); + reduceWarps<2>(ctx, acc, out); + // 8 tokens x 4 vectors of 8 columns. + if (threadIdx.x < 32) + { + int const tok = threadIdx.x / 4; + int const v = threadIdx.x % 4; + uint32_t w[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + int const r = 8 * v + 2 * j; + w[j] = lamport(packBf16(out[r * kTok + tok], out[(r + 1) * kTok + tok])); + } + uint8_t* mc = ptr(ctx, kPH1Mc) + static_cast(ctx.parity) * kHopBytes; + st128(mc + hop1Off(tok, ctx.rank, 32 * unit + 8 * v), make_uint4(w[0], w[1], w[2], w[3])); + } +} + +// GLM52_ROUTER_LG: this launch's copy of the partial logits (after the top-k exchange's copies): this GPU's, and its +// multicast address. +__device__ __forceinline__ uint8_t* lgUc(Ctx const& ctx) +{ + return ptr(ctx, kPTkUc) + static_cast(ctx.numCopies) * kTkBytes + + static_cast(ctx.parity) * kLgBytes; +} + +__device__ __forceinline__ uint8_t* lgMc(Ctx const& ctx) +{ + return ptr(ctx, kPTkMc) + static_cast(ctx.numCopies) * kTkBytes + + static_cast(ctx.parity) * kLgBytes; +} + +// acc += A x B for one 64-wide chunk with the A fragments in registers (pieces 0-3 of the packed chunk, lane's 16 B). +__device__ __forceinline__ void mmaChunkBf16Regs(float (&acc)[4], uint4 const (&p)[4], uint4 const (&b)[2]) +{ + mmaBf16(acc, p[0].x, p[2].x, p[0].y, p[2].y, b[0].x, b[0].y); + mmaBf16(acc, p[0].z, p[2].z, p[0].w, p[2].w, b[0].z, b[0].w); + mmaBf16(acc, p[1].x, p[3].x, p[1].y, p[3].y, b[1].x, b[1].y); + mmaBf16(acc, p[1].z, p[3].z, p[1].w, p[3].w, b[1].z, b[1].w); +} + +__device__ __noinline__ void moeWarmUp(Ctx const& ctx, uint32_t wOff); + +// GLM52_ROUTER_LG, the 16 ranks' partials of logit unit u (warp 0): summed in rank order ((0..7) + (8..15)) into kArLg +// (lane l: token l / 4, experts 16 u + 4 (l % 4) .. + 3), then the 16 slots back to empty (single reader; their next +// writers are the launch numCopies on), after the logits (a CTA's stores leave in issue order). +__device__ __forceinline__ void lgReduce(Ctx const& ctx, int u) +{ + int const lane = threadIdx.x & 31; + uint8_t* const src = lgUc(ctx) + u * kLgSlotBytes + 16 * lane; + uint4 v[kRanks]; + pollStrided(ctx, v, src, kLgUnits * kLgSlotBytes, 192); + float a[4] = {0.f, 0.f, 0.f, 0.f}; + float b[4] = {0.f, 0.f, 0.f, 0.f}; +#pragma unroll + for (int r = 0; r < kRanks; ++r) + { + uint32_t const w[4] = {v[r].x, v[r].y, v[r].z, v[r].w}; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + if (r < 8) + { + a[j] += __uint_as_float(w[j]); + } + else + { + b[j] += __uint_as_float(w[j]); + } + } + } + uint4 const o + = make_uint4(lamport(__float_as_uint((0.f + a[0]) + b[0])), lamport(__float_as_uint((0.f + a[1]) + b[1])), + lamport(__float_as_uint((0.f + a[2]) + b[2])), lamport(__float_as_uint((0.f + a[3]) + b[3]))); + st128(arena(ctx, kArLg) + ((lane >> 2) * kExperts + 16 * u + 4 * (lane & 3)) * 4, o); + if (GLM52_STAMPS(ctx) != nullptr && lane == 0) + { + GLM52_STAMPS(ctx)[kMaxRoles * 4 + 4 + 131] = globalTimer(); + } + uint4 const e = make_uint4(kEmpty, kEmpty, kEmpty, kEmpty); + uint64_t const policy = policyEvictFirst(); +#pragma unroll + for (int r = 0; r < kRanks; ++r) + { + st128Hint(src + r * kLgUnits * kLgSlotBytes, e, policy); + } +} + +// GLM52_ROUTER_LG on CTAs 148-163: o_proj unit `unit` exactly as stageOproj (its hop-1 shard goes out first), then on +// the same polled O logit unit u = unit - kLgOprojFirst: this rank's partial logits of experts [16 u, +16) = +// M_r [O | residual columns [384 r, +384)], one 16-row tile (its O part's 64 KB in shared memory after o_proj's 128 KB; +// warp w < 6 also residual chunk w, its A fragments and the residual loaded from global memory at the unit's start), +// into every GPU's slot [rank][u] as FP32 [8 tokens][16 experts]. Then the CTA's MoE warm-up (kFlagWarmMoe; the MoE +// stage skips it, kMoeWarmed) while the other ranks' partials are on their way, and their sum (lgReduce). +__device__ __noinline__ void stageOprojLg( + Ctx const& ctx, int unit, uint32_t wOff, uint32_t bar, __nv_bfloat16 const* residual) +{ + int const warp = threadIdx.x >> 5; + int const lane = threadIdx.x & 31; + int const u = unit - kLgOprojFirst; + constexpr int kPer = kOChunks / kWarps; + // The residual (the launch's input, in since the dependency wait) and the residual chunk's weights, long before O. + uint4 br[2] = {make_uint4(0, 0, 0, 0), make_uint4(0, 0, 0, 0)}; + uint4 ar[4] = {make_uint4(0, 0, 0, 0), make_uint4(0, 0, 0, 0), make_uint4(0, 0, 0, 0), make_uint4(0, 0, 0, 0)}; + if (warp < kLgResChunks) + { + loadB(br, residual, kHidden, kLgResChunks * ctx.rank + warp); + uint8_t const* const a = ptr(ctx, kPWLg) + + (static_cast(u) * kLgChunks + kOChunks + warp) * kBf16Block + 16 * lane; +#pragma unroll + for (int k = 0; k < 4; ++k) + { + ar[k] = ldcg128(a + 512 * k); + } + } + uint4 b[kPer][2]; +#if GLM52_IO_MODE == 1 || GLM52_IO_MODE == 3 + int const orr = static_cast(blockIdx.x) & 3; + uint8_t const* const oSrc = arena(ctx, orr == 0 ? kArO : kArORep + (orr - 1) * kORepBytes); +#else + uint8_t const* const oSrc = arena(ctx, kArO); +#endif + float acc[2][4] = {{0.f, 0.f, 0.f, 0.f}, {0.f, 0.f, 0.f, 0.f}}; + pollB(ctx, b, oSrc, kHeads * kDv, warp * kPer, 111); + stampInput(ctx); + waitBar(ctx, bar, 0, 112); +#pragma unroll + for (int i = 0; i < kPer; ++i) + { +#pragma unroll + for (int t = 0; t < 2; ++t) + { + mmaChunkBf16(acc[t], ctx.smemBase + wOff + (t * kOChunks + warp * kPer + i) * kBf16Block, b[i]); + } + } + float* out = reinterpret_cast(ctx.smem + kSmemRed + 8192); + reduceWarps<2>(ctx, acc, out); + if (threadIdx.x < 32) + { + int const tok = threadIdx.x / 4; + int const v = threadIdx.x % 4; + uint32_t w[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + int const r = 8 * v + 2 * j; + w[j] = lamport(packBf16(out[r * kTok + tok], out[(r + 1) * kTok + tok])); + } + uint8_t* mc = ptr(ctx, kPH1Mc) + static_cast(ctx.parity) * kHopBytes; + st128(mc + hop1Off(tok, ctx.rank, 32 * unit + 8 * v), make_uint4(w[0], w[1], w[2], w[3])); + } + // The logit tile on the same O fragments (reduceWarps<1> writes its warp sums below o_proj's out and syncs before + // overwriting out, which threads 0-31 read above). + float acc1[1][4] = {{0.f, 0.f, 0.f, 0.f}}; +#pragma unroll + for (int i = 0; i < kPer; ++i) + { + mmaChunkBf16(acc1[0], ctx.smemBase + wOff + (2 * kOChunks + warp * kPer + i) * kBf16Block, b[i]); + } + if (warp < kLgResChunks) + { + mmaChunkBf16Regs(acc1[0], ar, br); + } + reduceWarps<1>(ctx, acc1, out); + if (threadIdx.x < 32) + { + // Lane l: token l / 4, experts 4 (l % 4) .. + 3 of the logit unit. + int const tok = threadIdx.x >> 2; + int const e0 = 4 * (threadIdx.x & 3); + uint4 const w = make_uint4(lamport(__float_as_uint(out[(e0 + 0) * kTok + tok])), + lamport(__float_as_uint(out[(e0 + 1) * kTok + tok])), lamport(__float_as_uint(out[(e0 + 2) * kTok + tok])), + lamport(__float_as_uint(out[(e0 + 3) * kTok + tok]))); + st128(lgMc(ctx) + (ctx.rank * kLgUnits + u) * kLgSlotBytes + 16 * threadIdx.x, w); + } + subStamp(ctx, 130); + if (GLM52_FLAGS(ctx) & kFlagWarmMoe) + { + moeWarmUp(ctx, 0); + if (threadIdx.x == 0) + { + reinterpret_cast(ctx.smem + kSmemSmall + kSmallMoe)[kMoeWarmed] = 1; + } + } + if (warp == 0) + { + lgReduce(ctx, u); + } +} + +// Empties this GPU's copy of the next launch's buffers (their readers finished before this launch began): the two hop +// buffers (counted: the MoE all-reduce's publish waits for them), then the arena (counted too: kCountedClearUnits). +__device__ __noinline__ void stageClear(Ctx const& ctx, int unit) +{ + int const other = (ctx.parity + 1) % ctx.numCopies; + uint4 const e = make_uint4(kEmpty, kEmpty, kEmpty, kEmpty); + // Under the L2 evict-first policy: only a later launch reads these ~5 MB (-1.1 us per slice, mk-v116a). + uint64_t const policy = policyEvictFirst(); + if (unit < kHopClearUnits) + { + int const begin = unit * kClearUnitBytes; + int const which = begin / kHopBytes; + uint8_t* base = which == 0 ? ptr(ctx, kPH1Uc) + static_cast(other) * kHopBytes + begin + : which == 1 ? ptr(ctx, kPH2Uc) + static_cast(other) * kHopBytes + (begin - kHopBytes) + : ptr(ctx, kPTkUc) + static_cast(other) * kTkBytes + (begin - 2 * kHopBytes); + for (int off = 16 * threadIdx.x; off < kClearUnitBytes; off += 16 * kThreads) + { + st128Hint(base + off, e, policy); + } + signalCounter(ctx, kCClear); + } + else + { + int const begin = (unit - kHopClearUnits) * kClearUnitBytes; + int const end = min(begin + kClearUnitBytes, kArenaBytes); + uint8_t* base = ptr(ctx, kPArena) + static_cast(other) * kArenaBytes; + for (int off = begin + 16 * threadIdx.x; off < end; off += 16 * kThreads) + { + st128Hint(base + off, e, policy); + } + signalCounter(ctx, kCClear); + } +} + +// Polls this GPU's copy of the 16 slots of 8 columns (token tok, columns col..col+7; hop 1's layout with first); FP32 +// sum in rank order ((0..7) + (8..15)), as the one-shot hop. +__device__ __forceinline__ void pollReduce8( + Ctx const& ctx, uint8_t const* buf, int tok, int col, float (&sum)[8], int code, bool first) +{ + uint4 v[kRanks]; + if (first) + { + pollStrided( + ctx, v, buf + hop1Off(tok, 0, col), static_cast(hop1Off(tok, 1, col) - hop1Off(tok, 0, col)), code); + } + else + { + pollStrided(ctx, v, buf + (static_cast(tok * kRanks) * kHidden + col) * 2, kHidden * 2, code); + } + float a[8], b[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) + { + a[j] = 0.f; + b[j] = 0.f; + } +#pragma unroll + for (int r = 0; r < kRanks; ++r) + { + uint32_t const w[4] = {v[r].x, v[r].y, v[r].z, v[r].w}; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + if (r < 8) + { + a[2 * j] += bf16Lo(w[j]); + a[2 * j + 1] += bf16Hi(w[j]); + } + else + { + b[2 * j] += bf16Lo(w[j]); + b[2 * j + 1] += bf16Hi(w[j]); + } + } + } +#pragma unroll + for (int j = 0; j < 8; ++j) + { + sum[j] = (0.f + a[j]) + b[j]; + } +} + +// The two all-reduce consumers: token t, columns [1536 s, +1536). Threads 0-191 own 8 columns each. +// H1 (second false): res = s + residual_in (kept in shared memory), h = RMSNorm_post(res) -> arena (BF16 and NVFP4). +// H2 (second true): res' = s + res, x_next = RMSNorm_next(res'), residual_out = res'. One copy of the code for both +// (the CTAs that run them run both). +__device__ __noinline__ void moeWarmUp(Ctx const& ctx, uint32_t wOff); + +// kFlagHPrime: 1 / rms of token t from its four H1 units' sum-of-squares partials in kArSsq1 (H1's own formula), +// polled. +__device__ __forceinline__ float hopRcp(Ctx const& ctx, int t, int code) +{ + uint4 const all = poll4(ctx, arena(ctx, kArSsq1) + t * 4, code); + float const S + = (((0.f + __uint_as_float(all.x)) + __uint_as_float(all.y)) + __uint_as_float(all.z)) + __uint_as_float(all.w); + return rsqrtf(S / static_cast(kHidden) + kEps); +} + +// kFlagHopSelfClear: the 16 slots of this launch's hop 1 that H1 unit (t, col) read, and those of the previous +// launch's hop 2 (its H2 read them; their next writers are the launch numCopies on): back to empty, nothing waits for +// them. At H1's end, or with kFlagHopClearLate at the start of the same CTA's H2 unit, which waits for hop 2 anyway. +__device__ __forceinline__ void hopSelfClear(Ctx const& ctx, int t, int col) +{ + uint4 const e = make_uint4(kEmpty, kEmpty, kEmpty, kEmpty); + uint64_t const policy = policyEvictFirst(); + uint8_t* const h1 = ptr(ctx, kPH1Uc) + static_cast(ctx.parity) * kHopBytes; + uint8_t* const h2 + = ptr(ctx, kPH2Uc) + static_cast((ctx.parity + ctx.numCopies - 1) % ctx.numCopies) * kHopBytes; +#pragma unroll + for (int r = 0; r < kRanks; ++r) + { + st128Hint(h1 + hop1Off(t, r, col), e, policy); + st128Hint(h2 + (static_cast(t * kRanks + r) * kHidden + col) * 2, e, policy); + } +} + +__device__ __noinline__ void stageHop( + Ctx const& ctx, int unit, bool second, __nv_bfloat16 const* residualIn, __nv_bfloat16* xOut, __nv_bfloat16* resOut) +{ + int const t = unit / 4; + int const s4 = unit % 4; + int const tid = threadIdx.x; + float* sc = reinterpret_cast(ctx.smem + kSmemSmall + kSmallScalars); + uint4* sRes = reinterpret_cast(ctx.smem + kSmemRes); + bool const active = tid < kColsPerHopUnit / 8; + int const col = kColsPerHopUnit * s4 + 8 * tid; + uint8_t const* buf = (second ? ptr(ctx, kPH2Uc) : ptr(ctx, kPH1Uc)) + + static_cast(ctx.parity) * kHopBytes; + // H2 of a launch whose successor reads x' from its arena (kFlagNextChained): x' and the residual only. + bool const xOnly = second && (kSpecializedStaticFlags & kFlagXPrime) != 0 && (GLM52_FLAGS(ctx) & kFlagNextChained); + // The next launch's arena copy (H2: its x, kFlagEarlyX there). + uint8_t* const nextArena + = ptr(ctx, kPArena) + static_cast((ctx.parity + 1) % ctx.numCopies) * kArenaBytes; + uint4 resPrev = make_uint4(0, 0, 0, 0); + uint4 gam = make_uint4(0, 0, 0, 0); + // H1: the layer's FC1 activation global scale (Alpha1[kExperts]; 1 for GLM-5.2-NVFP4), before the hop wait. + float inScale = 1.f; + if (active) + { + resPrev + = second ? sRes[tid] : *reinterpret_cast(residualIn + static_cast(t) * kHidden + col); + gam = __ldg(reinterpret_cast( + (second ? ptr<__nv_bfloat16 const>(ctx, kPGammaNext) : ptr<__nv_bfloat16 const>(ctx, kPGammaPost)) + col)); + if (!second) + { + inScale = __ldg(ptr(ctx, kPAlpha1) + kExperts); + } + } + if constexpr ((kSpecializedStaticFlags & kFlagHopSelfClear) != 0 + && (kSpecializedStaticFlags & kFlagHopClearLate) != 0) + { + if (second && active) + { + hopSelfClear(ctx, t, col); // this CTA's H1 unit (same t, col) read them + } + } + if constexpr ((kSpecializedStaticFlags & kFlagWarmInHop) != 0) + { + if (!second && (GLM52_FLAGS(ctx) & kFlagWarmMoe)) + { + // The CTA's MoE warm-up while hop 1's data is on its way (its MoE region and TMEM are free in H1): it would + // otherwise sit between H1 and this CTA's routing (~1.6 us; these CTAs set the MoE tail). + moeWarmUp(ctx, 0); + } + } + float res[8]; + float ssq = 0.f; + if (active) + { + float sum[8]; + pollReduce8(ctx, buf, t, col, sum, second ? 120 : 113, !second); + uint32_t const rp[4] = {resPrev.x, resPrev.y, resPrev.z, resPrev.w}; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + float const s0 = bf16Round(sum[2 * j]); + float const s1 = bf16Round(sum[2 * j + 1]); + res[2 * j] = bf16Round(s0 + bf16Lo(rp[j])); + res[2 * j + 1] = bf16Round(s1 + bf16Hi(rp[j])); + } +#pragma unroll + for (int j = 0; j < 8; ++j) + { + ssq += bf16Round(res[j] * res[j]); + } + if constexpr ((kSpecializedStaticFlags & kFlagHPrime) != 0) + { + if (!second) + { + // h' = res x gamma for the router and the shared gate_up, before the exchange (they scale by 1 / rms). + uint32_t const gw[4] = {gam.x, gam.y, gam.z, gam.w}; + uint32_t hp[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + hp[j] = lamport( + packBf16(bf16Round(res[2 * j] * bf16Lo(gw[j])), bf16Round(res[2 * j + 1] * bf16Hi(gw[j])))); + } + st128(arena(ctx, kArHp) + (static_cast(t) * kHidden + col) * 2, + make_uint4(hp[0], hp[1], hp[2], hp[3])); + } + } + if constexpr ((kSpecializedStaticFlags & kFlagXPrime) != 0) + { + if (second) + { + // The next launch's x' = res' x gamma_next, before the exchange (its fused-A scales by 1 / rms). + uint32_t const gw[4] = {gam.x, gam.y, gam.z, gam.w}; + uint32_t xp[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + xp[j] = lamport( + packBf16(bf16Round(res[2 * j] * bf16Lo(gw[j])), bf16Round(res[2 * j + 1] * bf16Hi(gw[j])))); + } + st128(nextArena + kArX + xArOff(t, col), make_uint4(xp[0], xp[1], xp[2], xp[3])); + } + } + } +#pragma unroll + for (int off = 16; off > 0; off >>= 1) + { + ssq += __shfl_xor_sync(0xffffffffu, ssq, off); + } + int const warp = tid >> 5; + if ((tid & 31) == 0) + { + sc[warp] = ssq; + } + __syncthreads(); + // Sum-of-squares partials of the token's 4 units through the arena. + if (tid == 0) + { + float part = 0.f; + for (int w = 0; w < kWarps; ++w) + { + part += sc[w]; + } + if constexpr ((kSpecializedStaticFlags & kFlagXPrime) != 0) + { + if (second) + { + st32(nextArena + kArXSsq + (t * 4 + s4) * 4, lamport(__float_as_uint(part))); + } + } + if (!xOnly) + { + float* ssqBuf = arena(ctx, second ? kArSsq2 : kArSsq1) + t * 4; + st32(ssqBuf + s4, lamport(__float_as_uint(part))); + uint4 const all = poll4(ctx, ssqBuf, second ? 121 : 114); + sc[8] = __uint_as_float(all.x); + sc[9] = __uint_as_float(all.y); + sc[10] = __uint_as_float(all.z); + sc[11] = __uint_as_float(all.w); + } + } + __syncthreads(); + stampInput(ctx); + if (xOnly) + { + // The next launch takes x' (published above) and this residual: no normalized x, no exchange. + if (active) + { + *reinterpret_cast(resOut + static_cast(t) * kHidden + col) = make_uint4( + packBf16(res[0], res[1]), packBf16(res[2], res[3]), packBf16(res[4], res[5]), packBf16(res[6], res[7])); + } + return; + } + float const S = (((0.f + sc[8]) + sc[9]) + sc[10]) + sc[11]; + float const rcp = rsqrtf(S / static_cast(kHidden) + kEps); + if (active) + { + uint32_t const gw[4] = {gam.x, gam.y, gam.z, gam.w}; + float hv[8]; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + hv[2 * j] = bf16Round((res[2 * j] * rcp) * bf16Lo(gw[j])); + hv[2 * j + 1] = bf16Round((res[2 * j + 1] * rcp) * bf16Hi(gw[j])); + } + uint4 const hPacked = make_uint4( + packBf16(hv[0], hv[1]), packBf16(hv[2], hv[3]), packBf16(hv[4], hv[5]), packBf16(hv[6], hv[7])); + uint4 const rPacked = make_uint4( + packBf16(res[0], res[1]), packBf16(res[2], res[3]), packBf16(res[4], res[5]), packBf16(res[6], res[7])); + if (second) + { + *reinterpret_cast(xOut + static_cast(t) * kHidden + col) = hPacked; + *reinterpret_cast(resOut + static_cast(t) * kHidden + col) = rPacked; + // The next launch's x (kFlagEarlyX there): its arena copy's slot, emptied before this layer's shards went + // out (with kFlagXPrime x' went out before the exchange). + if constexpr ((kSpecializedStaticFlags & kFlagXPrime) == 0) + { + st128(nextArena + kArX + xArOff(t, col), + make_uint4(lamport(hPacked.x), lamport(hPacked.y), lamport(hPacked.z), lamport(hPacked.w))); + } + } + else + { + sRes[tid] = rPacked; + st128(arena(ctx, kArH) + (static_cast(t) * kHidden + col) * 2, + make_uint4(lamport(hPacked.x), lamport(hPacked.y), lamport(hPacked.z), lamport(hPacked.w))); + // NVFP4 of h (16-element blocks: this thread and its neighbour). + float vmax = 0.f; +#pragma unroll + for (int j = 0; j < 8; ++j) + { + vmax = fmaxf(vmax, fabsf(hv[j])); + } + vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, 1)); + // As the stock quantizer: block scales and values of inScale h. + float const sfValue = inScale * (vmax * rcpApproxFtz(6.f)); + uint32_t const sf = toE4m3(sfValue); + float const sff = static_cast(*reinterpret_cast<__nv_fp8_e4m3 const*>(&sf)); + float const outScale = vmax != 0.f ? rcpApproxFtz(sff * rcpApproxFtz(inScale)) : 0.f; + uint32_t packed = 0; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + packed |= toE2m1x2(hv[2 * j] * outScale, hv[2 * j + 1] * outScale) << (8 * j); + } + st32(arena(ctx, kArHfp4) + t * 3072 + col / 2, lamport(packed)); + // Scales: lane 8k packs the blocks of lanes 8k, 8k + 2, 8k + 4, 8k + 6 (the warp is active as a whole). + uint32_t const s1 = __shfl_down_sync(0xffffffffu, sf, 2); + uint32_t const s2 = __shfl_down_sync(0xffffffffu, sf, 4); + uint32_t const s3 = __shfl_down_sync(0xffffffffu, sf, 6); + if ((tid & 7) == 0) + { + st32(arena(ctx, kArHsf) + t * 384 + col / 16, + lamport(sf | (s1 << 8) | (s2 << 16) | (s3 << 24))); + } + } + } + if constexpr ((kSpecializedStaticFlags & kFlagHopSelfClear) != 0 + && (kSpecializedStaticFlags & kFlagHopClearLate) == 0) + { + if (!second && active) + { + hopSelfClear(ctx, t, col); + } + } +} + +// Router GEMV: logits of experts [16 et, +16) over K quarter kq. +__device__ __noinline__ void stageRouterGemv(Ctx const& ctx, int unit, uint32_t wOff, uint32_t bar) +{ + int const et = unit / 4; + int const kq = unit % 4; + int const warp = threadIdx.x >> 5; + constexpr int kPer = 24 / kWarps; + if (GLM52_FLAGS(ctx) & kFlagCanaryH) + { + // The first word of the H1 unit (token t, quarter kq) of every token. + uint8_t const* a = arena(ctx, kArH) + 1536 * kq * 2; + waitCanaries( + ctx, kTok, [&](int t) { return a + t * kHidden * 2; }, 141); + } + uint4 b[kPer][2]; + pollB(ctx, b, + arena(ctx, (kSpecializedStaticFlags & kFlagHPrime) ? kArHp : kArH) + 1536 * kq * 2, kHidden, + warp * kPer, 115); + stampInput(ctx); + waitBar(ctx, bar, 0, 116); + float acc[1][4] = {{0.f, 0.f, 0.f, 0.f}}; +#pragma unroll + for (int i = 0; i < kPer; ++i) + { + mmaChunkBf16(acc[0], ctx.smemBase + wOff + (warp * kPer + i) * kBf16Block, b[i]); + } + float* out = reinterpret_cast(ctx.smem + kSmemRed + 8192); + reduceWarps<1>(ctx, acc, out); + if (threadIdx.x < 128) + { + int const row = threadIdx.x / kTok; + int const tok = threadIdx.x % kTok; + st32(arena(ctx, kArLogit) + (kq * kTok + tok) * kExperts + 16 * et + row, + lamport(__float_as_uint(out[row * kTok + tok]))); + } +} + +__device__ __forceinline__ float sigmoidAccurate(float x) +{ + return 0.5f * tanhf(0.5f * x) + 0.5f; +} + +// noaux_tc routing of the 8 tokens (warp t = token t) from the polled router logits: sigmoid + BF16 bias, top-8 (ties +// to the smaller expert), weights 2.5 * sig / sum, into the MoE work list (E, slot experts, slot and weight of every +// (token, k)). With publish, also into the arena (the station check reads it there). +__device__ __forceinline__ void routeTokens(Ctx const& ctx, int* smemMoe, bool publish) +{ + int const warp = threadIdx.x >> 5; + int const lane = threadIdx.x & 31; + int* sTopExpert = reinterpret_cast(ctx.smem + kSmemRed); + float* sTopWeight = reinterpret_cast(ctx.smem + kSmemRed + 256); + uint32_t* sMask = reinterpret_cast(ctx.smem + kSmemRed + 512); + uint32_t* sTokMask = reinterpret_cast(ctx.smem + kSmemRed + 2048); // per expert: the tokens choosing it + // GLM52_ROUTER_LG == 2: the reduced logits (one word per expert, kArLg); else the router GEMV's K quarters. + constexpr int kQ = kRouterLg == 2 ? 1 : 4; + float const* lp = arena(ctx, kRouterLg == 2 ? kArLg : kArLogit); + __nv_bfloat16 const* bias = ptr<__nv_bfloat16 const>(ctx, kPRouterBias); + int const t = warp; + sTokMask[threadIdx.x] = 0; + // The lane's 8 bias values first (constants; their loads overlap the poll). + uint32_t bz[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + bz[i] = ldPinnedU16(bias + 32 * i + lane); + } + // kFlagHPrime: the token's 4 sum-of-squares partials (in long before the logits), requested with them (read after + // the poll, their L2 round trip sat between the logits and the top-8). + uint4 ssq = make_uint4(0u, 0u, 0u, 0u); + if constexpr ((kSpecializedStaticFlags & kFlagHPrime) != 0) + { + if (lane == 0) + { + ssq = ldPoll128(arena(ctx, kArSsq1) + t * 4); + } + } + // Lane: experts 32 i + lane, the kQ partials of each (8 kQ words, polled together). + uint32_t lw[8][kQ]; +#pragma unroll + for (int i = 0; i < 8; ++i) + { +#pragma unroll + for (int k = 0; k < kQ; ++k) + { + lw[i][k] = ldPoll32(lp + (k * kTok + t) * kExperts + 32 * i + lane); + } + } + constexpr bool kRstdShared = (kSpecializedStaticFlags & kFlagHPrime) && (kSpecializedStaticFlags & kFlagRstdShared); + { + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int i = 0; i < 8; ++i) + { +#pragma unroll + for (int k = 0; k < kQ; ++k) + { + ok &= lw[i][k] != kEmpty; + } + } + if constexpr (kRstdShared) + { + ok &= lane != 0 || full4(ssq); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, 117, t, 0); + if constexpr (kRstdShared) + { + if (lane == 0 && !full4(ssq)) + { + ssq = ldPoll128(arena(ctx, kArSsq1) + t * 4); + } + } +#pragma unroll + for (int i = 0; i < 8; ++i) + { +#pragma unroll + for (int k = 0; k < kQ; ++k) + { + if (lw[i][k] == kEmpty) + { + lw[i][k] = ldPoll32(lp + (k * kTok + t) * kExperts + 32 * i + lane); + } + } + } + } + } + stampInput(ctx); + subStamp(ctx, 14); + // kFlagHPrime: the router ran on h' = res x gamma: the token's 1 / rms (hopRcp's). + float rcpT = 1.f; + if constexpr ((kSpecializedStaticFlags & kFlagHPrime) != 0) + { + if (lane == 0) + { + if (!full4(ssq)) + { + ssq = poll4(ctx, arena(ctx, kArSsq1) + t * 4, 146); + } + float const S = (((0.f + __uint_as_float(ssq.x)) + __uint_as_float(ssq.y)) + __uint_as_float(ssq.z)) + + __uint_as_float(ssq.w); + rcpT = rsqrtf(S / static_cast(kHidden) + kEps); + if constexpr (kRstdShared) + { + smemMoe[kMoeRcp + t] = __float_as_int(rcpT); + } + } + rcpT = __shfl_sync(0xffffffffu, rcpT, 0); + } + // Selection score = sigmoid + bias; the weights use the sigmoid itself. + float sg[8]; + float score[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + float logit = 0.f; +#pragma unroll + for (int k = 0; k < kQ; ++k) + { + logit += __uint_as_float(lw[i][k]); + } + if constexpr ((kSpecializedStaticFlags & kFlagHPrime) != 0) + { + logit *= rcpT; + } + sg[i] = sigmoidAccurate(logit); + score[i] = sg[i] + __uint_as_float(bz[i] << 16); + } + uint32_t taken = 0; + float topSig = 0.f; + int topExpert = 0; + // Top-8 with the warp reductions: per round the largest score (order-preserving bits), then the smallest expert + // among the lanes holding it (ties go to the smaller expert, as a sorted top-k). + uint32_t key[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + uint32_t const b = __float_as_uint(score[i]); + key[i] = b ^ ((b >> 31) ? 0xffffffffu : 0x80000000u); + } +#pragma unroll 1 + for (int k = 0; k < 8; ++k) + { + // The lane's best remaining candidate (the smaller i on a tie: the smaller expert). + uint32_t best = 0; + int bi = 0; +#pragma unroll + for (int i = 7; i >= 0; --i) + { + if (!(taken & (1u << i)) && key[i] >= best) + { + best = key[i]; + bi = i; + } + } + uint32_t const wmax = __reduce_max_sync(0xffffffffu, best); + uint32_t const e = static_cast(32 * bi + lane); + uint32_t const win = __reduce_min_sync(0xffffffffu, best == wmax ? e : 0xffffu); + float s0 = 0.f; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + s0 = i == bi ? sg[i] : s0; + } + float const ws = __shfl_sync(0xffffffffu, s0, static_cast(win & 31u)); + if (static_cast(lane) == (win & 31u)) + { + taken |= 1u << (win >> 5); + } + if (lane == k) + { + topSig = ws; + topExpert = static_cast(win); + } + } + subStamp(ctx, 15); + float const sig = lane < 8 ? topSig : 0.f; + float sum = sig; + sum += __shfl_xor_sync(0xffffffffu, sum, 4); + sum += __shfl_xor_sync(0xffffffffu, sum, 2); + sum += __shfl_xor_sync(0xffffffffu, sum, 1); + if (lane < 8) + { + float const w = bf16Round(sig * 2.5f / (sum + 0.f)); + sTopExpert[t * 8 + lane] = topExpert; + sTopWeight[t * 8 + lane] = w; + } + uint32_t* sSlotMask = reinterpret_cast(ctx.smem + kSmemRed + 1024); + int* sSlotOfExpert = reinterpret_cast(ctx.smem + kSmemRed + 12288); + __syncthreads(); + if (lane < 8) + { + atomicOr(&sTokMask[topExpert], 1u << t); + } + __syncthreads(); + // Thread e (one per expert): the tokens that chose it; the selected experts in index order are the slots, so a + // warp's ballot and the earlier warps' counts give e's slot. + { + int const e = threadIdx.x; + uint32_t const tokMask = sTokMask[e]; + uint32_t const sel = __ballot_sync(0xffffffffu, tokMask != 0); + if (lane == 0) + { + sMask[warp] = sel; + } + __syncthreads(); + int slot = __popc(sel & ((1u << lane) - 1u)); + int total = 0; +#pragma unroll + for (int w = 0; w < kWarps; ++w) + { + int const c = __popc(sMask[w]); + slot += w < warp ? c : 0; + total += c; + } + if (tokMask != 0) + { + smemMoe[kMoeSlotExpert + slot] = e; + sSlotMask[slot] = tokMask; + sSlotOfExpert[e] = slot; + } + if (threadIdx.x == 0) + { + smemMoe[kMoeE] = total; + } + } + __syncthreads(); + if (threadIdx.x < kTok * 8) + { + smemMoe[kMoeSlotOfTk + threadIdx.x] = sSlotOfExpert[sTopExpert[threadIdx.x]]; + smemMoe[kMoeWeight + threadIdx.x] = __float_as_int(sTopWeight[threadIdx.x]); + } + __syncthreads(); + if (publish && threadIdx.x < kTok * 8) + { + int* route = arena(ctx, kArRoute); + int const E = smemMoe[kMoeE]; + if (threadIdx.x < E) + { + st32(route + kRouteSlotExpert + threadIdx.x, static_cast(smemMoe[kMoeSlotExpert + threadIdx.x])); + } + st32(route + kRouteSlotOfTk + threadIdx.x, static_cast(smemMoe[kMoeSlotOfTk + threadIdx.x])); + st32(route + kRouteWeight + threadIdx.x, static_cast(smemMoe[kMoeWeight + threadIdx.x])); + if (threadIdx.x == 0) + { + st32(route + kRouteE, static_cast(E)); + } + } +} + +// Shared expert gate_up: rows [16 ft, +16) (gate) and [128 + 16 ft, +16) (up) over K quarter kq. +__device__ __noinline__ void stageSharedGu(Ctx const& ctx, int unit, uint32_t wOff, uint32_t bar) +{ + int const ft = unit / 4; + int const kq = unit % 4; + int const warp = threadIdx.x >> 5; + constexpr int kPer = 24 / kWarps; + if constexpr (kRouterLg == 2) + { + if (GLM52_FLAGS(ctx) & kFlagWarmMoe) + { + // The CTA's MoE warm-up while h' is on its way (as H1's in its hop wait): with the reduced logits in + // before h', it would otherwise sit between this unit and the CTA's routing. Its stale operands include + // this unit's weights (the MMAs' results are discarded), so they land first: the copy's completion is + // the only edge that orders its writes before the tensor cores' reads. + waitBar(ctx, bar, 0, 119); + moeWarmUp(ctx, 0); + if (threadIdx.x == 0) + { + reinterpret_cast(ctx.smem + kSmemSmall + kSmallMoe)[kMoeWarmed] = 1; + } + } + } + if (GLM52_FLAGS(ctx) & kFlagCanaryH) + { + uint8_t const* a = arena(ctx, kArH) + 1536 * kq * 2; + waitCanaries( + ctx, kTok, [&](int t) { return a + t * kHidden * 2; }, 142); + } + uint4 b[kPer][2]; + pollB(ctx, b, + arena(ctx, (kSpecializedStaticFlags & kFlagHPrime) ? kArHp : kArH) + 1536 * kq * 2, kHidden, + warp * kPer, 118); + stampInput(ctx); + waitBar(ctx, bar, 0, 119); + float acc[2][4] = {{0.f, 0.f, 0.f, 0.f}, {0.f, 0.f, 0.f, 0.f}}; +#pragma unroll + for (int i = 0; i < kPer; ++i) + { +#pragma unroll + for (int t = 0; t < 2; ++t) + { + mmaChunkBf16(acc[t], ctx.smemBase + wOff + (t * 24 + warp * kPer + i) * kBf16Block, b[i]); + } + } + float* out = reinterpret_cast(ctx.smem + kSmemRed + 8192); + reduceWarps<2>(ctx, acc, out); + float* sh = arena(ctx, kArSh) + ((kq * 8 + ft) * 32) * kTok; + st32(sh + threadIdx.x, lamport(__float_as_uint(out[threadIdx.x]))); +} + +// The MoE work cut: n pieces into kCtas runs by weight per CTA class (as the role table places the units): kCutWOs for +// the CTAs with a shared-expert unit ([kOsFirst, +kOsCtas): that unit runs before their first piece; with equal runs +// they ended the layer's MoE ~2 us after the others), kCutWSg for the shared gate_up CTAs ([kSgFirst, +kSgCtas)) and +// kCutWH1 for the H1 CTAs ([kH1Cta0, kCtas)): their routing starts after those units' ends (GLM52_ROUTER_LG == 2: the +// others route as soon as the reduced logits and H1's sums of squares are in), kCutW for the rest. The first piece of +// CTA c's run, and the CTA whose run holds piece g (the largest c whose run starts at or before g). OS 2 of 3 since the +// shared expert takes 1 / rms from the routing (kFlagRstdShared; 9:5 matched its four rstd polls). Valid weights keep +// every aligned cut's FC1 runs >= 0 for E 27-64 (gen_cut.py checks; kFc2ListsPacked and kFc2ListsCut come from it). +constexpr int kOsFirst = 100; +constexpr int kOsCtas = 48; +constexpr int kSgFirst = 65; // the shared gate_up units' CTAs (host build_roles: 65 + unit) +constexpr int kSgCtas = 32; +constexpr int kCutW = kFc2ListsCut[0]; +constexpr int kCutWOs = kFc2ListsCut[1]; +constexpr int kCutWSg = kFc2ListsCut[2]; +constexpr int kCutWH1 = kFc2ListsCut[3]; + +// CTAs of [a, b) below c. +__host__ __device__ constexpr int cutSegment(int c, int a, int b) +{ + return c <= a ? 0 : c >= b ? b - a : c - a; +} + +// The cut's weight of CTAs [0, c). +__host__ __device__ constexpr int moeCutWeight(int c) +{ + return kCutW + * (cutSegment(c, 0, kSgFirst) + cutSegment(c, kSgFirst + kSgCtas, kOsFirst) + + cutSegment(c, kOsFirst + kOsCtas, kH1Cta0)) + + kCutWSg * cutSegment(c, kSgFirst, kSgFirst + kSgCtas) + kCutWOs * cutSegment(c, kOsFirst, kOsFirst + kOsCtas) + + kCutWH1 * cutSegment(c, kH1Cta0, kCtas); +} + +constexpr int kCutTotal = moeCutWeight(kCtas); +static_assert(kSgFirst + kSgCtas <= kOsFirst && kOsFirst + kOsCtas <= kH1Cta0, "cut segments in CTA order"); + +__device__ __forceinline__ int moeRunStart(int c, int n) +{ + return moeCutWeight(c) * n / kCutTotal; +} + +// (GLM52_LISTS_MODE < 2 only) the largest c whose run starts at or before piece g. +__device__ __forceinline__ int moeCtaOf(int g, int n) +{ + int lo = 0; + int hi = kCtas - 1; + while (lo < hi) + { + int const mid = (lo + hi + 1) >> 1; + if (moeRunStart(mid, n) <= g) + { + lo = mid; + } + else + { + hi = mid - 1; + } + } + return lo; +} + +// The MoE work cut of E slots: CTA c's FC1 run [f1(c), f1(c + 1)) of the 16 E FC1 pieces and its FC2 run +// [f2(c), f2(c + 1)) of the 8 E FC2 pieces. With E >= 27 (aligned): slot s's 8 FC2 pieces split evenly over CTAs +// [s kCtas / E, (s + 1) kCtas / E) only, and every CTA fills its share of the joint (weighted) cut of all 24 E with FC1 +// pieces. No CTA then holds FC2 pieces of two slots: a second FC2 group, with its own wait for partials, SwiGLU and +// quantization, ended such a CTA ~2 us after the median, the layer's last (-8 us per slice, mk-v93a); the totals keep +// the spread of a plain joint cut (checked for E 27-64: FC1 runs monotone). Fewer experts: separate weighted cuts of +// the FC1 and FC2 pieces (a joint cut's FC2 counts could go negative there). +struct MoeCut +{ + int E; + int const* table; // aligned cut: the FC1 starts f1(c), c <= kCtas, in shared memory; else null + + // The aligned cut's FC2 start of CTA c: its slot s is the last whose first CTA s kCtas / E is at or before c. + __device__ __forceinline__ static int alignedF2(int c, int E) + { + if (c >= kCtas) + { + return 8 * E; + } + int const s = min(((c + 1) * E - 1) / kCtas, E - 1); + int const c0 = s * kCtas / E; + return 8 * s + 8 * (c - c0) / ((s + 1) * kCtas / E - c0); + } + + __device__ __forceinline__ int f1(int c) const + { + return table != nullptr ? table[c] : moeRunStart(c, 16 * E); + } + + __device__ __forceinline__ int f2(int c) const + { + if (table != nullptr) + { + return moeRunStart(c, 24 * E) - table[c]; + } + return E >= 27 ? moeRunStart(c, 24 * E) - moeRunStart(c, 16 * E) : moeRunStart(c, 8 * E); + } + + // The CTA whose FC1 run holds FC1 piece g (the last CTA starting at or before it). + __device__ __forceinline__ int ctaOfF1(int g) const + { + if (table == nullptr) + { + return moeCtaOf(g, 16 * E); + } + int lo = 0; + int hi = kCtas - 1; + while (lo < hi) + { + int const mid = (lo + hi + 1) >> 1; + if (table[mid] <= g) + { + lo = mid; + } + else + { + hi = mid - 1; + } + } + return lo; + } +}; + +#if GLM52_LISTS_MODE >= 1 +// Lists for E1..64, each expert's two FC1 tiles. Header4 bits, up to8 ascending9-bit(CTA,run) codes. +// Index E*(E-1)+2*slot+tile;3 aligned32-bit loads per list, no runtime divisions or binary searches. +static_assert(kCtas == 212 && kOsFirst == 100 && kOsCtas == 48 && kSgFirst == 65 && kSgCtas == 32 && kH1Cta0 == 180, + "regenerate packed FC2 lists for a new cut (gen_cut.py: its CTA classes)"); + +__device__ __forceinline__ void loadFc2PartialList(int E, int slot, int tile, uint32_t* list) +{ + int const index = E * (E - 1) + 2 * slot + tile; + uint32_t const a = kFc2ListsPacked[index][0]; + uint32_t const b = kFc2ListsPacked[index][1]; + uint32_t const c = kFc2ListsPacked[index][2]; + uint32_t const nPart = a & 15u; +#pragma unroll + for (int q = 0; q < 8; ++q) + { + if (static_cast(q) < nPart) + { + int const bit = 4 + 9 * q; + int const word = bit >> 5; + int const shift = bit & 31; + uint32_t const lo = word == 0 ? a : word == 1 ? b : c; + uint32_t const hi = word == 0 ? b : c; + uint32_t code = lo >> shift; + if (shift > 23) + { + code |= hi << (32 - shift); + } + list[1 + q] = code & 511u; + } + } + list[0] = nPart; +} +#endif + +// kFlagFc2Direct: a lane's 32 bytes (at base in run 0's P1 block, 4 KiB per run) of each FC1 partial run of its FC2 +// tile (list: the count, then the (CTA, run) codes in ascending CTA order), summed into sum[8], polled in one round of +// kN >= count runs: all loads in flight together, then again for the vectors not complete. kN = 8 for a group with a +// tile of 5-8 runs (the OS region's slots at cut weight 2, 7-11 FC1 contributors per slot): in rounds of four the +// second round's loads left only after the first four runs were in, and the latest run is mostly a first-listed one +// (the lowest CTA ends the tile with its second run), one more L2 round trip on the slot's hand-off (+0.3 us stamped vs +// 4 runs). The sums are taken in every turn of the spin from that turn's values (the empty asm keeps them in the loop), +// so that their code is warm when the last run lands (a cold sum after the loop cost ~0.1 us per waiting group). Runs +// past the count stay +0: full, and adding +0 to the +0-started sum is exact, so these are the same q-by-q FP32 adds as +// summing the listed runs. +template +__device__ __forceinline__ void pollFc2Runs( + Ctx const& ctx, uint8_t const* base, uint32_t const* list, int nPart, float (&sum)[8], int slot) +{ + uint4 w[kN][2]; +#pragma unroll + for (int q = 0; q < kN; ++q) + { + bool const use = q < nPart; + uint8_t const* const src = base + static_cast(use ? list[1 + q] : 0u) * (128 * kTok * 4); + w[q][0] = use ? ldPoll128(src) : make_uint4(0, 0, 0, 0); + w[q][1] = use ? ldPoll128(src + 16) : make_uint4(0, 0, 0, 0); + } + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int q = 0; q < kN; ++q) + { + ok &= full4(w[q][0]) && full4(w[q][1]); + } + bool const done = warpAll(ok); +#pragma unroll + for (int e = 0; e < 8; ++e) + { + sum[e] = 0.f; + } +#pragma unroll + for (int q = 0; q < kN; ++q) + { +#pragma unroll + for (int v = 0; v < 2; ++v) + { + sum[4 * v] += __uint_as_float(w[q][v].x); + sum[4 * v + 1] += __uint_as_float(w[q][v].y); + sum[4 * v + 2] += __uint_as_float(w[q][v].z); + sum[4 * v + 3] += __uint_as_float(w[q][v].w); + } + } +#pragma unroll + for (int e = 0; e < 8; ++e) + { + asm volatile("" : "+f"(sum[e])); + } + if (done) + { + break; + } + spinCheck(ctx, t0, 124, slot, kN); +#pragma unroll + for (int q = 0; q < kN; ++q) + { + uint8_t const* const src = base + static_cast(q < nPart ? list[1 + q] : 0u) * (128 * kTok * 4); +#pragma unroll + for (int v = 0; v < 2; ++v) + { + if (!full4(w[q][v])) + { + w[q][v] = ldPoll128(src + 16 * v); + } + } + } + } +} + +// The source of MoE piece `piece` of this CTA (its FC1 pieces, then its FC2 pieces): 48 KiB of weights then 6 KiB of +// scales, packed together on the host ([expert][tile][8 pieces] for FC1, [expert][8 eighths] for FC2, kPieceBytes +// each). +__device__ __forceinline__ int moePieceSources( + Ctx const& ctx, int const* smemMoe, int piece, uint8_t const* (&src)[4], uint32_t (&dst)[4], uint32_t (&n)[4]) +{ + int const n1 = smemMoe[kMoeF1B] - smemMoe[kMoeF1A]; + dst[0] = 0; + n[0] = kPieceBytes; + if (piece < n1) + { + int const g = smemMoe[kMoeF1A] + piece; + size_t const tile = static_cast(smemMoe[kMoeSlotExpert + (g >> 4)]) * 2 + ((g >> 3) & 1); + src[0] = ptr(ctx, kPW13) + (tile * 8 + (g & 7)) * kPieceBytes; + return 1; + } + int const j = smemMoe[kMoeF2A] + piece - n1; + src[0] = ptr(ctx, kPW2) + + (static_cast(smemMoe[kMoeSlotExpert + (j >> 3)]) * 8 + (j & 7)) * kPieceBytes; + return 1; +} + +// FC1 MMAs of one piece (warp 1): the 12 K steps' weight scales into TMEM (sfa), then the 12 K steps of 128 rows x 8 +// tokens into D (the first one accumulating only with accFirst), committed on bar. hbq: the token operand of the +// piece's first K block; sfb: its first K step's token scales. +__device__ __noinline__ void fc1PieceMma( + uint32_t tmem, uint32_t base, uint32_t hbq, uint32_t sfa, uint32_t sfb, uint32_t accFirst, uint32_t bar) +{ + tcFenceAfter(); + if (electOne()) + { +#pragma unroll + for (int st = 0; st < 12; ++st) + { + utccp1(sfa + 4 * st, base + kPieceW + 512 * st); + } +#pragma unroll + for (int st = 0; st < 12; ++st) + { + int const kb = st >> 1; + int const ks = st & 1; + mmaNvf4(tmem + kTmemD, base + kb * 8192 + 32 * ks, hbq + kb * 512 + 32 * ks, sfa + 4 * st, sfb + 2 * st, + (accFirst != 0 || st > 0) ? 1u : 0u); + } + commit1(bar); + } + __syncwarp(); +} + +// FC2 MMAs of one piece (warp 1): the token scales (fc2Sfb) and the weight scales into TMEM, then 6 tiles x 2 K steps +// into accumulator columns [d + 8 ti, +8), committed on bar. +__device__ __noinline__ void fc2PieceMma( + uint32_t tmem, uint32_t base, uint32_t fc2B, uint32_t fc2Sfb, uint32_t sfa, uint32_t bar, uint32_t d = kTmemD) +{ + tcFenceAfter(); + if (electOne()) + { + utccp1(tmem + kTmemSfb2, fc2Sfb); +#pragma unroll + for (int st = 0; st < 12; ++st) + { + utccp1(sfa + 4 * st, base + kPieceW + 512 * st); + } +#pragma unroll + for (int st = 0; st < 12; ++st) + { + int const ti = st >> 1; + int const ks = st & 1; + mmaNvf4(tmem + d + 8 * ti, base + ti * 8192 + 32 * ks, fc2B + 32 * ks, sfa + 4 * st, + tmem + kTmemSfb2 + 2 * ks, static_cast(ks)); + } + commit1(bar); + } + __syncwarp(); +} + +// Requests MoE piece `piece` of this CTA into a ring slot (thread 0 only). The slot was last read by the tensor cores +// (async proxy) or, for the first pieces, before the stage's proxy fence. +__device__ __noinline__ void issueMoePiece(Ctx const& ctx, int const* smemMoe, int piece, uint32_t slot, uint32_t bar) +{ + uint8_t const* src[4]; + uint32_t dst[4]; + uint32_t n[4]; + int const parts = moePieceSources(ctx, smemMoe, piece, src, dst, n); + mbarExpectTx(bar, kPieceBytes); + if (GLM52_FLAGS(ctx) & kFlagEvictFirst) + { + uint64_t const policy = policyEvictFirst(); + for (int i = 0; i < parts; ++i) + { + bulkCopyHint(slot + dst[i], src[i], n[i], bar, policy); + } + } + else + { + for (int i = 0; i < parts; ++i) + { + bulkCopy(slot + dst[i], src[i], n[i], bar); + } + } +} + +// Prefetches MoE piece `piece` of this CTA into L2 (one lane). +__device__ __noinline__ void prefetchMoePiece(Ctx const& ctx, int const* smemMoe, int piece) +{ + uint8_t const* src[4]; + uint32_t dst[4]; + uint32_t n[4]; + int const parts = moePieceSources(ctx, smemMoe, piece, src, dst, n); + bool const hint = (GLM52_FLAGS(ctx) & kFlagEvictFirst) && !(GLM52_FLAGS(ctx) & kFlagMoeL2Normal); + uint64_t const policy = hint ? policyEvictFirst() : 0; + for (int i = 0; i < parts; ++i) + { + if (hint) + { + prefetchBulkL2Hint(src[i], n[i], policy); + } + else + { + prefetchBulkL2(src[i], n[i]); + } + } +} + +// NVFP4(h) into shared memory as the FC1 token operand: E2M1 in 48 SWIZZLE_64B atoms ([K block][8 rows][64 B]) and +// the scales as they are ([8][384]); polled (thread t: vectors t, t + 256, ...). +constexpr int kHVecE = kHfp4Bytes / 16; +constexpr int kHVec = kHVecE + kHsfBytes / 16; +constexpr int kHN = (kHVec + kThreads - 1) / kThreads; + +// The thread's 16-byte vectors of NVFP4(h) and its scales (vector tid + kThreads i), loaded without waiting. +__device__ __forceinline__ void loadH(Ctx const& ctx, uint4 (&v)[kHN]) +{ + uint8_t const* src = arena(ctx, kArHfp4); +#pragma unroll + for (int i = 0; i < kHN; ++i) + { + int const j = threadIdx.x + kThreads * i; + v[i] = j < kHVec ? ldPoll128(src + 16 * j) : make_uint4(0, 0, 0, 0); + } +} + +// The loaded vectors complete (polled again while empty), then into shared memory (h swizzled for the FC1 MMA). +__device__ __forceinline__ void finishH(Ctx const& ctx, uint4 (&v)[kHN], uint8_t* hb, uint8_t* hsf) +{ + constexpr int kVecE = kHVecE; + constexpr int kVec = kHVec; + constexpr int kN = kHN; + uint8_t const* src = arena(ctx, kArHfp4); + int const tid = threadIdx.x; + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int i = 0; i < kN; ++i) + { + ok &= full4(v[i]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, 131, 0, 0); +#pragma unroll + for (int i = 0; i < kN; ++i) + { + if (!full4(v[i])) + { + v[i] = ldPoll128(src + 16 * (tid + kThreads * i)); + } + } + } +#pragma unroll + for (int i = 0; i < kN; ++i) + { + int const j = tid + kThreads * i; + if (j < kVecE) + { + int const t = j / 192; + int const c16 = j % 192; + *reinterpret_cast(hb + (c16 >> 2) * 512 + t * 64 + (((c16 & 3) ^ ((t >> 1) & 3)) << 4)) = v[i]; + } + else if (j < kVec) + { + *reinterpret_cast(hsf + 16 * (j - kVecE)) = v[i]; + } + } +} + +__device__ __noinline__ void pollCopyH(Ctx const& ctx, uint8_t* hb, uint8_t* hsf) +{ + uint4 v[kHN]; + loadH(ctx, v); + finishH(ctx, v, hb, hsf); +} + +// The FC1 token scales of K step pairs [24 h, +24) (h = 0, 1) into TMEM columns kTmemSfb + 96 h + 4 pair: per pair of +// steps 32 rows x 16 B staged in shared memory (12 KiB at stage), row t < 8 = (scales of the even step, 0, scales of +// the odd step, 0); warp 1 copies them (all threads call; the caller waits for the copies before stage is rewritten). +__device__ __forceinline__ void stageTokenScales( + Ctx const& ctx, uint8_t const* hsf, uint8_t* stage, uint32_t tmem, int h) +{ + for (int v = threadIdx.x; v < 24 * 32; v += kThreads) + { + int const pair = 24 * h + v / 32; + int const row = v % 32; + uint4 val = make_uint4(0, 0, 0, 0); + if (row < kTok) + { + uint32_t const* sf = reinterpret_cast(hsf + row * 384) + 2 * pair; + val = make_uint4(sf[0], 0u, sf[1], 0u); + } + *reinterpret_cast(stage + 16 * v) = val; + } + fenceProxyAsyncSmem(); + __syncthreads(); + if ((threadIdx.x >> 5) == 1) + { + tcFenceAfter(); + if (electOne()) + { + for (int pair = 0; pair < 24; ++pair) + { + utccp1(tmem + kTmemSfb + 96 * h + 4 * pair, smemU32(stage + 512 * pair)); + } + } + __syncwarp(); + } +} + +// NVFP4(h) into shared memory (hb, hsf) and the token scales of all 96 K steps into TMEM, in two halves through 12 KiB +// of the reduction area (free once the routing is done; two phases of barMma from mmaPhase on, the next parity +// returned). +__device__ __noinline__ uint32_t moeStageH(Ctx const& ctx, uint8_t* hb, uint8_t* hsf, uint32_t tmem, uint32_t barMma, + uint32_t mmaPhase, bool staged, bool scalesStaged +#if GLM52_IO_MODE == 2 || GLM52_IO_MODE == 3 + , + bool scaleWork +#endif +) +{ + if (!staged) + { + pollCopyH(ctx, hb, hsf); + } + __syncthreads(); + subStamp(ctx, 88); +#if GLM52_IO_MODE == 2 || GLM52_IO_MODE == 3 + // Zero-piece CTAs must not issue asynchronous copies: no later MMA completion + // would drain them before TMEM deallocation. scaleWork is CTA-uniform. + if (!scaleWork) + { + return mmaPhase; + } + // Row t < 8 of pair p: (scale word 2 p of token t, 0, word 2 p + 1, 0); 48 pairs x 128 B. + uint8_t* stage = ctx.smem + kSmemRed + 2048; + for (int v = threadIdx.x; v < 48 * kTok; v += kThreads) + { + int const pair = v / kTok; + int const row = v % kTok; + uint32_t const* sf = reinterpret_cast(hsf + row * 384) + 2 * pair; + *reinterpret_cast(stage + 16 * v) = make_uint4(sf[0], 0u, sf[1], 0u); + } + fenceProxyAsyncSmem(); + __syncthreads(); + if ((threadIdx.x >> 5) == 1) + { + tcFenceAfter(); + if (electOne()) + { + for (int pair = 0; pair < 48; ++pair) + { + utccp1Rows8(tmem + kTmemSfb + 4 * pair, smemU32(stage + 128 * pair)); + } + } + __syncwarp(); + } + return mmaPhase; +#else + for (int h = 0; h < (scalesStaged ? 0 : 2); ++h) + { + stageTokenScales(ctx, hsf, ctx.smem + kSmemRed + 2048, tmem, h); + if ((threadIdx.x >> 5) == 1) + { + commitElect(barMma); + } + waitBar(ctx, barMma, mmaPhase, 136); + mmaPhase ^= 1; + __syncthreads(); + if (h == 0) + { + subStamp(ctx, 89); + } + } + return mmaPhase; +#endif +} + +// Routed experts on the tensor cores. The CTA routes the 8 tokens itself, then runs its FC1 run (the pieces' MMAs +// accumulate per (slot, tile); each run's FP32 partial D[128 rows][8 tokens] goes into the arena) and its FC2 run +// (per slot: the slot's FC1 partials summed, SwiGLU + NVFP4 = the token operand, then per eighth 6 tiles of 128 rows x +// K 128, the rows of the tokens routed to the expert into fc2out). Weights stream through a ring of three 54 KiB +// slots; warp 1 issues the scale copies and the MMAs, warps 0-3 read the accumulators. +__device__ __noinline__ void stageOwnerShared(Ctx const& ctx, int J, int const* smemMoe = nullptr); + +// The routed experts of the 8 tokens on this CTA's share of the work list. ownerShared >= 0: the CTA's shared-expert +// unit runs between the first pieces' requests and their arrival (its inputs are in by then; before the routing it +// delayed this CTA's requests). +// kFlagWarmMoe: one FC1 and one FC2 MMA batch on the stale contents of the MoE region's ring slots 0 and 1 (the MMA +// code and pipe warm before the real pieces; worth ~1.5 us per layer): D and the token-scale columns are rewritten +// before use (the first K step of every real tile does not accumulate; the token scales are staged after it). The +// barrier completes two phases, so its parity is 0 again. Every thread calls it. +__device__ __noinline__ void moeWarmUp(Ctx const& ctx, uint32_t wOff) +{ + int const warp = threadIdx.x >> 5; + constexpr uint32_t kRegion2 = kRingSlots * kPieceBytes; + uint32_t const hb = ctx.smemBase + wOff + kRegion2; + uint32_t const fc2B = hb + 32768; + uint32_t const fc2Sfb = fc2B + 512; + uint32_t const barMma = ctx.smemBase + kSmemSmall + kSmallBars + (kMaxRoles + 8) * 8 + kRingSlots * 8; + uint32_t const tmem = *reinterpret_cast(ctx.smem + kSmemSmall + kSmallScalars + 4 * 120); + fenceProxyAsyncSmem(); + tcFenceBefore(); + __syncthreads(); + tcFenceAfter(); + if (warp == 1) + { + fc1PieceMma(tmem, ctx.smemBase + wOff, hb, tmem + kTmemSfa, tmem + kTmemSfb, 0u, barMma); + } + waitBar(ctx, barMma, 0, 134); + __syncthreads(); + tcFenceAfter(); + if (warp == 1) + { + fc2PieceMma(tmem, ctx.smemBase + wOff + kPieceBytes, fc2B, fc2Sfb, tmem + kTmemSfa + 48, barMma); + } + waitBar(ctx, barMma, 1, 135); + tcFenceBefore(); + __syncthreads(); + tcFenceAfter(); +} + +__device__ __noinline__ void stageMoe(Ctx const& ctx, int ownerShared, uint32_t wOff) +{ + int const tid = threadIdx.x; + int const warp = tid >> 5; + int const lane = tid & 31; + int const cta = blockIdx.x; + constexpr uint32_t kRegion2 = kRingSlots * kPieceBytes; // 32 KiB: h during FC1, the slot's FC1 partials in FC2 + uint8_t* region2 = ctx.smem + wOff + kRegion2; + uint32_t const hb = ctx.smemBase + wOff + kRegion2; + uint8_t* hsf = region2 + kHfp4Bytes; + uint8_t* fc2B = region2 + 32768; // FC2 token operand: one atom, 8 rows x 64 B + uint8_t* fc2Sfb = fc2B + 512; // its scales: 32 rows x 16 B (tcgen05.cp source) + uint32_t const barRing0 = ctx.smemBase + kSmemSmall + kSmallBars + (kMaxRoles + 8) * 8; + uint32_t const barMma = barRing0 + kRingSlots * 8; + uint32_t* sTmem = reinterpret_cast(ctx.smem + kSmemSmall + kSmallScalars + 4 * 120); + int* smemMoe = reinterpret_cast(ctx.smem + kSmemSmall + kSmallMoe); + if ((GLM52_FLAGS(ctx) & kFlagOsPrefetch) && ownerShared >= 0 && tid == 0) + { + // Its 32 KiB of down weights go out ahead of the expert pieces (behind them its loads took ~4 us and its + // first FC1 MMA came 1.8 us after the other CTAs'). + prefetchBulkL2( + ptr(ctx, kPWD) + static_cast(ownerShared) * 8 * 2 * kBf16Block, 8 * 2 * kBf16Block); + } + bool const canary = (GLM52_FLAGS(ctx) & kFlagCanaryMoe) != 0; + if (canary) + { + // The first logit word of every router unit (experts [16 et, +16), K quarter kq, token 0). + uint8_t const* a = arena(ctx, kArLogit); + waitCanaries( + ctx, 64, [&](int u) { return a + ((u % 4) * kTok * kExperts + 16 * (u / 4)) * 4; }, 144); + } + // TMEM (taken at kernel start) and the proxy fence for the first pieces' copies into shared memory used before. + fenceProxyAsyncSmem(); + tcFenceBefore(); + __syncthreads(); + tcFenceAfter(); + uint32_t const tmem = *sTmem; + uint32_t mmaPhase = 0; + // The warm-up (kFlagWarmMoe), unless this CTA's H1 unit ran it in its hop wait (kFlagWarmInHop: CTAs 180-211) or + // another earlier unit did (kMoeWarmed: GLM52_ROUTER_LG's logit units, and the shared gate_up with mode 2). + if ((GLM52_FLAGS(ctx) & kFlagWarmMoe) && !((kSpecializedStaticFlags & kFlagWarmInHop) != 0 && cta >= kH1Cta0) + && !(kRouterLg != 0 && smemMoe[kMoeWarmed] != 0)) + { + moeWarmUp(ctx, wOff); + } + // h's loads out now (h is in since the attention hop): they land while the routing runs (after the first pieces' + // requests, h's staging held the first MMA back ~1.2 us: mk-v154a 772.15 vs 781.9 us). + constexpr bool hEarly = true; + uint4 hv[kHN]; + if (hEarly) + { + loadH(ctx, hv); + } + // The token scales early (vectors 1536 + tid of h's loads, tid < 192: row tid / 24, K step pairs 2 (tid % 24) and + // + 1) into TMEM, staged in ring slot 2: off (in two halves after the requests instead). + constexpr bool scalesEarly = false; + if (scalesEarly) + { + uint8_t* stage = ctx.smem + wOff + 2 * kPieceBytes; + for (int v = tid; v < 48 * 24; v += kThreads) + { + *reinterpret_cast(stage + 512 * (v / 24) + 16 * (kTok + v % 24)) = make_uint4(0, 0, 0, 0); + } + if (tid < kHVec - kHVecE) + { + uint4& sv = hv[kHN - 1]; + uint8_t const* src = arena(ctx, kArHfp4) + 16 * (kHVecE + tid); + uint64_t t0 = 0; + while (!full4(sv)) + { + spinCheck(ctx, t0, 138, tid, 0); + sv = ldPoll128(src); + } + int const row = tid / 24; + int const pair = 2 * (tid % 24); + *reinterpret_cast(stage + 512 * pair + 16 * row) = make_uint4(sv.x, 0u, sv.y, 0u); + *reinterpret_cast(stage + 512 * (pair + 1) + 16 * row) = make_uint4(sv.z, 0u, sv.w, 0u); + } + fenceProxyAsyncSmem(); + __syncthreads(); + if (warp == 1) + { + tcFenceAfter(); + if (electOne()) + { + for (int pair = 0; pair < 48; ++pair) + { + utccp1(tmem + kTmemSfb + 4 * pair, smemU32(stage + 512 * pair)); + } + } + __syncwarp(); + commitElect(barMma); + } + waitBar(ctx, barMma, mmaPhase, 139); + mmaPhase ^= 1; + __syncthreads(); + tcFenceAfter(); + } + // This CTA's run bounds f1(c), f1(c + 1), f2(c), f2(c + 1) for every expert count E = 1 + tid / 4, while the + // router finishes (the cut after the routing took ~0.7 us before the first request: -1.6 us per slice, mk-v153a). + int* const sPre = reinterpret_cast(ctx.smem + kSmemRed + 6144); + { + int const e = 1 + (tid >> 2); + int const c = cta + (tid & 1); + int v; + if (e >= 27) + { + int const a2 = MoeCut::alignedF2(c, e); + v = (tid & 2) ? a2 : moeRunStart(c, 24 * e) - a2; + } + else + { + v = (tid & 2) ? moeRunStart(c, 8 * e) : moeRunStart(c, 16 * e); + } + sPre[tid] = v; + } + // The routing first: its requests start the layer's longest stream. h and the token scales are staged while the + // first pieces land (staged before the routing, they outlasted the router's remaining time and held the requests + // back by 1-2 us). + subStamp(ctx, 12); + routeTokens(ctx, smemMoe, cta == 0); + subStamp(ctx, 13); + int const E = smemMoe[kMoeE]; + // The aligned cut's FC1 starts, one per thread into a table (its divisions and the partials' lookups otherwise + // held the CTA ~9 us), in the reduction area's routing-free middle. + int* const sCut = reinterpret_cast(ctx.smem + kSmemRed + 4096); + bool const aligned = E >= 27; + [[maybe_unused]] MoeCut const cut{E, aligned ? sCut : nullptr}; + int const f1a = sPre[4 * (E - 1)]; + int const f1b = sPre[4 * (E - 1) + 1]; + int const f2a = sPre[4 * (E - 1) + 2]; + int const f2b = sPre[4 * (E - 1) + 3]; + int const nPieces = (f1b - f1a) + (f2b - f2a); + { + // The slots' token masks into the list (the routing's scratch is reused by the shared expert). + uint32_t const* sSlotMask = reinterpret_cast(ctx.smem + kSmemRed + 1024); + if (tid < 64) + { + reinterpret_cast(smemMoe + kMoeSlotMask)[tid] + = static_cast(tid < E ? sSlotMask[tid] : 0u); + } + if (tid == 0) + { + smemMoe[kMoePieces] = nPieces; + smemMoe[kMoeF1A] = f1a; + smemMoe[kMoeF1B] = f1b; + smemMoe[kMoeF2A] = f2a; + smemMoe[kMoeF2B] = f2b; + subStamp(ctx, 40); + } + } + __syncthreads(); + // Piece 0 first, pieces 1-2 after the FC2 groups' inputs (with all three at once, every CTA's piece 0 landed only + // with the end of the whole 35 MB burst, ~4.4 us after the request: mk-v140a 793.35 vs 783.95 us per slice; after + // h is staged, ~1 us later still, +9.1 us, mk-v142a); kFlagMoeAllFirst: all three at once. + bool const stagger = !(GLM52_FLAGS(ctx) & kFlagMoeAllFirst); + if (lane == 0 && warp < (stagger ? 1 : kRingSlots) && warp < nPieces) + { + issueMoePiece(ctx, smemMoe, warp, ctx.smemBase + wOff + warp * kPieceBytes, barRing0 + 8 * warp); + } + if (tid == 0) + { + subStamp(ctx, 41); + } + fenceProxyAsyncSmem(); +#if GLM52_LISTS_MODE < 2 + if (aligned) + { + // The table for the FC2 groups' partial lists below. + for (int c = tid; c <= kCtas; c += kThreads) + { + sCut[c] = moeRunStart(c, 24 * E) - MoeCut::alignedF2(c, E); + } + } +#endif + __syncthreads(); + // The FC2 groups' inputs now, off their path (each took an L2 round trip and a loop of divisions there): per group + // g (the CTA's g-th FC2 slot, at most 4) and tile, the count then the (CTA, run) of the FC1 partials, at sFc2 + 9 + // (2 g + tile); the slot's expert scales alpha1 / alpha2 at sFc2[72 + g] / [76 + g]. + uint32_t* const sFc2 = reinterpret_cast(ctx.smem + kSmemRed + 14336); + { + int const slot0 = f2a >> 3; + int const nGroups = f2b > f2a ? ((f2b - 1) >> 3) - slot0 + 1 : 0; + if (nGroups > 4) + { + fail(ctx, 137, nGroups, f2b - f2a); + } + if (tid < 2 * nGroups) + { + int const tile = tid & 1; + int const g0 = 16 * (slot0 + (tid >> 1)) + 8 * tile; + uint32_t* const list = sFc2 + 9 * tid; +#if GLM52_LISTS_MODE >= 1 + loadFc2PartialList(E, slot0 + (tid >> 1), tile, list); +#else + int nPart = 0; + int const cEnd = cut.ctaOfF1(g0 + 7); + for (int c = cut.ctaOfF1(g0); c <= cEnd && nPart < 8; ++c) + { + int const a = cut.f1(c); + if (a < cut.f1(c + 1)) + { + list[1 + nPart++] = static_cast(c * 2 + ((a >> 3) == (g0 >> 3) ? 0 : 1)); + } + } + list[0] = static_cast(nPart); +#endif + } + else if (tid >= 32 && tid < 32 + nGroups) + { + int const expert = smemMoe[kMoeSlotExpert + slot0 + tid - 32]; + sFc2[72 + tid - 32] = __float_as_uint(__ldg(ptr(ctx, kPAlpha1) + expert)); + sFc2[76 + tid - 32] = __float_as_uint(__ldg(ptr(ctx, kPAlpha2) + expert)); + } + } + if (stagger && lane == 0 && warp >= 1 && warp < kRingSlots && warp < nPieces) + { + issueMoePiece(ctx, smemMoe, warp, ctx.smemBase + wOff + warp * kPieceBytes, barRing0 + 8 * warp); + } + // kFlagMoeL2: the rest of the CTA's pieces into L2 behind its ring's requests (one lane per piece), so the ring's + // refills come from L2. + if (GLM52_FLAGS(ctx) & kFlagMoeL2) + { + __syncthreads(); + if (lane == 0 && kRingSlots + warp < nPieces) + { + prefetchMoePiece(ctx, smemMoe, kRingSlots + warp); + } + } + + if (GLM52_STAMPS(ctx) != nullptr && tid == 0) + { + subStamp(ctx, 42); + GLM52_STAMPS(ctx)[kMaxRoles * 4 + 3] = globalTimer(); + } + if (hEarly) + { + finishH(ctx, hv, region2, hsf); + } +#if GLM52_IO_MODE == 2 || GLM52_IO_MODE == 3 + mmaPhase = moeStageH(ctx, region2, hsf, tmem, barMma, mmaPhase, hEarly, scalesEarly, nPieces > 0); +#else + mmaPhase = moeStageH(ctx, region2, hsf, tmem, barMma, mmaPhase, hEarly, scalesEarly); +#endif + stampInput(ctx); +#if GLM52_OWNER_FC1_OVERLAP + // Only the first FC1 piece: its async TCGEN operation can run during the + // unchanged shared-expert activation/HMMA. No ring refill or new phase is + // issued until the original FC1 completion wait and CTA join below. + bool const ownerFirst = ownerShared >= 0 && f1a < f1b; + if (ownerFirst) + { + subStamp(ctx, 80); + waitBar(ctx, barRing0, 0, 123); + subStamp(ctx, 16); + if (warp == 1) + { + int const k0 = f1a & 7; + fc1PieceMma( + tmem, ctx.smemBase + wOff, hb + 6 * k0 * 512, tmem + kTmemSfa, tmem + kTmemSfb + 24 * k0, 0u, barMma); + } + } +#endif + if (ownerShared >= 0) + { + constexpr bool kRstdShared + = (kSpecializedStaticFlags & kFlagHPrime) && (kSpecializedStaticFlags & kFlagRstdShared); + stageOwnerShared(ctx, ownerShared, kRstdShared ? smemMoe : nullptr); + __syncthreads(); + } + uint8_t* p1Base = arena(ctx, kArP1); + bool const fc2Direct = (GLM52_FLAGS(ctx) & kFlagFc2Direct) != 0; +#if GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5 + bool const sparseP1 = fc2Direct && (GLM52_FLAGS(ctx) & kFlagSparseP1); +#endif + int p = 0; + // FC1: a run of pieces of one (slot, tile) accumulates in D; at its end the partial goes to P1[cta][run] (the + // CTA's first or second run). + int run = 0; + for (int g = f1a; g < f1b; ++g, ++p) + { + int const k = g & 7; + bool const first = g == f1a || k == 0; + bool const last = g + 1 == f1b || k == 7; + int const rs = p % kRingSlots; + uint32_t const base = ctx.smemBase + wOff + rs * kPieceBytes; +#if GLM52_OWNER_FC1_OVERLAP + if (!(ownerFirst && p == 0)) + { +#endif + if (p < 8) + { + subStamp(ctx, 80 + p); + } + waitBar(ctx, barRing0 + 8 * rs, (p / kRingSlots) & 1, 123); + if (p < 8) + { + subStamp(ctx, 16 + 2 * p); + } + if (warp == 1) + { + fc1PieceMma(tmem, base, hb + 6 * k * 512, tmem + kTmemSfa + 48 * rs, tmem + kTmemSfb + 24 * k, + first ? 0u : 1u, barMma); + } +#if GLM52_OWNER_FC1_OVERLAP + } +#endif + waitBar(ctx, barMma, mmaPhase, 132); + mmaPhase ^= 1; + // Every thread has seen this phase before the next commit can complete the barrier's next one (a waiter + // that slept through both would wait on the parity of a phase that never comes). + __syncthreads(); + tcFenceAfter(); + if (p < 8) + { + subStamp(ctx, 17 + 2 * p); + } + // The slot's MMAs are complete: request the piece three ahead into it. + if (tid == 0 && p + kRingSlots < nPieces) + { + issueMoePiece(ctx, smemMoe, p + kRingSlots, base, barRing0 + 8 * rs); + } + if (last) + { + // The run's partial: warp w < 4 = rows [32 w, +32) of the tile, 8 tokens per row. + if (warp < 4) + { + float v[8]; + tmemLd<8>(tmem + kTmemD + (static_cast(32 * warp) << 16), v); + if (fc2Direct) + { + // Old row = 32 b + 16 j + m + 8 gate, feature = 2 m + j. For each token the warp's scalar + // stores cover one contiguous 128-byte block, in [16 features][up, gate] order. + int const feature = 2 * (lane & 7) + (lane >> 4); + int const gate = (lane >> 3) & 1; + uint8_t* dst = p1Base + static_cast(cta * 2 + run) * 128 * kTok * 4; +#if GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5 + int const tokenMask + = sparseP1 ? reinterpret_cast(smemMoe + kMoeSlotMask)[g >> 4] : 255; +#endif +#pragma unroll + for (int t = 0; t < kTok; ++t) + { +#if GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5 + if (!((tokenMask >> t) & 1)) + { + continue; // every reader of this run uses the same expert slot mask + } +#endif + st32(dst + 4 * (t * 128 + 32 * warp + 2 * feature + gate), lamport(__float_as_uint(v[t]))); + } + } + else + { + uint8_t* dst = p1Base + ((static_cast(cta * 2 + run) * 128 + 32 * warp + lane) * kTok) * 4; + st128(dst, + make_uint4(lamport(__float_as_uint(v[0])), lamport(__float_as_uint(v[1])), + lamport(__float_as_uint(v[2])), lamport(__float_as_uint(v[3])))); + st128(dst + 16, + make_uint4(lamport(__float_as_uint(v[4])), lamport(__float_as_uint(v[5])), + lamport(__float_as_uint(v[6])), lamport(__float_as_uint(v[7])))); + } + } + ++run; + tcFenceBefore(); + __syncthreads(); + tcFenceAfter(); + } + } + // FC2: the CTA's eighths slot by slot; per slot the FC1 partials are summed (SwiGLU + NVFP4 = the token operand) + // once for all its eighths here. + // The layer's FC2 activation global scale (Alpha2[kExperts]; 1 for GLM-5.2-NVFP4). + float const fc2InScale = __ldg(ptr(ctx, kPAlpha2) + kExperts); + int grp = 0; + for (int j = f2a; j < f2b; ++grp) + { + int const slot = j >> 3; + int const jEnd = min(f2b, 8 * slot + 8); + int const mask = reinterpret_cast(smemMoe + kMoeSlotMask)[slot]; + // The partials of each tile (the CTAs whose FC1 run covers pieces [16 slot + 8 tile, +8), their run 0 or 1) + // and the expert's scales: listed before the FC1 pieces. + uint32_t const* const lists = sFc2 + 18 * grp; + float const alphaUp = __uint_as_float(sFc2[72 + grp]); + float const alphaDown = __uint_as_float(sFc2[76 + grp]); +#if GLM52_FC2_SPARSE_WIDE && (GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5) + int const sparseTokens = __popc(static_cast(mask)); + bool const useSparseWide = fc2Direct && sparseP1 && sparseTokens >= 1 && sparseTokens <= 2; + if (useSparseWide) + { + // The full token operand and scale padding are cleared before active + // writers. The original preparation already has this CTA join. + if (tid < 32) + { + *reinterpret_cast(fc2B + 16 * tid) = make_uint4(0, 0, 0, 0); + *reinterpret_cast(fc2Sfb + 16 * tid) = make_uint4(0, 0, 0, 0); + } + __syncthreads(); + if (grp < 2) + { + subStamp(ctx, 73 + grp); + } + int const activeGroup = tid >> 7; + if (activeGroup < sparseTokens) + { + // Four whole warps own one active token, one feature per lane. + // A warp stays within one 64-feature tile and one producer list. + unsigned tokenBits = static_cast(mask); + if (activeGroup != 0) + { + tokenBits &= tokenBits - 1; + } + int const t = __ffs(static_cast(tokenBits)) - 1; + int const f = tid & 127; + int const tile = f >> 6; + int const nPart = static_cast(lists[9 * tile]); + uint32_t const byteOff = 4 * (t * 128 + 2 * (f & 63)); + float up = 0.f; + float gate = 0.f; +#pragma unroll 1 + for (int q0 = 0; q0 < nPart; q0 += 4) + { + uint8_t const* src[4]; + uint2 words[4]; + bool use[4]; +#pragma unroll + for (int q = 0; q < 4; ++q) + { + use[q] = q0 + q < nPart; + src[q] = nullptr; + if (use[q]) + { + src[q] + = p1Base + static_cast(lists[9 * tile + 1 + q0 + q]) * 128 * kTok * 4 + byteOff; + } + words[q] = use[q] ? ldPoll64(src[q]) : make_uint2(0, 0); + } + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int q = 0; q < 4; ++q) + { + ok &= !use[q] || full2(words[q]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, 124, slot, q0); +#pragma unroll + for (int q = 0; q < 4; ++q) + { + if (use[q] && !full2(words[q])) + { + words[q] = ldPoll64(src[q]); + } + } + } + // Preserve +0 and the original ascending q-by-q FP32 adds. +#pragma unroll + for (int q = 0; q < 4; ++q) + { + if (use[q]) + { + up += __uint_as_float(words[q].x); + gate += __uint_as_float(words[q].y); + } + } + } + float const alpha = alphaUp; + float const gv = fmaf(up, alpha, 0.f) + * (gate * rcpApproxFtz(1.f + ex2ApproxFtz(-(gate * (1.4426950408889634f * alpha))))); + float amax = fabsf(gv); + amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, 8)); + amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, 4)); + amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, 2)); + amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, 1)); + uint32_t const sf = toE4m3(fc2InScale * (amax * alpha * 0.16666667f)); + float const sff = static_cast(*reinterpret_cast<__nv_fp8_e4m3 const*>(&sf)); + float const r = sff != 0.f ? rcpApproxFtz(sff * rcpApproxFtz(fc2InScale)) : 0.f; + float const scaled = (gv * r) * alpha; + float const adjacent = __shfl_xor_sync(0xffffffffu, scaled, 1); + if ((f & 1) == 0) + { + uint32_t const q01 = toE2m1x2(scaled, adjacent); + fc2B[t * 64 + (((f >> 5) ^ ((t >> 1) & 3)) << 4) + ((f >> 1) & 15)] = static_cast(q01); + } + if ((f & 15) == 0) + { + int const b = f >> 4; + fc2Sfb[16 * t + (b < 4 ? b : 4 + b)] = static_cast(sf); + } + } + if (grp < 2) + { + subStamp(ctx, 75 + grp); + } + } + else + { +#endif + if (tid < 32) + { + *reinterpret_cast(fc2Sfb + 16 * tid) = make_uint4(0, 0, 0, 0); + } + __syncthreads(); + // A thread owns four adjacent features of one token: [up0, gate0, ..., up3, gate3]. The optional P1 + // layout lets it sum those values directly; the baseline first transposes all partial sums through region2. + int const t = tid / 32; + int const b = (tid / 4) % 8; + int const sub = tid % 4; + float sum[8]; +#if GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5 + bool const activeP1 = !sparseP1 || ((mask >> t) & 1); // one token per whole consumer warp +#endif + if (grp < 2) + { + subStamp(ctx, 73 + grp); + } + if (fc2Direct) + { +#if GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5 + if (activeP1) + { +#endif + int const tile = b >> 2; + int const nPart = static_cast(lists[9 * tile]); + // Both half-warps run the same poll (4 or 8 runs), even when their tiles have different partial + // counts. Inactive slots contribute nothing and never load a list entry. All 32 lanes reach each + // warpAll. + int const nMax = max(static_cast(lists[0]), static_cast(lists[9])); + uint8_t const* const base = p1Base + 4 * (t * 128 + 32 * (b & 3) + 8 * sub); + if (nMax > 4) + { + pollFc2Runs<8>(ctx, base, lists + 9 * tile, nPart, sum, slot); + } + else + { + pollFc2Runs<4>(ctx, base, lists + 9 * tile, nPart, sum, slot); + } +#if GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5 + } +#endif + if (grp < 2) + { + subStamp(ctx, 75 + grp); + } + } + else + { + float* psum = reinterpret_cast(region2); + { + int const nPart0 = static_cast(lists[0]); + int const nPart1 = static_cast(lists[9]); + float4 acc[2] = {make_float4(0.f, 0.f, 0.f, 0.f), make_float4(0.f, 0.f, 0.f, 0.f)}; +#pragma unroll 1 + for (int q0 = 0; q0 < max(nPart0, nPart1); q0 += 4) + { + uint8_t const* src[2][4]; + uint4 w[2][4]; + bool use[2][4]; +#pragma unroll + for (int tile = 0; tile < 2; ++tile) + { + int const nPart = tile ? nPart1 : nPart0; +#pragma unroll + for (int q = 0; q < 4; ++q) + { + use[tile][q] = q0 + q < nPart; + src[tile][q] = p1Base + + static_cast(lists[9 * tile + 1 + (use[tile][q] ? q0 + q : 0)]) * 128 + * kTok * 4 + + 16 * tid; + w[tile][q] = use[tile][q] ? ldPoll128(src[tile][q]) : make_uint4(0, 0, 0, 0); + } + } + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int tile = 0; tile < 2; ++tile) + { +#pragma unroll + for (int q = 0; q < 4; ++q) + { + ok &= !use[tile][q] || full4(w[tile][q]); + } + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, 124, slot, q0); +#pragma unroll + for (int tile = 0; tile < 2; ++tile) + { +#pragma unroll + for (int q = 0; q < 4; ++q) + { + if (use[tile][q] && !full4(w[tile][q])) + { + w[tile][q] = ldPoll128(src[tile][q]); + } + } + } + } +#pragma unroll + for (int tile = 0; tile < 2; ++tile) + { +#pragma unroll + for (int q = 0; q < 4; ++q) + { + if (use[tile][q]) + { + acc[tile].x += __uint_as_float(w[tile][q].x); + acc[tile].y += __uint_as_float(w[tile][q].y); + acc[tile].z += __uint_as_float(w[tile][q].z); + acc[tile].w += __uint_as_float(w[tile][q].w); + } + } + } + } + *reinterpret_cast(psum + 4 * tid) = acc[0]; + *reinterpret_cast(psum + 128 * kTok + 4 * tid) = acc[1]; + } + if (grp < 2) + { + subStamp(ctx, 75 + grp); + } + __syncthreads(); +#pragma unroll + for (int u = 0; u < 4; ++u) + { + int const lf = 4 * sub + u; + int const upRow = 32 * b + 16 * (lf & 1) + (lf >> 1); + sum[2 * u] = psum[upRow * kTok + t]; + sum[2 * u + 1] = psum[(upRow + 8) * kTok + t]; + } + } + // SwiGLU + NVFP4 is shared by both paths, with identical feature ownership and arithmetic. +#if GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5 + if (activeP1) +#endif + { + float const alpha = alphaUp; + float gv[4]; +#pragma unroll + for (int u = 0; u < 4; ++u) + { + float const x0 = sum[2 * u]; + float const x1 = sum[2 * u + 1]; + gv[u] = fmaf(x0, alpha, 0.f) + * (x1 * rcpApproxFtz(1.f + ex2ApproxFtz(-(x1 * (1.4426950408889634f * alpha))))); + } + float amax = fmaxf(fmaxf(fabsf(gv[0]), fabsf(gv[1])), fmaxf(fabsf(gv[2]), fabsf(gv[3]))); + amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, 1)); + amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, 2)); + uint32_t const sf = toE4m3(fc2InScale * (amax * alpha * 0.16666667f)); + float const sff = static_cast(*reinterpret_cast<__nv_fp8_e4m3 const*>(&sf)); + float const r = sff != 0.f ? rcpApproxFtz(sff * rcpApproxFtz(fc2InScale)) : 0.f; + uint32_t const q01 = toE2m1x2((gv[0] * r) * alpha, (gv[1] * r) * alpha); + uint32_t const q23 = toE2m1x2((gv[2] * r) * alpha, (gv[3] * r) * alpha); + // Token t's 64 B row: byte 8 b + 2 sub = chunk b / 2 (swizzled), byte 8 (b & 1) + 2 sub in it. + *reinterpret_cast(fc2B + t * 64 + (((b >> 1) ^ ((t >> 1) & 3)) << 4) + 8 * (b & 1) + 2 * sub) + = static_cast(q01 | (q23 << 8)); + if (sub == 0) + { + fc2Sfb[16 * t + (b < 4 ? b : 4 + b)] = static_cast(sf); + } + } +#if GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5 + else + { + // This warp's token is absent from the expert. No P1 words exist for it; write a complete zero operand. + // Each lane owns the same disjoint two bytes as the active quantizer; the scale padding was zeroed + // above. + *reinterpret_cast(fc2B + t * 64 + (((b >> 1) ^ ((t >> 1) & 3)) << 4) + 8 * (b & 1) + 2 * sub) + = 0; + if (sub == 0) + { + fc2Sfb[16 * t + (b < 4 ? b : 4 + b)] = 0; + } + } +#endif +#if GLM52_FC2_SPARSE_WIDE && (GLM52_FLAGS_MODE == 3 || GLM52_FLAGS_MODE == 5) + } +#endif + fenceProxyAsyncSmem(); + __syncthreads(); + if (grp < 8) + { + subStamp(ctx, 32 + grp); + } + for (; j < jEnd; ++j, ++p) + { + int const o = j & 7; + int const rs = p % kRingSlots; + uint32_t const base = ctx.smemBase + wOff + rs * kPieceBytes; + if (p < 8) + { + subStamp(ctx, 80 + p); + } + waitBar(ctx, barRing0 + 8 * rs, (p / kRingSlots) & 1, 125); + if (p < 8) + { + subStamp(ctx, 16 + 2 * p); + } + if (warp == 1) + { + fc2PieceMma(tmem, base, smemU32(fc2B), smemU32(fc2Sfb), tmem + kTmemSfa + 48 * rs, barMma); + } + waitBar(ctx, barMma, mmaPhase, 133); + mmaPhase ^= 1; + if (p + kRingSlots < nPieces) + { + // Every thread has seen the slot's phase before the refill below can complete its next one (as in + // FC1: a waiter that slept through both would wait on the parity of a phase that never comes). + __syncthreads(); + } + tcFenceAfter(); + if (p < 8) + { + subStamp(ctx, 17 + 2 * p); + } + if (tid == 0 && p + kRingSlots < nPieces) + { + issueMoePiece(ctx, smemMoe, p + kRingSlots, base, barRing0 + 8 * rs); + } + // Output rows: warp w reads TMEM lane quarter q = w % 4 (rows [32 q, +32) of each tile) of tiles [3 (w / + // 4), +3), the three tiles in one wait; row 32 q + l holds feature 32 q + 4 (l & 7) + l / 8, so lane g < 8 + // gathers features 4 g .. 4 g + 3 from lanes g, g + 8, g + 16, g + 24. + { + int const q = warp & 3; + int const h = warp >> 2; + float const a2 = alphaDown; + uint8_t* outRows = reinterpret_cast( + ptr<__nv_bfloat16>(ctx, kPFc2out) + static_cast(slot) * kTok * kHidden); + float vt[3][8]; + tmemLd3x8(tmem + kTmemD + 24 * h + (static_cast(32 * q) << 16), vt); + // Tokens outer, the three tiles inner: one mask test per token (the parallel agent's FC2 drain order; + // tiles outer was ~7 us slower per slice). +#pragma unroll + for (int t = 0; t < kTok; ++t) + { + if ((GLM52_FLAGS(ctx) & kFlagFc2Masked) && !((mask >> t) & 1)) + { + continue; // identical mask for every lane; all shuffles remain converged + } +#pragma unroll + for (int i = 0; i < 3; ++i) + { + int const ti = 3 * h + i; + float const* v = vt[i]; + int const col = 128 * (6 * o + ti) + 32 * q + 4 * (lane & 7); + float const x = (v[t] + 0.f) * a2; + float const x1 = __shfl_down_sync(0xffffffffu, x, 8); + float const x2 = __shfl_down_sync(0xffffffffu, x, 16); + float const x3 = __shfl_down_sync(0xffffffffu, x, 24); + if (lane < 8 && ((mask >> t) & 1)) + { + st64(outRows + (static_cast(t) * kHidden + col) * 2, + make_uint2(lamport(packBf16(x, x1)), lamport(packBf16(x2, x3)))); + } + } + } + } + tcFenceBefore(); + __syncthreads(); + tcFenceAfter(); + } + } + tcFenceBefore(); + __syncthreads(); + if (warp == 0) + { + tcFenceAfter(); + tmemDealloc(tmem, kTmemCols); + } +} + +// Shared expert of output columns [128 J, +128) (SiLU(gate) * up, down), before the CTA's MoE unit or inside it +// (kFlagOsInMoe): BF16 [8 tokens][128] into the CTA's residual area for its owner unit. smemMoe (inside the MoE, with +// kFlagRstdShared): the routing's MoE list, the tokens' 1 / rms in it. +__device__ __noinline__ void stageOwnerShared(Ctx const& ctx, int J, int const* smemMoe) +{ + int const tid = threadIdx.x; + int const warp = tid >> 5; + int const lane = tid & 31; + __nv_bfloat16* sA = reinterpret_cast<__nv_bfloat16*>(ctx.smem + kSmemRed); // [8][128] + __nv_bfloat16* sOut = reinterpret_cast<__nv_bfloat16*>(ctx.smem + kSmemRes); // [8][128] + // The down-projection weights of the warp's tile (output columns [16 w, +16) of the owner's 128, K = 128 in two + // chunks) straight into the MMA fragment registers: their loads overlap the polls below. + uint4 wa[2][4]; + { + uint8_t const* wd + = ptr(ctx, kPWD) + static_cast(J * 8 + warp) * 2 * kBf16Block + lane * 16; +#pragma unroll + for (int ch = 0; ch < 2; ++ch) + { +#pragma unroll + for (int pc = 0; pc < 4; ++pc) + { + wa[ch][pc] = ldcg128(wd + ch * kBf16Block + pc * 512); + } + } + } + if (GLM52_FLAGS(ctx) & kFlagCanaryOwner) + { + // The first word of every shared gate_up unit (rows [16 ft, +16) x K quarter kq). + uint8_t const* a = arena(ctx, kArSh); + waitCanaries( + ctx, 32, [&](int u) { return a + ((u % 4) * 8 + u / 4) * 32 * kTok * 4; }, 143); + } + // Shared expert activation: thread (feature f, token half): gate and up of 4 tokens over the 4 K quarters. + { + int const f = tid / 2; + int const half = tid % 2; + int const ft = f / 16; + int const r = f % 16; + uint8_t const* sh = arena(ctx, kArSh) + ((ft * 32 + r) * kTok + 4 * half) * 4; + // Vector 2 kq: gate rows (row r), 2 kq + 1: up rows (row 16 + r); K quarter kq at + kq * 8 * 32 * 8 floats. + uint4 v[8]; + { + uint8_t const* srcs[8]; +#pragma unroll + for (int kq = 0; kq < 4; ++kq) + { + srcs[2 * kq] = sh + kq * 8 * 32 * kTok * 4; + srcs[2 * kq + 1] = sh + (kq * 8 * 32 + 16) * kTok * 4; + } +#pragma unroll + for (int i = 0; i < 8; ++i) + { + v[i] = ldPoll128(srcs[i]); + } + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int i = 0; i < 8; ++i) + { + ok &= full4(v[i]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, 126, J, 0); +#pragma unroll + for (int i = 0; i < 8; ++i) + { + if (!full4(v[i])) + { + v[i] = ldPoll128(srcs[i]); + } + } + } + } + float gate[4] = {0.f, 0.f, 0.f, 0.f}; + float up[4] = {0.f, 0.f, 0.f, 0.f}; +#pragma unroll + for (int kq = 0; kq < 4; ++kq) + { + uint32_t const gw[4] = {v[2 * kq].x, v[2 * kq].y, v[2 * kq].z, v[2 * kq].w}; + uint32_t const uw[4] = {v[2 * kq + 1].x, v[2 * kq + 1].y, v[2 * kq + 1].z, v[2 * kq + 1].w}; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + gate[j] += __uint_as_float(gw[j]); + up[j] += __uint_as_float(uw[j]); + } + } + if constexpr ((kSpecializedStaticFlags & kFlagHPrime) != 0) + { + // The shared gate_up ran on h' = res x gamma: its tokens' 1 / rms. +#pragma unroll + for (int j = 0; j < 4; ++j) + { + float const r = smemMoe != nullptr ? __int_as_float(smemMoe[kMoeRcp + 4 * half + j]) + : hopRcp(ctx, 4 * half + j, 147); + gate[j] *= r; + up[j] *= r; + } + } +#pragma unroll + for (int j = 0; j < 4; ++j) + { + float const gb = bf16Round(gate[j]); + float const ub = bf16Round(up[j]); + float const sig = 1.f / (1.f + __expf(-gb)); + sA[(4 * half + j) * 128 + f] = __float2bfloat16_rn(gb * sig * ub); + } + } + __syncthreads(); + { + uint4 b[2][2]; + loadBSmem(b[0], sA, 128, 0); + loadBSmem(b[1], sA, 128, 1); + float acc[4] = {0.f, 0.f, 0.f, 0.f}; +#pragma unroll + for (int ch = 0; ch < 2; ++ch) + { + uint4 const& p0 = wa[ch][0]; + uint4 const& p1 = wa[ch][1]; + uint4 const& p2 = wa[ch][2]; + uint4 const& p3 = wa[ch][3]; + mmaBf16(acc, p0.x, p2.x, p0.y, p2.y, b[ch][0].x, b[ch][0].y); + mmaBf16(acc, p0.z, p2.z, p0.w, p2.w, b[ch][0].z, b[ch][0].w); + mmaBf16(acc, p1.x, p3.x, p1.y, p3.y, b[ch][1].x, b[ch][1].y); + mmaBf16(acc, p1.z, p3.z, p1.w, p3.w, b[ch][1].z, b[ch][1].w); + } + int const g = lane >> 2; + int const c = lane & 3; + // Tile w = output columns [16 w, +16) of the owner's 128. + sOut[(2 * c) * 128 + 16 * warp + g] = __float2bfloat16_rn(acc[0]); + sOut[(2 * c + 1) * 128 + 16 * warp + g] = __float2bfloat16_rn(acc[1]); + sOut[(2 * c) * 128 + 16 * warp + g + 8] = __float2bfloat16_rn(acc[2]); + sOut[(2 * c + 1) * 128 + 16 * warp + g + 8] = __float2bfloat16_rn(acc[3]); + } +} + +// Owner of output columns [128 J, +128): the top-8 combine of the routed FC2 rows (which it empties: it is their only +// reader) plus the shared expert's output (stageOwnerShared, same CTA), published as the MoE all-reduce shard of this +// rank into every GPU's slot. +__device__ __noinline__ void stageOwner(Ctx const& ctx, int J) +{ + int const tid = threadIdx.x; + __nv_bfloat16 const* sOut = reinterpret_cast<__nv_bfloat16 const*>(ctx.smem + kSmemRes); // [8][128] + // Thread 128: this GPU's hop buffers and x slot of the next launch are emptied before the shard goes out (the CTA's + // barrier below orders the publish after it), concurrently with the polls of the routed rows. + if (tid == 128) + { + uint32_t const* c = ctx.counters + kCClear; + uint64_t t0 = 0; + uint32_t v; + while ((v = ldAcquire(c)) < static_cast(kCountedClear)) + { + spinCheck(ctx, t0, 129, kCClear, static_cast(v)); + } + } + // Thread (token, vector of 8 columns): 8 x 16 = 128 threads. + uint4 shard = make_uint4(0, 0, 0, 0); + size_t off = 0; + uint8_t* srcs[8]; + bool const clearFirst = (GLM52_FLAGS(ctx) & kFlagOwnerClearFirst) != 0; + if (tid < 128) + { + int const tok = tid / 16; + int const v = tid % 16; + int const col = 128 * J + 8 * v; + // Slots and weights of the token's top-8: from this CTA's MoE work list (the MoE unit ran before). + int const* smemMoe = reinterpret_cast(ctx.smem + kSmemSmall + kSmallMoe); + int slots[8]; + float wts[8]; +#pragma unroll + for (int k = 0; k < 8; ++k) + { + slots[k] = smemMoe[kMoeSlotOfTk + tok * 8 + k]; + wts[k] = __int_as_float(smemMoe[kMoeWeight + tok * 8 + k]); + } + uint8_t* fc2 = reinterpret_cast(ptr<__nv_bfloat16>(ctx, kPFc2out)); + uint4 o[8]; +#pragma unroll + for (int k = 0; k < 8; ++k) + { + srcs[k] = fc2 + ((static_cast(slots[k]) * kTok + tok) * kHidden + col) * 2; + o[k] = ldPoll128(srcs[k]); + } + { + uint64_t t0 = 0; + while (true) + { + bool ok = true; +#pragma unroll + for (int k = 0; k < 8; ++k) + { + ok &= full4(o[k]); + } + if (warpAll(ok)) + { + break; + } + spinCheck(ctx, t0, 130, J, tok); +#pragma unroll + for (int k = 0; k < 8; ++k) + { + if (!full4(o[k])) + { + o[k] = ldPoll128(srcs[k]); + } + } + } + } + if (clearFirst) + { +#pragma unroll + for (int k = 0; k < 8; ++k) + { + st128(srcs[k], make_uint4(kEmpty, kEmpty, kEmpty, kEmpty)); + } + } + stampInput(ctx); + float routed[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) + { + routed[j] = 0.f; + } +#pragma unroll + for (int k = 0; k < 8; ++k) + { + uint32_t const ow[4] = {o[k].x, o[k].y, o[k].z, o[k].w}; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + routed[2 * j] = fmaf(wts[k], bf16Lo(ow[j]), routed[2 * j]); + routed[2 * j + 1] = fmaf(wts[k], bf16Hi(ow[j]), routed[2 * j + 1]); + } + } + uint32_t pk[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) + { + float const s0 = __bfloat162float(sOut[tok * 128 + 8 * v + 2 * j]); + float const s1 = __bfloat162float(sOut[tok * 128 + 8 * v + 2 * j + 1]); + pk[j] = lamport(packBf16(s0 + bf16Round(routed[2 * j]), s1 + bf16Round(routed[2 * j + 1]))); + } + shard = make_uint4(pk[0], pk[1], pk[2], pk[3]); + off = (static_cast(tok * kRanks + ctx.rank) * kHidden + col) * 2; + } + __syncthreads(); + if (tid < 128) + { + st128(ptr(ctx, kPH2Mc) + static_cast(ctx.parity) * kHopBytes + off, shard); + // The routed rows back to empty for a later launch, after the shard (stores leave in issue order: the merge + // units' clears before their output held o_proj's input up by ~1 us, mk-v111a). + if (!clearFirst) + { +#pragma unroll + for (int k = 0; k < 8; ++k) + { + st128(srcs[k], make_uint4(kEmpty, kEmpty, kEmpty, kEmpty)); + } + } + } +} diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py index cf3914dbdff0..fd5f18340773 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py @@ -301,3 +301,19 @@ def fake(*args, **kwargs): _host_query(f"{_NS}::glm_dense_mlp_small_m_supported") _host_query(f"{_NS}::glm_intra_node_reduce_supported") _host_query(f"{_NS}::glm_mnnvl_hop_supported") +_host_query(f"{_NS}::glm_megakernel_info") + + +@torch.library.register_fake(f"{_NS}::glm_megakernel_indexer_tmap") +def _glm_megakernel_indexer_tmap(pool: torch.Tensor, rows: int): + return torch.empty((128,), dtype=torch.uint8, device="cpu") + + +@torch.library.register_fake(f"{_NS}::glm_megakernel_weight_table") +def _glm_megakernel_weight_table(params: torch.Tensor, offset: int): + return None + + +@torch.library.register_fake(f"{_NS}::glm_megakernel") +def _glm_megakernel(params, x, residual, topk, block_table, kv_lens, state, kv_cache, pdl): + return torch.empty_like(x), torch.empty_like(residual) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/modeling.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/modeling.py index f345055bead0..19e0e1a73afe 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/modeling.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/modeling.py @@ -46,6 +46,7 @@ from . import weights as _weights from ._dsa import forward_generation_sparse_attn from ._full_context import FullContextIndexer +from ._megakernel import GlmMegakernel from ._model import Glm52Model from ._model import setup_aliases as _setup_aliases from ._mtp_worker import MTPDraftWorker @@ -321,6 +322,7 @@ def __init__(self, core: "Glm52Nvfp4Sm107LowLatencyCore") -> None: self._full_context = FullContextIndexer(self, core._decode_context_capacity, device) if self.mapping.tp_size == 8: self._prepare_tp8_rows(device) + self._megakernel = GlmMegakernel.create(self) def _prepare_tp8_rows(self, device: torch.device) -> None: """Check the complete TP8 projection/collective route before profiling writes KV.""" @@ -639,7 +641,17 @@ def forward( return hidden_states sharded = self._shard_indexer_keys(attn_metadata) self._keys_sharded = sharded + megakernel = self._megakernel + if megakernel is not None and ( + not sharded or not megakernel.begin(hidden_states, attn_metadata, spec_metadata) + ): + megakernel = None for layer in layers: + if megakernel is not None and layer.layer_idx in megakernel.layers: + hidden_states, residual = megakernel.layers[layer.layer_idx].forward( + hidden_states, residual, attn_metadata, self._hist_buffers["topk"][:8] + ) + continue attn_out = self._attention(layer, position_ids, hidden_states, attn_metadata, sharded) quantized = None if self._ipc is not None and not isinstance(layer.mlp, GatedMLP): diff --git a/tests/integration/defs/accuracy/modeling_v2/test_glm_moe_dsa.py b/tests/integration/defs/accuracy/modeling_v2/test_glm_moe_dsa.py index 06d294250168..5682b1b356d4 100644 --- a/tests/integration/defs/accuracy/modeling_v2/test_glm_moe_dsa.py +++ b/tests/integration/defs/accuracy/modeling_v2/test_glm_moe_dsa.py @@ -280,6 +280,7 @@ def test_gsm8k_identity_vs_mtp1_and_acceptance(self, monkeypatch: pytest.MonkeyP _RANK_ENV = { "TRTLLM_DSA_INDEXER_BF16": "1", "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS": "0", + "TRTLLM_GLM5_MEGAKERNEL": os.environ.get("TRTLLM_GLM5_MEGAKERNEL", "0"), } @@ -421,7 +422,9 @@ def _assert_optimized_paths(llm: LLM, trace_dir: Path, *, optimized: bool = True # These kernels are exclusive to the GLM optimized decode and MTP paths. # Requiring every TP rank prevents a route-selection or warmup-only pass. required_kernels = ( - "routingFusedRouterKernel", + "glm52PlainLayerKernel" + if _RANK_ENV["TRTLLM_GLM5_MEGAKERNEL"] == "1" + else "routingFusedRouterKernel", "draftEhprojTableKernel", "draftSampleGlueKernel", ) From 20fb70c5b6595198b41f329c96c7187be2df465a Mon Sep 17 00:00:00 2001 From: Dongfeng Yu Date: Tue, 3 Nov 2026 19:27:21 -0800 Subject: [PATCH 2/5] [None][fix] Reserve speculative headroom in the Rubin megakernel Include one additional 64-key page in the candidate capacity so the 1M-token C1 configuration also accommodates the KV manager speculative lead. Preserve the capacity guard and existing indexer bounds. Signed-off-by: Dongfeng Yu --- .../models/glm_moe_dsa/csrc/CMakeLists.txt | 40 +++++++++---------- .../csrc/glmMegakernel.cu | 5 ++- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/csrc/CMakeLists.txt b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/csrc/CMakeLists.txt index fb682545f69a..c9d5570fa2ac 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/csrc/CMakeLists.txt +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/csrc/CMakeLists.txt @@ -28,36 +28,34 @@ add_modeling_v2_target( # Preserve the C1 megakernel specialization and native Rubin code generation. set(mega_source - "${CMAKE_CURRENT_SOURCE_DIR}/../glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu") + "${CMAKE_CURRENT_SOURCE_DIR}/../glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu" +) set_property( SOURCE "${mega_source}" APPEND PROPERTY COMPILE_DEFINITIONS - GLM52_FC2_SPARSE_WIDE=1 - GLM52_OWNER_FC1_OVERLAP=1 - GLM52_ROUTER_LG=2 - GLM52_CAND_FILTER_ROLL=0 - GLM52_SEL_BATCH_MMA=1 - GLM52_LEAN=1 - GLM52_FLAGS_MODE=5 - GLM52_IO_MODE=3 - GLM52_STAMPS_MODE=1 - GLM52_SETUP_MODE=1 - GLM52_LISTS_MODE=2 - GLM52_TOPK_UNION=1 - GLM52_TOPK_RSH=1 - GLM52_STATIC_FLAGS=2752ll - GLM52_SMEM_MAIN=200704 -) + GLM52_FC2_SPARSE_WIDE=1 + GLM52_OWNER_FC1_OVERLAP=1 + GLM52_ROUTER_LG=2 + GLM52_CAND_FILTER_ROLL=0 + GLM52_SEL_BATCH_MMA=1 + GLM52_LEAN=1 + GLM52_FLAGS_MODE=5 + GLM52_IO_MODE=3 + GLM52_STAMPS_MODE=1 + GLM52_SETUP_MODE=1 + GLM52_LISTS_MODE=2 + GLM52_TOPK_UNION=1 + GLM52_TOPK_RSH=1 + GLM52_STATIC_FLAGS=2752ll + GLM52_SMEM_MAIN=200704) set_property( - SOURCE "${mega_source}" - APPEND + SOURCE "${mega_source}" APPEND PROPERTY COMPILE_OPTIONS --maxrregcount=200 --ftz=false --prec-div=true --prec-sqrt=true) if(107 IN_LIST CMAKE_CUDA_ARCHITECTURES_ORIG) set_property( - SOURCE "${mega_source}" - APPEND + SOURCE "${mega_source}" APPEND PROPERTY COMPILE_OPTIONS --generate-code=arch=compute_107a,code=sm_107a) endif() diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu index 1b3f4ae4e616..82d3219af58a 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu @@ -200,8 +200,9 @@ static_assert(kIdxPageBytes % 128 == 0 && kIdxRows == 128 * kIdxQUnitRows, "inde constexpr int kIdxMaxPages = 16; // pages per IdxSel round (its K region's) constexpr int kSelBatchPages = kIdxMaxPages / 2; // the row shard's page batches (half the K region each) #if GLM52_TOPK_UNION -constexpr int kCandList = 1 << 20; // candidates per row (list A): any row's keys (kIdxMaxKeys) -constexpr int kCandCap = kCandList + 8192; // per row: list A, then the union's 13,568 words +// Reserve one page for speculative lookahead beyond a 1M-token logical context. +constexpr int kCandList = (1 << 20) + kIdxPageKeys; +constexpr int kCandCap = kCandList + 8192; // per row: list A, then the union's 13,568 words #else constexpr int kCandList = 1 << 18; // candidates per list (list A, then list B): any row's keys constexpr int kCandCap = 2 * kCandList; // top-k candidates per row: list A, then list B From 972b2a6668270b8622cb7b93012f085b98ea7bcf Mon Sep 17 00:00:00 2001 From: Dongfeng Yu Date: Sat, 7 Nov 2026 09:58:40 -0800 Subject: [PATCH 3/5] [None][fix] Keep the megakernel setup query on its native implementation Signed-off-by: Dongfeng Yu --- .../glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py index fd5f18340773..90171ef562c5 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py @@ -301,7 +301,6 @@ def fake(*args, **kwargs): _host_query(f"{_NS}::glm_dense_mlp_small_m_supported") _host_query(f"{_NS}::glm_intra_node_reduce_supported") _host_query(f"{_NS}::glm_mnnvl_hop_supported") -_host_query(f"{_NS}::glm_megakernel_info") @torch.library.register_fake(f"{_NS}::glm_megakernel_indexer_tmap") From 71eaa660dda2ec7bd5a946aece95dc8fd0974ae8 Mon Sep 17 00:00:00 2001 From: Dongfeng Yu Date: Sat, 7 Nov 2026 10:08:42 -0800 Subject: [PATCH 4/5] [None][fix] Honor the CUDA tensor-map type alignment Signed-off-by: Dongfeng Yu --- .../glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp index 9f27d4a9345d..36f31377f392 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp @@ -67,7 +67,7 @@ at::Tensor glmMegakernelIndexerTmap(at::Tensor const& pool, int64_t rows) "glm_megakernel_indexer_tmap: pool storage must contain rows of 128 bytes"); c10::cuda::CUDAGuard const deviceGuard(pool.device()); auto out = at::empty({128}, at::TensorOptions().dtype(at::kByte).device(at::kCPU)); - alignas(64) CUtensorMap tensorMap{}; + CUtensorMap tensorMap{}; auto const result = mega::glm52PlainLayerIdxTmap(&tensorMap, pool.data_ptr(), rows); TORCH_CHECK(result == CUDA_SUCCESS, "glm_megakernel_indexer_tmap: CUDA driver error ", result); std::copy_n(reinterpret_cast(&tensorMap), sizeof(tensorMap), out.data_ptr()); From 42bbd1668afa3c2dc045faad25effb8bfafdfe46 Mon Sep 17 00:00:00 2001 From: Dongfeng Yu Date: Sat, 7 Nov 2026 13:46:22 -0800 Subject: [PATCH 5/5] [None][fix] Satisfy GLM megakernel native operator contracts Signed-off-by: Dongfeng Yu --- .../csrc/glmMegakernel.cu | 2 +- .../csrc/glmMegakernelOp.cpp | 7 ++++++- .../glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu index 82d3219af58a..f40eac45d416 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernel.cu @@ -6204,7 +6204,7 @@ __device__ __noinline__ void issueBlockedEarly(Ctx const& ctx, int blocker, int } } -#include "glmMegakernelStages.cuh" +#include "glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelStages.cuh" // ---------------------------------------------------------------------------------------------------------------- // Kernel diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp index 36f31377f392..2354d8d58615 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/csrc/glmMegakernelOp.cpp @@ -131,7 +131,7 @@ TRTLLM_MV2_NAMESPACE_END TRTLLM_MV2_LIBRARY_FRAGMENT(m) { - m.def("glm_megakernel_info() -> int[]", &TRTLLM_MV2_NS::torch_ext::glmMegakernelInfo); + m.def("glm_megakernel_info() -> int[]"); m.def("glm_megakernel_indexer_tmap(Tensor pool, int rows) -> Tensor"); m.def("glm_megakernel_weight_table(Tensor(a!) params, int offset) -> ()"); m.def( @@ -139,6 +139,11 @@ TRTLLM_MV2_LIBRARY_FRAGMENT(m) "Tensor kv_lens, Tensor(b!)[] state, Tensor(c!)[] kv_cache, bool pdl) -> (Tensor, Tensor)"); } +TRTLLM_MV2_LIBRARY_IMPL(CompositeExplicitAutograd, m) +{ + m.impl("glm_megakernel_info", &TRTLLM_MV2_NS::torch_ext::glmMegakernelInfo); +} + TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) { m.impl("glm_megakernel_indexer_tmap", &TRTLLM_MV2_NS::torch_ext::glmMegakernelIndexerTmap); diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py index 90171ef562c5..fd5f18340773 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/glm_moe_dsa/glm_5_2_nvfp4__sm_107__low_latency/fake_ops.py @@ -301,6 +301,7 @@ def fake(*args, **kwargs): _host_query(f"{_NS}::glm_dense_mlp_small_m_supported") _host_query(f"{_NS}::glm_intra_node_reduce_supported") _host_query(f"{_NS}::glm_mnnvl_hop_supported") +_host_query(f"{_NS}::glm_megakernel_info") @torch.library.register_fake(f"{_NS}::glm_megakernel_indexer_tmap")