From 07077c02b9c97f58595a58ee0dddfb4382a9a36a Mon Sep 17 00:00:00 2001 From: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> Date: Thu, 12 Nov 2026 02:21:46 +0000 Subject: [PATCH 1/5] [None][refactor] Unify performance collection and exposure Signed-off-by: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> --- docs/source/commands/trtllm-serve/index.rst | 1 + .../performance-metrics-collection.md | 59 +++++ .../_torch/pyexecutor/iteration_stats.py | 38 +++ tensorrt_llm/_torch/pyexecutor/profiling.py | 98 +++---- tensorrt_llm/_torch/pyexecutor/py_executor.py | 84 ++++-- tensorrt_llm/executor/base_worker.py | 72 ++--- tensorrt_llm/executor/result.py | 21 +- tensorrt_llm/grpc/openengine/metrics.py | 9 +- tensorrt_llm/metrics/__init__.py | 10 +- tensorrt_llm/metrics/collector.py | 50 +++- tensorrt_llm/metrics/logging.py | 66 +++++ tensorrt_llm/metrics/perf.py | 247 ++++++++++++++++++ tensorrt_llm/serve/openai_server.py | 10 +- .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../pyexecutor/test_iter_stats_interval.py | 16 +- .../pyexecutor/test_iter_stats_populate.py | 6 + .../pyexecutor/test_profile_endpoints.py | 2 + .../executor/test_stats_serializer.py | 93 +++++++ tests/unittest/metrics/test_collector.py | 78 ++++++ tests/unittest/metrics/test_perf.py | 212 +++++++++++++++ 20 files changed, 1044 insertions(+), 129 deletions(-) create mode 100644 docs/source/commands/trtllm-serve/performance-metrics-collection.md create mode 100644 tensorrt_llm/_torch/pyexecutor/iteration_stats.py create mode 100644 tensorrt_llm/metrics/logging.py create mode 100644 tensorrt_llm/metrics/perf.py create mode 100644 tests/unittest/metrics/test_perf.py diff --git a/docs/source/commands/trtllm-serve/index.rst b/docs/source/commands/trtllm-serve/index.rst index bec1594f33f6..c00b395deb95 100644 --- a/docs/source/commands/trtllm-serve/index.rst +++ b/docs/source/commands/trtllm-serve/index.rst @@ -7,4 +7,5 @@ trtllm-serve trtllm-serve render-endpoints + performance-metrics-collection run-benchmark-with-trtllm-serve diff --git a/docs/source/commands/trtllm-serve/performance-metrics-collection.md b/docs/source/commands/trtllm-serve/performance-metrics-collection.md new file mode 100644 index 000000000000..f4c1dc9091ec --- /dev/null +++ b/docs/source/commands/trtllm-serve/performance-metrics-collection.md @@ -0,0 +1,59 @@ + + +# Shared performance metrics collection + +## Shared collection and exposure + +Performance measurements are collected before selecting their output. The +backend-independent `PerfMetricsCollector` creates owned immutable +`PerfMetricsRecord` values with three categories: + +| Category | Measurements | +| --- | --- | +| `per_req` | Candidate-completion latency/tokens/cache metrics, or explicitly enabled diagnostic request iteration records. Diagnostic request records are not a second completion stream. | +| `per_iter` | Completed-batch iteration data and separately identified loop-boundary observations: host/device timing, scheduled/active/unscheduled/dummy work, fetched requests, and model state. | +| `per_rank` | Rank-owned KV snapshots, memory and capacity limits. Actual snapshot capture iteration/time remain explicit; `batchIter` separately identifies the batch whose processing triggered collection. | + +Request and iteration records also retain their source rank. A delayed batch +record and its attached `executorLoop` can describe different iterations under +overlap or pipeline parallelism. The loop has its own iteration, rank and +`loopSequence`; the sequence advances even while the executor iteration counter +is stalled. Free-form model state and iteration identifiers remain diagnostic +data, not Prometheus labels. + +`LoggerExposure` formats collected loop records in the executor thread using +the existing iteration-line fields. It also writes structured `per_rank` +records when a new rank snapshot is collected. With `enable_iter_req_stats` +and `enable_iter_perf_stats`, request iteration diagnostics are available to +the logger as categorized records. Detailed rank snapshots are exposed when +collected, so the final snapshot need not wait for another executor loop. + +`PrometheusExposure` is the long-lived serving-side adapter. HTTP and OpenEngine +normalize transported records through the same category model before exposing +them. `MetricsCollector`, `log_iteration_stats` and `log_request_metrics_dict` +remain compatibility interfaces. Scraping and idle periods do not trigger +source reads, reset counters or require an active request. + +Neither exposure reads an executor/KV manager or drains native events. Source +collection alone owns those operations. When batch stats are enabled, the loop +logger reuses collected rank snapshots without an additional KV query. In +logger-only mode the collector retains the existing basic-KV refresh cadence +of 16 loop boundaries on selected logging ranks. Detailed KV/tier collection +still requires `enable_iter_perf_stats`; enabling text alone does not enable +all expensive request/tier diagnostics. + +| Use case | Existing controls | +| --- | --- | +| Production Prometheus | `return_perf_metrics: true`, `enable_iter_perf_stats: true`, `print_iter_log: false` | +| Offline iteration text with basic KV utilization | `print_iter_log: true`; no HTTP server or Prometheus client instance is required | +| Offline text plus detailed rank observations | `print_iter_log: true`, `enable_iter_perf_stats: true`; optionally enable request iteration diagnostics | +| Both outputs | Enable iteration text alongside production collection; source reads and instrumentation are shared | + +This separation adds record construction/serialization and optional logging +work. It introduces no CUDA events or synchronization beyond the existing +collection/timing settings. The existing device-step profiler timing policy is +preserved. CPU and +end-to-end overhead of this refactor have not been benchmarked. diff --git a/tensorrt_llm/_torch/pyexecutor/iteration_stats.py b/tensorrt_llm/_torch/pyexecutor/iteration_stats.py new file mode 100644 index 000000000000..03c1d20b7e43 --- /dev/null +++ b/tensorrt_llm/_torch/pyexecutor/iteration_stats.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Named transport records for finalized executor iteration statistics.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Literal, NamedTuple + +if TYPE_CHECKING: + from tensorrt_llm.bindings.executor import IterationStats, RequestStats + + from .kv_cache_stats import KVCacheV2IterationStatsReport + + +class IterationStatsRecord(NamedTuple): + """Tuple-compatible record buffered by the executor and read by the worker. + + Append optional fields at the end to preserve historical tuple layouts. + The record owns references to its observations; it does not copy native stats. + """ + + iteration_stats: IterationStats + request_stats: list[RequestStats] | None = None + kv_iteration_stats: dict[int, object] | KVCacheV2IterationStatsReport | None = None + attention_dp_rank: int | None = None + host_step_time_ms: float | None = None + prev_device_step_time_ms: float | None = None + scheduler_mode: Literal["overlap", "non_overlap"] | None = None + gpu_forward_time_ms: float | None = None + kv_transfer_outcomes: tuple[int, int, int] | None = None + executor_loop: dict[str, object] | None = None + + @classmethod + def from_tuple(cls, stats: tuple) -> IterationStatsRecord: + """Normalize legacy tuples, ignoring trailing fields from newer producers.""" + if isinstance(stats, cls): + return stats + return cls(*stats[: len(cls._fields)]) diff --git a/tensorrt_llm/_torch/pyexecutor/profiling.py b/tensorrt_llm/_torch/pyexecutor/profiling.py index b816276ddf5d..b5297b92ffa3 100644 --- a/tensorrt_llm/_torch/pyexecutor/profiling.py +++ b/tensorrt_llm/_torch/pyexecutor/profiling.py @@ -52,6 +52,8 @@ import torch from tensorrt_llm.logger import logger +from tensorrt_llm.metrics.logging import LoggerExposure +from tensorrt_llm.metrics.perf import PerfMetricsRecord from tensorrt_llm.tools.layer_wise_benchmarks import get_calibrator from tensorrt_llm.tools.profiler.host_profile_tools.host_profiler import get_global_profiler @@ -121,6 +123,14 @@ class PyExecutorProfileManager: def __init__(self, executor: "PyExecutor") -> None: # noqa: F821 self._executor_ref = weakref.ref(executor) + self.metrics = executor._perf_metrics + self.metrics_logger = LoggerExposure(logger.info) + self._metrics_log_rank_enabled = False + + def expose_metrics(self, record: PerfMetricsRecord) -> None: + """Select executor-loop logging after collection and rank filtering.""" + if self._executor.print_log and self._metrics_log_rank_enabled: + self.metrics_logger.emit(record) # ---- helpers --------------------------------------------------- @@ -626,13 +636,6 @@ def profile_step(self): timing_stream = torch.cuda.Stream() if one_timing_event else None if one_timing_event: logger.info("[M3 host] iteration timing: one side-stream event per iteration boundary") - # Cumulative fetch counter as of the previous iter log line; the - # per-iteration ``num_fetched`` on the log line is the delta. - prev_fetch_requests_cur_rank = 0 - # Last kv_cache_util printed and the iteration it was computed at. - kv_util_str = "N/A" - kv_util_iter = None - env_torch_trace_path = os.environ.get(PROFILE_TRACE_ENV_VAR_NAME, None) if env_torch_trace_path is not None: # Append the rank so each rank writes to its own file. @@ -672,6 +675,7 @@ def profile_step(self): else: log_all_ranks = False log_ranks = {int(r) for r in log_ranks_str.split(",")} + self._metrics_log_rank_enabled = log_all_ranks or executor.dist.rank in log_ranks calibrator = get_calibrator() @@ -683,9 +687,8 @@ def profile_step(self): def profile_step_fn(): nonlocal it, enabled, start_time nonlocal start_event_1, end_event_1, start_event_2, end_event_2 - nonlocal prev_device_step_time, prev_fetch_requests_cur_rank + nonlocal prev_device_step_time nonlocal boundary_count - nonlocal kv_util_str, kv_util_iter nonlocal torch_profiler, active_torch_trace_path nonlocal active_enable_torch_trace calibrator.post_step(it) @@ -770,54 +773,35 @@ def profile_step_fn(): executor._latest_host_step_time_ms = host_step_time executor._latest_prev_device_step_time_ms = prev_device_step_time - if executor.print_log and (log_all_ranks or executor.dist.rank in log_ranks): - if prev_device_step_time is None: - prev_device_step_time_str = "N/A" # first iteration - else: - prev_device_step_time_str = f"{prev_device_step_time}ms" - if executor.kv_cache_manager is not None and ( - kv_util_iter is None or it - kv_util_iter >= ITER_LOG_KV_UTIL_REFRESH_ITERS - ): - kv_util_iter = it - kv_util_str = "N/A" - kv_stats = executor.kv_cache_manager.get_kv_cache_stats() - if kv_stats.max_num_blocks > 0: - kv_util_str = ( - f"{1.0 - kv_stats.free_num_blocks / kv_stats.max_num_blocks:.3f}" - ) - import datetime - - formatted_timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - # Per-rank routing/scheduling observability for - # attention-DP load-imbalance diagnosis. ``num_fetched`` - # is the number of new requests the ADP router assigned - # to this rank since the previous log line. - # ``num_dummy`` / ``num_unscheduled`` are sampled at - # schedule time, so here they lag one iteration (like - # ``num_scheduled_requests``). - num_fetched = ( - executor.num_fetch_requests_cur_rank - prev_fetch_requests_cur_rank - ) - prev_fetch_requests_cur_rank = executor.num_fetch_requests_cur_rank - logger.info( - f"iter = {executor.iter_counter}, " - f"global_rank = {executor.global_rank}, " - f"rank = {executor.dist.rank}, " - f"num_scheduled_requests = {executor.num_scheduled_requests}, " - f"adp_dummy_ctx_tokens = {executor._iter_adp_dummy_ctx_tokens}, " - f"adp_dummy_gen_tokens = {executor._iter_adp_dummy_gen_tokens}, " - f"kv_cache_util = {kv_util_str}, " - f"currank_total_requests = {executor.num_fetch_requests_cur_rank}/" - f"{executor.num_fetch_requests}, " - f"host_step_time = {host_step_time}ms, " - f"prev_device_step_time = {prev_device_step_time_str}, " - f"timestamp = {formatted_timestamp}, " - f"states = {executor.model_engine.iter_states}, " - f"num_active = {len(executor.active_requests)}, " - f"num_fetched = {num_fetched}, " - f"num_dummy = {executor.num_dummy_requests}, " - f"num_unscheduled = {executor.num_unscheduled_requests}" - ) + read_kv_cache = None + if ( + executor.print_log + and not executor.enable_iter_perf_stats + and self._metrics_log_rank_enabled + and executor.kv_cache_manager is not None + ): + read_kv_cache = executor.kv_cache_manager.get_kv_cache_stats + record = self.metrics.collect_loop( + { + "iter": executor.iter_counter, + "globalRank": executor.global_rank, + "rank": executor.dist.rank, + "numScheduledRequests": executor.num_scheduled_requests, + "adpDummyCtxTokens": executor._iter_adp_dummy_ctx_tokens, + "adpDummyGenTokens": executor._iter_adp_dummy_gen_tokens, + "numFetchedRequestsTotal": executor.num_fetch_requests_cur_rank, + "numFetchedRequestsAllRanks": executor.num_fetch_requests, + "hostStepTimeMS": host_step_time, + "prevDeviceStepTimeMS": prev_device_step_time, + "states": str(executor.model_engine.iter_states), + "numActiveRequests": len(executor.active_requests), + "numDummyRequests": executor.num_dummy_requests, + "numUnscheduledRequests": executor.num_unscheduled_requests, + }, + read_kv_cache=read_kv_cache, + kv_refresh_interval=ITER_LOG_KV_UTIL_REFRESH_ITERS, + ) + self.expose_metrics(record) # profile_step runs at the start of each executor loop after # logging the preceding loop. Clear the counters before the next diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index cd5703847df2..755ef55d3a2c 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -4,6 +4,7 @@ import atexit import dataclasses import datetime +import json import math import os import sys @@ -56,6 +57,7 @@ _reapply_current_thread_affinity_to_all_threads from tensorrt_llm.logger import logger from tensorrt_llm.mapping import CpType, Mapping +from tensorrt_llm.metrics.perf import PerfMetricsCollector from tensorrt_llm.runtime.kv_cache_manager_v2 import OutOfPagesError from tensorrt_llm.tools.profiler.host_profile_tools.host_profiler import \ host_profiler_context @@ -102,6 +104,7 @@ hard_kill_on_rank_crash, propagate_hard_kill, start_rank_crash_kill_watchdog) from .hang_diagnostics import create_executor_hang_diagnostics +from .iteration_stats import IterationStatsRecord from .kv_cache.kv_cache_manager_v2 import KVCacheManagerV2 from .kv_cache.mamba_cache_manager import (BaseMambaCacheManager, MixedMambaHybridCacheManager) @@ -807,6 +810,7 @@ def __init__( # ``PyExecutorProfileManager``. ``PyExecutor`` keeps the state # fields above on ``self`` for backward-compat introspection; # the manager operates on those fields via a weakref. + self._perf_metrics = PerfMetricsCollector() self._profile_manager = PyExecutorProfileManager(self) self.model_engine = model_engine @@ -3271,6 +3275,32 @@ def _update_iter_stats( else: self._latest_kv_iter_stats = None + rank_metrics = { + "rank": self.dist.rank, + "globalRank": self.global_rank, + "batchIter": stats.iter, + "gpuMemUsage": stats.gpu_mem_usage, + "cpuMemUsage": stats.cpu_mem_usage, + "pinnedMemUsage": stats.pinned_mem_usage, + } + if kv_cache_manager is not None: + rank_metrics["kvCacheStats"] = { + "maxNumBlocks": kv_stats.max_num_blocks, + "freeNumBlocks": kv_stats.free_num_blocks, + "usedNumBlocks": kv_stats.used_num_blocks, + "tokensPerBlock": kv_stats.tokens_per_block, + "allocTotalBlocks": kv_stats.alloc_total_blocks, + "allocNewBlocks": kv_stats.alloc_new_blocks, + "reusedBlocks": kv_stats.reused_blocks, + "missedBlocks": kv_stats.missed_blocks, + "cacheHitRate": kv_stats.cache_hit_rate, + } + append_kv_cache_iteration_stats(rank_metrics, + self._latest_kv_iter_stats) + rank_record = self._perf_metrics.collect_rank( + rank_metrics, capture_iter=self.iter_counter) + self._profile_manager.expose_metrics(rank_record) + paused_requests = (scheduled_batch.paused_requests + scheduled_batch.recompute_paused_requests) @@ -3528,7 +3558,9 @@ def _append_iter_stats(self, attention_dp_rank: Optional[int] = None, host_step_time_ms: Optional[float] = None, prev_device_step_time_ms: Optional[float] = None, - gpu_forward_time_ms: Optional[float] = None): + gpu_forward_time_ms: Optional[float] = None, + kv_transfer_outcomes: Optional[tuple[int, int, + int]] = None): """Append one iteration's finalized stats to the export buffer. The normal Attention-DP path fans out rank-local rows before calling @@ -3554,6 +3586,7 @@ def _append_iter_stats(self, gpu_forward_time_ms: Batch-matched GPU forward time captured by the events surrounding this batch's ``_forward_step``. Surfaces as ``gpuForwardTimeMS`` in the /metrics JSON. + kv_transfer_outcomes: Rank-local cumulative request success/error/timeout counts. """ # Non-ADP appends immediately, so the latest KV stats belong to this # IterationStats. ADP appends later and passes the saved iter-matched @@ -3580,6 +3613,14 @@ def _append_iter_stats(self, or self.dist.pp_size > 1) scheduler_mode = "overlap" if is_overlap_like else "non_overlap" + loop_metrics = None + runtime_metrics = getattr(self, "_perf_metrics", None) + if (attention_dp_rank is None or attention_dp_rank + == self.dist.rank) and runtime_metrics is not None: + loop = runtime_metrics.latest_loop + if loop is not None: + loop_metrics = {"kind": "loop", "per_iter": dict(loop.per_iter)} + tp_size = getattr(self.dist, "tp_size", 1) gather_all_ranks = os.environ.get("TLLM_METRICS_ALL_RANKS", "0") == "1" if (gather_all_ranks and self.enable_iter_perf_stats and tp_size > 1 @@ -3597,32 +3638,34 @@ def _append_iter_stats(self, local_dict["prevDeviceStepTimeMS"] = prev_device_step_time_ms if gpu_forward_time_ms is not None: local_dict["gpuForwardTimeMS"] = gpu_forward_time_ms + if kv_transfer_outcomes is not None: + local_dict["kvTransferOutcomes"] = dict( + zip(("success", "error", "timeout"), kv_transfer_outcomes)) local_dict["schedulerMode"] = scheduler_mode local_dict["rank"] = self.dist.tp_rank + if loop_metrics is not None: + local_dict["executorLoop"] = loop_metrics # Buffer for _flush_iter_stats_synced; in-place tp_allgather would # desync (reached from per-rank-divergent gates). self._pending_iter_stats_dict = local_dict return - # Legacy path: rank-0-only (single-rank or iter stats disabled). - # Tuple layout (length is sniffed by _stats_serializer with - # len() > N guards so older 4-tuples remain readable): - # [0] stats: IterationStats - # [1] req_stats: Optional[List[RequestStats]] - # [2] kv_iter_stats: Optional[Dict[int, KvCacheIterationStats]] - # [3] attention_dp_rank: Optional[int] - # [4] host_step_time_ms: Optional[float] - # [5] prev_device_step_time_ms: Optional[float] - # [6] scheduler_mode: "overlap" | "non_overlap" - # [7] gpu_forward_time_ms: Optional[float] with self.stats_lock: if (not _stats_buffer_is_unbounded(self.max_stats_len) and len(self.stats) > self.max_stats_len): self.stats.pop(0) self.stats.append( - (stats, req_stats, kv_iter_stats, attention_dp_rank, - host_step_time_ms, prev_device_step_time_ms, scheduler_mode, - gpu_forward_time_ms)) + IterationStatsRecord( + iteration_stats=stats, + request_stats=req_stats, + kv_iteration_stats=kv_iter_stats, + attention_dp_rank=attention_dp_rank, + host_step_time_ms=host_step_time_ms, + prev_device_step_time_ms=prev_device_step_time_ms, + scheduler_mode=scheduler_mode, + gpu_forward_time_ms=gpu_forward_time_ms, + kv_transfer_outcomes=kv_transfer_outcomes, + executor_loop=loop_metrics)) def _process_iter_stats( self, @@ -3688,6 +3731,17 @@ def _process_iter_stats( self.enable_iter_req_stats and self.enable_iter_perf_stats) else None + if req_stats and self.print_log: + request_record = self._perf_metrics.collect_iteration({ + "iter": + batch_state.iter_stats.iter, + "rank": + self.dist.rank, + "requestStats": + [json.loads(stat.to_json_str()) for stat in req_stats], + }) + self._profile_manager.expose_metrics(request_record) + stats = self._update_iter_stats(batch_state.iter_stats, iter_latency_ms, num_completed_requests, batch_state.scheduled_requests, diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index 2fcf106677c3..f5b8559f1c22 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -32,6 +32,7 @@ from tensorrt_llm.logger import logger from .._torch.peft.lora.manager import LoraManager +from .._torch.pyexecutor.iteration_stats import IterationStatsRecord from .._torch.pyexecutor.kv_cache_stats import append_kv_cache_iteration_stats from .._torch.pyexecutor.llm_request import LlmResponse from .._torch.pyexecutor.request_inputs import (ExecutorRequest, @@ -48,6 +49,7 @@ from ..llmapi.tokenizer import TokenizerBase from ..llmapi.tracer import global_tracer from ..llmapi.utils import _SyncQueue, configure_cpu_affinity, logger_debug +from ..metrics.perf import PerfMetricsCollector from ..sampling_params import SamplingParams from .executor import GenerationExecutor from .ipc import IpcQueue @@ -1283,69 +1285,69 @@ def stop_profile(self) -> None: return stop_fn() - # Define a Callable to join iteration and request stats @staticmethod - def _stats_serializer(stats) -> str: - # Per-rank path: stats is ("per_rank_dict", {..., "rank": N}). - # Already serialized on the producing rank via allgather — just emit. - if (isinstance(stats, tuple) and len(stats) == 2 - and stats[0] == "per_rank_dict"): - return json.dumps(stats[1]) - - iteration_stats, req_stats = stats[0], stats[1] - kv_iter_stats = stats[2] if len(stats) > 2 else None - attention_dp_rank = stats[3] if len(stats) > 3 else None - # Newer slots — guarded with len() checks so historical 4-tuples and - # any external code still appending the legacy shape keep working. - host_step_time_ms = stats[4] if len(stats) > 4 else None - prev_device_step_time_ms = stats[5] if len(stats) > 5 else None - scheduler_mode = stats[6] if len(stats) > 6 else None - gpu_forward_time_ms = stats[7] if len(stats) > 7 else None - - stats_dict = json.loads(iteration_stats.to_json_str()) + def _stats_serializer(stats: tuple) -> str: + """Serialize named stats records, legacy tuples, or tagged per-rank rows.""" + if isinstance(stats, tuple) and len(stats) == 2: + tag, rank_stats = stats + if tag == "per_rank_dict": + return PerfMetricsCollector.collect_iteration( + rank_stats).iteration_json() + + entry = IterationStatsRecord.from_tuple(stats) + + stats_dict = json.loads(entry.iteration_stats.to_json_str()) # Always tag the row so Dynamo's adapter can read # stat["attentionDpRank"] without a missing-key branch. Non-ADP stats # default to rank 0; ADP stats carry the rank supplied by PyExecutor. - stats_dict["attentionDpRank"] = (0 if attention_dp_rank is None else - attention_dp_rank) + stats_dict["attentionDpRank"] = (0 if entry.attention_dp_rank is None + else entry.attention_dp_rank) - if req_stats is not None and len(req_stats) > 0: + if entry.request_stats is not None and len(entry.request_stats) > 0: stats_dict["requestStats"] = [] - for req_stat in req_stats: + for req_stat in entry.request_stats: stats_dict["requestStats"].append( json.loads(req_stat.to_json_str())) - append_kv_cache_iteration_stats(stats_dict, kv_iter_stats) + append_kv_cache_iteration_stats(stats_dict, entry.kv_iteration_stats) # Per-loop CPU wall captured by profile_step() — always a clean # single-loop measurement, matching the log line's `host_step_time`. # Prefer this over iterLatencyMS when you need absolute per-loop # CPU cost, especially under the overlap scheduler where # iterLatencyMS measures the batch's full lifecycle (~2 loops). - if host_step_time_ms is not None: - stats_dict["hostStepTimeMS"] = host_step_time_ms + if entry.host_step_time_ms is not None: + stats_dict["hostStepTimeMS"] = entry.host_step_time_ms # GPU forward time read via the ping-pong CUDA event pair in # profile_step(). Note the "prev" in the name: under steady state # the value lags its sibling host_step_time on the same record by # one loop (the event-pair being read corresponds to the loop # before the one host_step_time describes). See the ping-pong # comment in PyExecutor._profiler for the design rationale. - if prev_device_step_time_ms is not None: - stats_dict["prevDeviceStepTimeMS"] = prev_device_step_time_ms + if entry.prev_device_step_time_ms is not None: + stats_dict["prevDeviceStepTimeMS"] = entry.prev_device_step_time_ms # Batch-matched GPU forward time measured from the CUDA events around # this record's _forward_step. This is the preferred field for # ForwardPassMetrics wall_time. - if gpu_forward_time_ms is not None: - stats_dict["gpuForwardTimeMS"] = gpu_forward_time_ms + if entry.gpu_forward_time_ms is not None: + stats_dict["gpuForwardTimeMS"] = entry.gpu_forward_time_ms # Scheduler mode for this record. "overlap" means iterLatencyMS # spans ~2 loops (use hostStepTimeMS for clean per-loop cost); # "non_overlap" means iterLatencyMS is itself the clean per-loop # CPU wall. Set per-record so consumers do not need server config. - if scheduler_mode is not None: - stats_dict["schedulerMode"] = scheduler_mode - - # Convert back to JSON string - return json.dumps(stats_dict) + if entry.scheduler_mode is not None: + stats_dict["schedulerMode"] = entry.scheduler_mode + + if entry.kv_transfer_outcomes is not None: + stats_dict["kvTransferOutcomes"] = dict( + zip(("success", "error", "timeout"), + entry.kv_transfer_outcomes)) + + if entry.executor_loop is not None: + stats_dict["executorLoop"] = entry.executor_loop + # Preserve the public JSON shape while collecting categories before exposure. + record = PerfMetricsCollector.collect_iteration(stats_dict) + return record.iteration_json() @staticmethod def _kv_cache_capacity_serializer(capacity) -> str: diff --git a/tensorrt_llm/executor/result.py b/tensorrt_llm/executor/result.py index 138212103f83..75c7bfd7c4e3 100644 --- a/tensorrt_llm/executor/result.py +++ b/tensorrt_llm/executor/result.py @@ -24,7 +24,9 @@ from ..llmapi.tracer import global_tracer from ..llmapi.utils import AsyncQueue, print_traceback_on_error from ..logger import logger -from ..metrics import MetricNames, MetricsCollector, RequestEventTiming +from ..metrics import MetricNames, RequestEventTiming +from ..metrics.perf import (FINISH_REASON_LABEL, PerfMetricsCollector, + PerfMetricsRecord) from ..metrics.perf_utils import \ process_req_perf_metrics as _process_req_perf_metrics from ..sampling_params import LogprobParams, SamplingParams @@ -249,6 +251,7 @@ def __init__(self, self._aborted = False self.metrics_dict = {} self.candidate_metrics: list[dict] = [] + self.candidate_metric_records: list[PerfMetricsRecord] = [] self.trace_headers: Optional[dict[str, str]] = None # Multi-beam search does not report logprobs incrementally: each response # carries the full list, so it is sliced against _last_logprobs_len rather @@ -809,10 +812,7 @@ def record_stats(self, return metrics_stats = {} if output.finish_reason: - metrics_stats.update({ - MetricsCollector.labelname_finish_reason: - output.finish_reason - }) + metrics_stats.update({FINISH_REASON_LABEL: output.finish_reason}) processed_metrics_stat = _process_req_perf_metrics( stats, len(output.token_ids)) if processed_metrics_stat: @@ -833,10 +833,11 @@ def record_stats(self, spec_dec_logged = False if self.per_pos_drafted is not None and any( d > 0 for d in self.per_pos_drafted): - metrics_stats[MetricNames.SPEC_DEC_ACCEPTED_PER_POS] = \ - self.per_pos_accepted - metrics_stats[MetricNames.SPEC_DEC_DRAFTED_PER_POS] = \ - self.per_pos_drafted + metrics_stats[MetricNames.SPEC_DEC_ACCEPTED_PER_POS] = ( + list(self.per_pos_accepted) + if self.per_pos_accepted is not None else None) + metrics_stats[MetricNames.SPEC_DEC_DRAFTED_PER_POS] = list( + self.per_pos_drafted) spec_dec_logged = True if not spec_dec_logged and output.request_perf_metrics is not None: spec_dec = output.request_perf_metrics.speculative_decoding @@ -900,6 +901,8 @@ def record_stats(self, if gen_ppl is not None and math.isfinite(gen_ppl): metrics_stats[MetricNames.GENERATION_PERPLEXITY] = gen_ppl + record = PerfMetricsCollector.collect_request(metrics_stats) + self.candidate_metric_records.append(record) self.candidate_metrics.append(metrics_stats) self.metrics_dict.update(metrics_stats) diff --git a/tensorrt_llm/grpc/openengine/metrics.py b/tensorrt_llm/grpc/openengine/metrics.py index f7b0e629ebf7..3d76ed7d0fda 100644 --- a/tensorrt_llm/grpc/openengine/metrics.py +++ b/tensorrt_llm/grpc/openengine/metrics.py @@ -16,6 +16,7 @@ from tensorrt_llm.logger import logger from tensorrt_llm.metrics.collector import MetricsCollector +from tensorrt_llm.metrics.perf import PerfMetricsCollector from .coordinator import CoordinationError @@ -255,8 +256,12 @@ def record_result(self, result: Any) -> None: if not self._collect_requests: return try: + records = getattr(result, "candidate_metric_records", None) candidates = getattr(result, "candidate_metrics", None) - if candidates: + if records: + for record in records: + self.collector.emit(record) + elif candidates: for metrics in candidates: self.collector.log_request_metrics_dict(metrics) elif metrics := getattr(result, "metrics_dict", None): @@ -315,7 +320,7 @@ def _collect_once(self) -> None: for stats in self._llm.get_stats(timeout=0.1): if self._stopping.is_set(): return - self.collector.log_iteration_stats(stats) + self.collector.emit(PerfMetricsCollector.collect_iteration(stats)) except Exception as error: self._record_error("iteration", error) diff --git a/tensorrt_llm/metrics/__init__.py b/tensorrt_llm/metrics/__init__.py index 5df5dde1f439..8d35f9a847b8 100644 --- a/tensorrt_llm/metrics/__init__.py +++ b/tensorrt_llm/metrics/__init__.py @@ -1,8 +1,14 @@ -from .collector import MetricsCollector +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from .collector import MetricsCollector, PrometheusExposure from .enums import MetricNames, RequestEventTiming +from .logging import LoggerExposure +from .perf import PerfMetricsCollector, PerfMetricsRecord from .perf_utils import process_req_perf_metrics __all__ = [ "MetricsCollector", "MetricNames", "RequestEventTiming", - "process_req_perf_metrics" + "process_req_perf_metrics", "PerfMetricsCollector", "PerfMetricsRecord", + "LoggerExposure", "PrometheusExposure" ] diff --git a/tensorrt_llm/metrics/collector.py b/tensorrt_llm/metrics/collector.py index d96309c4ebcd..d3855697aeb4 100644 --- a/tensorrt_llm/metrics/collector.py +++ b/tensorrt_llm/metrics/collector.py @@ -22,6 +22,8 @@ from tensorrt_llm.logger import logger from .enums import MetricNames +from .perf import (FINISH_REASON_LABEL, PerfMetricsCollector, PerfMetricsRecord, + kv_cache_utilization) if TYPE_CHECKING: from prometheus_client import CollectorRegistry, Counter @@ -38,7 +40,7 @@ # Adapted from https://github.com/vllm-project/vllm/blob/v0.10.0rc1/vllm/engine/metrics.py#L30 -class MetricsCollector: +class PrometheusExposure: """Collects and logs metrics from TensorRT-LLM engine stats to Prometheus. Used by OpenAIServer in tensorrt_llm/serve/openai_server.py. @@ -120,7 +122,7 @@ class MetricsCollector: trtllm_speculative_config_info trtllm_kv_cache_config_info """ - labelname_finish_reason = "finished_reason" + labelname_finish_reason = FINISH_REASON_LABEL labelname_cache_level = "cache_level" labelname_cache_tier = "cache_tier" labelname_dp_rank = "dp_rank" @@ -195,6 +197,7 @@ def __init__( f"{name} must be strictly increasing, got {buckets}.") self.last_log_time = time.time() self.labels = labels + self._last_rank_snapshots: dict[str, object] = {} iteration_labelnames = (*self.labels, self.labelname_dp_rank) self.metric_prefix = "trtllm_" @@ -824,7 +827,42 @@ def _log_cumulative_kv_counter(self, counter: "Counter", value: int, self._log_counter(counter, {self.labelname_dp_rank: rank}, delta) previous[rank] = value + def emit(self, record: PerfMetricsRecord) -> None: + """Expose collected observations through the legacy statistics adapters.""" + if record.kind == "request": + self._expose_request_metrics(record.per_req) + elif record.kind == "loop": + self._expose_rank_snapshot(record) + self._expose_iteration_stats( + {"executorLoop": { + "kind": "loop", + "per_iter": record.per_iter + }}) + elif record.kind == "rank": + self._expose_rank_snapshot(record) + else: + self._expose_iteration_stats(record.iteration_view()) + + def _expose_rank_snapshot(self, record: PerfMetricsRecord) -> None: + """A cached rank snapshot may accompany several loop records; observe it once.""" + if not record.per_rank: + return + identity = str( + record.per_rank.get("globalRank", record.per_rank.get("rank", 0))) + if self._last_rank_snapshots.get(identity) is record.per_rank: + return + self._last_rank_snapshots[identity] = record.per_rank + self._expose_iteration_stats(record.per_rank) + def log_request_metrics_dict(self, metrics_dict: dict) -> None: + """Compatibility adapter for legacy candidate dictionaries.""" + self.emit(PerfMetricsCollector.collect_request(metrics_dict)) + + def log_iteration_stats(self, iteration_stats: dict) -> None: + """Compatibility adapter for legacy iteration dictionaries.""" + self.emit(PerfMetricsCollector.collect_iteration(iteration_stats)) + + def _expose_request_metrics(self, metrics_dict: dict) -> None: """Log per-request metrics from TRTLLM engine responses. This method updates Prometheus metrics including: @@ -940,7 +978,7 @@ def log_request_metrics_dict(self, metrics_dict: dict) -> None: self.last_log_time = time.time() - def log_iteration_stats(self, iteration_stats: dict) -> None: + def _expose_iteration_stats(self, iteration_stats: dict) -> None: """Log iteration-level statistics from TRTLLM engine. Updates Prometheus gauges/counters for queue load, memory usage, batch sizes, @@ -1083,7 +1121,7 @@ def _log_rank_iteration_stats(self, iteration_stats: dict, if "usedNumBlocks" in kv_stats and "maxNumBlocks" in kv_stats: max_num_blocks = kv_stats["maxNumBlocks"] if max_num_blocks: - utilization = kv_stats["usedNumBlocks"] / max_num_blocks + utilization = kv_cache_utilization(kv_stats) self._log_gauge(self.kv_cache_utilization, utilization, labels=rank_labels) @@ -1253,3 +1291,7 @@ def log_request_error(self, http_code: Union[int, str] = "") -> None: """Increment the error counter, labeled by HTTP status code.""" labels = {**self.labels, self.labelname_http_code: str(http_code)} self.counter_request_error.labels(**labels).inc(1) + + +class MetricsCollector(PrometheusExposure): + """Compatibility name for the long-lived Prometheus exposure adapter.""" diff --git a/tensorrt_llm/metrics/logging.py b/tensorrt_llm/metrics/logging.py new file mode 100644 index 000000000000..137983a2e12d --- /dev/null +++ b/tensorrt_llm/metrics/logging.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Executor-loop logger exposure for already collected performance records.""" + +import json +from collections.abc import Callable +from datetime import datetime + +from .enums import MetricNames +from .perf import PerfMetricsRecord, kv_cache_utilization + + +def _json_keys(value: object) -> object: + if isinstance(value, dict): + return { + (key.value if isinstance(key, MetricNames) else key): _json_keys(item) + for key, item in value.items() + } + if isinstance(value, list): + return [_json_keys(item) for item in value] + return value + + +class LoggerExposure: + """Format collected observations without accessing the runtime or CUDA.""" + + def __init__(self, write: Callable[[str], None]) -> None: + self._write = write + self._last_rank_snapshot: object = None + + def emit(self, record: PerfMetricsRecord) -> None: + """Expose a loop in the existing text format, with rank detail on change.""" + if record.kind == "rank": + if record.per_rank is self._last_rank_snapshot: + return + self._last_rank_snapshot = record.per_rank + if record.kind != "loop": + self._write(json.dumps(_json_keys(record.categories()), separators=(",", ":"))) + return + values = record.per_iter + device = values.get("prevDeviceStepTimeMS") + device_text = "N/A" if device is None else f"{device}ms" + utilization = kv_cache_utilization(record.per_rank.get("kvCacheStats", {})) + kv_text = "N/A" if utilization is None else f"{utilization:.3f}" + timestamp = datetime.fromtimestamp(values["timestamp"]).strftime("%Y-%m-%d %H:%M:%S") + self._write( + f"iter = {values['iter']}, " + f"global_rank = {values['globalRank']}, " + f"rank = {values['rank']}, " + f"num_scheduled_requests = {values['numScheduledRequests']}, " + f"adp_dummy_ctx_tokens = {values['adpDummyCtxTokens']}, " + f"adp_dummy_gen_tokens = {values['adpDummyGenTokens']}, " + f"kv_cache_util = {kv_text}, " + f"currank_total_requests = {values['numFetchedRequestsTotal']}/{values['numFetchedRequestsAllRanks']}, " + f"host_step_time = {values['hostStepTimeMS']}ms, " + f"prev_device_step_time = {device_text}, " + f"timestamp = {timestamp}, " + f"states = {values['states']}, " + f"num_active = {values['numActiveRequests']}, " + f"num_fetched = {values['numFetchedRequests']}, " + f"num_dummy = {values['numDummyRequests']}, " + f"num_unscheduled = {values['numUnscheduledRequests']}" + ) + # Detailed snapshots refresh at their collection cadence, not on every log line. + if record.per_rank: + self.emit(PerfMetricsRecord("rank", per_rank=record.per_rank)) diff --git a/tensorrt_llm/metrics/perf.py b/tensorrt_llm/metrics/perf.py new file mode 100644 index 000000000000..6f0d6380e053 --- /dev/null +++ b/tensorrt_llm/metrics/perf.py @@ -0,0 +1,247 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Backend-independent collection records shared by metric exposures. + +Collection owns source snapshots. Exposures receive immutable values and cannot +read runtime managers or consume reset-on-read statistics. +""" + +from __future__ import annotations + +import json +import time +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Literal, Protocol, TypeAlias + +from .enums import MetricNames + +FINISH_REASON_LABEL = "finished_reason" +MetricKey: TypeAlias = str | MetricNames +MetricScalar: TypeAlias = str | int | float | bool | None +MetricValue: TypeAlias = ( + MetricScalar | Mapping[MetricKey, "MetricValue"] | tuple["MetricValue", ...] +) + +# Unrecognized fields remain per-iteration so old/new producers can interoperate. +_RANK_FIELDS = frozenset( + { + "kvCacheStats", + "crossKvCacheStats", + "kvCacheIterationStats", + "kvCacheIterationStatsByLifecycle", + "kvCacheIterationStatsByPoolGroup", + "kvCacheIterationStatsByColdPoolGroup", + "kvCacheStorageStats", + "kvCacheMigrationTimings", + "kvCacheMigrationTimingsSkipped", + "kvCacheLevelTiers", + "kvTransferOutcomes", + "iterSuspendedRequests", + "iterResumedRequests", + "iterDiskPrefetchBlocks", + "iterCachedTokensByLevel", + "iterConnectorCachedTokens", + "gpuMemUsage", + "cpuMemUsage", + "pinnedMemUsage", + "maxNumActiveRequests", + "maxBatchSizeStatic", + "maxBatchSizeRuntime", + "maxNumTokensRuntime", + } +) + + +def _freeze(value: object) -> MetricValue: + if isinstance(value, Mapping): + return MappingProxyType({key: _freeze(item) for key, item in value.items()}) + if isinstance(value, (list, tuple)): + return tuple(_freeze(item) for item in value) + if value is None or isinstance(value, (str, int, float, bool)): + return value + raise TypeError(f"Unsupported metric value type: {type(value).__name__}") + + +def _freeze_fields(values: Mapping) -> Mapping[MetricKey, MetricValue]: + return MappingProxyType({key: _freeze(value) for key, value in values.items()}) + + +def _thaw(value: MetricValue) -> object: + if isinstance(value, Mapping): + return {key: _thaw(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thaw(item) for item in value] + return value + + +def kv_cache_utilization(stats: Mapping) -> float | None: + """Return non-reclaimable utilization; missing/zero capacity is unavailable.""" + capacity = stats.get("maxNumBlocks", 0) + if not isinstance(capacity, (int, float)) or capacity <= 0: + return None + used = stats.get("usedNumBlocks") + if used is None: + free = stats.get("freeNumBlocks") + if free is None: + return None + used = capacity - free + return used / capacity + + +def _empty_fields() -> Mapping[MetricKey, MetricValue]: + return MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class PerfMetricsRecord: + """One owned observation, partitioned by measurement scope. + + The kind distinguishes request completions, loop-boundary observations and + completed batch records. A batch's ``requestStats`` is diagnostic data, not a + second stream of request completions. Existing wire field names retain their + units (for example ``hostStepTimeMS`` is milliseconds). + """ + + kind: Literal["request", "loop", "iteration", "rank"] + per_req: Mapping[MetricKey, MetricValue] = field(default_factory=_empty_fields) + per_iter: Mapping[MetricKey, MetricValue] = field(default_factory=_empty_fields) + per_rank: Mapping[MetricKey, MetricValue] = field(default_factory=_empty_fields) + + def __post_init__(self) -> None: + # Collection factories already own these proxies. Also protect direct + # construction from mutable dict/list inputs without copying cached snapshots. + if not isinstance(self.per_req, MappingProxyType): + object.__setattr__(self, "per_req", _freeze_fields(self.per_req)) + if not isinstance(self.per_iter, MappingProxyType): + object.__setattr__(self, "per_iter", _freeze_fields(self.per_iter)) + if not isinstance(self.per_rank, MappingProxyType): + object.__setattr__(self, "per_rank", _freeze_fields(self.per_rank)) + + def iteration_view(self) -> Mapping[MetricKey, MetricValue]: + """Expose a read-only flat view without copying nested snapshots.""" + return MappingProxyType({**self.per_req, **self.per_iter, **self.per_rank}) + + def iteration_json(self) -> str: + """Encode immutable snapshots directly into the legacy wire shape.""" + + def encode_mapping(value: object) -> dict: + if isinstance(value, Mapping): + return dict(value) + raise TypeError(f"Unsupported metric JSON type: {type(value).__name__}") + + return json.dumps(self.iteration_view(), default=encode_mapping) + + def iteration_stats(self) -> dict: + """Materialize a fresh legacy-compatible transport dictionary.""" + return { + key: _thaw(value) + for group in (self.per_req, self.per_iter, self.per_rank) + for key, value in group.items() + } + + def request_metrics(self) -> dict: + """Materialize candidate completion metrics without changing enum keys.""" + return {key: _thaw(value) for key, value in self.per_req.items()} + + def categories(self) -> dict: + """Materialize categories for structured logging and loop transport.""" + return { + "kind": self.kind, + "per_req": _thaw(self.per_req), + "per_iter": _thaw(self.per_iter), + "per_rank": _thaw(self.per_rank), + } + + +class KvCacheSnapshot(Protocol): + max_num_blocks: int + free_num_blocks: int + + +class PerfMetricsCollector: + """Collect observations before selecting logging or Prometheus exposure. + + An executor owns one instance on its loop thread. Stateless adapters also use + the collection methods to normalize legacy dictionaries. No Prometheus, torch, + CUDA, MPI or logging dependency is needed here. + """ + + def __init__(self) -> None: + self.latest_loop: PerfMetricsRecord | None = None + self.latest_rank: Mapping[MetricKey, MetricValue] = _empty_fields() + self._previous_fetched = 0 + self._loop_sequence = 0 + self._last_rank_poll: int | None = None + + @staticmethod + def collect_request(metrics: Mapping) -> PerfMetricsRecord: + """Take an owned snapshot of one completed output candidate.""" + return PerfMetricsRecord("request", per_req=_freeze_fields(metrics)) + + @staticmethod + def collect_iteration(stats: Mapping) -> PerfMetricsRecord: + """Partition a source batch record without reading runtime state.""" + request = {"requestStats": stats["requestStats"]} if "requestStats" in stats else {} + rank = {key: value for key, value in stats.items() if key in _RANK_FIELDS} + iteration = { + key: value + for key, value in stats.items() + if key not in _RANK_FIELDS and key != "requestStats" + } + return PerfMetricsRecord( + "iteration", _freeze_fields(request), _freeze_fields(iteration), _freeze_fields(rank) + ) + + def collect_rank(self, stats: Mapping, *, capture_iter: int) -> PerfMetricsRecord: + """Remember a source-owned rank snapshot for subsequent loop exposure.""" + self.latest_rank = _freeze_fields( + {**stats, "captureIter": capture_iter, "captureTime": time.time()} + ) + return PerfMetricsRecord("rank", per_rank=self.latest_rank) + + def collect_loop( + self, + values: Mapping, + *, + read_kv_cache: Callable[[], KvCacheSnapshot] | None = None, + kv_refresh_interval: int = 16, + ) -> PerfMetricsRecord: + """Capture one loop; poll basic KV only for a logger-only executor. + + With batch collection enabled the caller supplies no reader: logging + reuses ``collect_rank`` snapshots instead of querying the manager again. + """ + iteration = dict(values) + sequence = self._loop_sequence + self._loop_sequence += 1 + iteration["loopSequence"] = sequence + fetched = iteration["numFetchedRequestsTotal"] + iteration["numFetchedRequests"] = max(0, fetched - self._previous_fetched) + self._previous_fetched = fetched + iteration.setdefault("timestamp", time.time()) + loop_id = iteration["iter"] + if read_kv_cache is not None and ( + self._last_rank_poll is None or sequence - self._last_rank_poll >= kv_refresh_interval + ): + stats = read_kv_cache() + self.collect_rank( + { + "rank": iteration["rank"], + "globalRank": iteration["globalRank"], + "captureLoopSequence": sequence, + "kvCacheStats": { + "maxNumBlocks": stats.max_num_blocks, + "freeNumBlocks": stats.free_num_blocks, + "usedNumBlocks": stats.max_num_blocks - stats.free_num_blocks, + }, + }, + capture_iter=loop_id, + ) + self._last_rank_poll = sequence + record = PerfMetricsRecord( + "loop", per_iter=_freeze_fields(iteration), per_rank=self.latest_rank + ) + self.latest_loop = record + return record diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 233512d9788b..f6b730d5bba4 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -65,6 +65,7 @@ from tensorrt_llm.media.encoding import image_to_bytes from tensorrt_llm.media.tensor_payload import is_tensor_format from tensorrt_llm.metrics.collector import MetricsCollector +from tensorrt_llm.metrics.perf import PerfMetricsCollector from tensorrt_llm.runtime.kv_cache_hash import \ get_effective_kv_cache_event_hash_algo from tensorrt_llm.sampling_params import GuidedDecodingParams, SamplingParams @@ -1938,7 +1939,11 @@ async def _extract_metrics(self, res: RequestOutput, raw_request: Request): if record is not None and raw_request is not None: raw_request.state.perf_metrics_records.append(record) if self.metrics_collector: - if res.candidate_metrics: + records = getattr(res, "candidate_metric_records", None) + if records: + for record in records: + self.metrics_collector.emit(record) + elif res.candidate_metrics: for candidate_m in res.candidate_metrics: self.metrics_collector.log_request_metrics_dict(candidate_m) elif res.metrics_dict: @@ -2059,7 +2064,8 @@ def _apply_spec_decode_stats_opt_in(self, self, "_spec_decode_num_spec_tokens", None) def _record_iteration_stat(self, llm_stat: dict) -> None: - self.metrics_collector.log_iteration_stats(llm_stat) + self.metrics_collector.emit( + PerfMetricsCollector.collect_iteration(llm_stat)) # Tee into the /metrics snapshot buffer so the HTTP handler can serve # without competing for the engine queue (nvbug 6102381). if self._iteration_stats_buffer is not None: diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index ff302874d5d6..95d3041933e3 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -417,6 +417,7 @@ l0_cpu: - unittest/llmapi/test_utils.py - unittest/llmapi/test_visual_gen_startup_shutdown.py - unittest/metrics/test_collector.py + - unittest/metrics/test_perf.py - unittest/models/test_quant_config_utils.py - unittest/others - unittest/quantization/test_mode.py diff --git a/tests/unittest/_torch/pyexecutor/test_iter_stats_interval.py b/tests/unittest/_torch/pyexecutor/test_iter_stats_interval.py index 0b2f472e29d0..3d5cdf74d7f5 100644 --- a/tests/unittest/_torch/pyexecutor/test_iter_stats_interval.py +++ b/tests/unittest/_torch/pyexecutor/test_iter_stats_interval.py @@ -27,6 +27,8 @@ from tensorrt_llm._torch.pyexecutor.py_executor import BatchState, PyExecutor from tensorrt_llm.bindings.executor import InflightBatchingStats, IterationStats +from tensorrt_llm.bindings.internal.batch_manager import KvCacheIterationStats +from tensorrt_llm.metrics.perf import PerfMetricsCollector pytestmark = pytest.mark.cpu_only @@ -234,9 +236,14 @@ def _kv_update_fake(kv_interval: int): missed_blocks=4, cache_hit_rate=0.0, ) - kv_cache_manager.get_iteration_stats.return_value = {32: "deltas"} + delta = KvCacheIterationStats() + delta.iter_reused_blocks = 3 + kv_cache_manager.get_iteration_stats.return_value = {32: delta} fake = MagicMock() + fake._perf_metrics = PerfMetricsCollector() + fake.dist.rank = 0 + fake.global_rank = 0 fake.max_num_active_requests = 8 fake.enable_attention_dp = False fake.executor_request_queue.get_request_queue_size.return_value = 0 @@ -283,7 +290,10 @@ def test_kv_iteration_stats_follow_construction_iter_not_live_counter(): # Live counter one ahead, as under the overlap scheduler. fake.iter_counter = stats_iter + 1 fetched[stats_iter] = _update(fake, stats_iter) - assert fetched == {0: {32: "deltas"}, 4: {32: "deltas"}, 8: {32: "deltas"}} + assert fetched == { + iteration: kv_cache_manager.get_iteration_stats.return_value for iteration in (0, 4, 8) + } + assert fake._perf_metrics.latest_rank["kvCacheIterationStats"]["32"]["iterReusedBlocks"] == 3 assert kv_cache_manager.get_iteration_stats.call_count == 3 @@ -291,7 +301,7 @@ def test_kv_iteration_stats_skip_non_interval_records_and_duplicates(): fake, kv_cache_manager = _kv_update_fake(kv_interval=4) fake.iter_counter = 100 assert _update(fake, 6) is None - assert _update(fake, 8) == {32: "deltas"} + assert _update(fake, 8) is kv_cache_manager.get_iteration_stats.return_value # A second record for the same iteration must not drain the deltas again. assert _update(fake, 8) is None assert kv_cache_manager.get_iteration_stats.call_count == 1 diff --git a/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py b/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py index 8afb1a0fed04..12a57f99f2fd 100644 --- a/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py +++ b/tests/unittest/_torch/pyexecutor/test_iter_stats_populate.py @@ -56,6 +56,7 @@ KvCacheStats, SpecDecodingStats, ) +from tensorrt_llm.metrics.perf import PerfMetricsCollector pytestmark = pytest.mark.cpu_only @@ -196,6 +197,9 @@ def _build_fake_self(queued_items, model_engine_iter_states, *, enable_attention from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor fake = MagicMock() + fake._perf_metrics = PerfMetricsCollector() + fake.dist.rank = 0 + fake.global_rank = 0 fake.max_num_active_requests = 64 fake.iter_counter = 1 fake.executor_request_queue.get_request_queue_size.return_value = len(queued_items) @@ -1183,3 +1187,5 @@ def test_update_iter_stats_does_not_overwrite_construction_iter(): # If someone re-introduces ``stats.iter = self.iter_counter`` this becomes 999. assert stats.iter == 17 + assert fake_self._perf_metrics.latest_rank["captureIter"] == 999 + assert fake_self._perf_metrics.latest_rank["batchIter"] == 17 diff --git a/tests/unittest/_torch/pyexecutor/test_profile_endpoints.py b/tests/unittest/_torch/pyexecutor/test_profile_endpoints.py index 94f6f2014272..cf7aa8b3e16a 100644 --- a/tests/unittest/_torch/pyexecutor/test_profile_endpoints.py +++ b/tests/unittest/_torch/pyexecutor/test_profile_endpoints.py @@ -24,6 +24,7 @@ from tensorrt_llm._torch.pyexecutor.profiling import PyExecutorProfileManager from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor +from tensorrt_llm.metrics.perf import PerfMetricsCollector def _bare_executor(): @@ -45,6 +46,7 @@ def _bare_executor(): # real PyExecutor.__init__ wires this up; replicate that here for # the bare-bones test double so ``executor.start_profile``, # ``executor.stop_profile``, etc. delegate correctly. + executor._perf_metrics = PerfMetricsCollector() executor._profile_manager = PyExecutorProfileManager(executor) return executor diff --git a/tests/unittest/executor/test_stats_serializer.py b/tests/unittest/executor/test_stats_serializer.py index 71a0970f6411..7ae82644fbdd 100644 --- a/tests/unittest/executor/test_stats_serializer.py +++ b/tests/unittest/executor/test_stats_serializer.py @@ -20,6 +20,7 @@ import pytest +from tensorrt_llm._torch.pyexecutor.iteration_stats import IterationStatsRecord from tensorrt_llm._torch.pyexecutor.kv_cache_stats import ( KVCacheV2IterationStatsReport, KVCacheV2LifeCycleIterationStats, @@ -122,6 +123,77 @@ def _make_mock_kv_iter_stats( class TestStatsSerializer: + def test_named_record_preserves_all_wire_fields(self): + iter_stats = _make_mock_iteration_stats() + req_stats = MagicMock() + req_stats.to_json_str.return_value = json.dumps({"id": 42}) + loop = {"kind": "loop", "per_iter": {"iter": 9, "globalRank": 4}} + record = IterationStatsRecord( + iteration_stats=iter_stats, + request_stats=[req_stats], + attention_dp_rank=3, + host_step_time_ms=12.5, + prev_device_step_time_ms=1.25, + scheduler_mode="overlap", + gpu_forward_time_ms=4.25, + kv_transfer_outcomes=(3, 2, 1), + executor_loop=loop, + ) + expected = { + **json.loads(iter_stats.to_json_str()), + "requestStats": [{"id": 42}], + "attentionDpRank": 3, + "hostStepTimeMS": 12.5, + "prevDeviceStepTimeMS": 1.25, + "schedulerMode": "overlap", + "gpuForwardTimeMS": 4.25, + "kvTransferOutcomes": {"success": 3, "error": 2, "timeout": 1}, + "executorLoop": loop, + } + assert json.loads(BaseWorker._stats_serializer(record)) == expected + assert json.loads(BaseWorker._stats_serializer(tuple(record))) == expected + + @pytest.mark.parametrize("field_count", range(2, 12)) + def test_legacy_tuple_prefixes_and_future_trailing_fields(self, field_count): + iter_stats = _make_mock_iteration_stats() + loop = {"kind": "loop", "per_iter": {"iter": 9}} + legacy = ( + iter_stats, + None, + None, + 3, + 12.5, + 1.25, + "overlap", + 4.25, + (3, 2, 1), + loop, + "future field", + )[:field_count] + expected = { + **json.loads(iter_stats.to_json_str()), + "attentionDpRank": 3 if field_count >= 4 else 0, + } + for required_fields, key, value in ( + (5, "hostStepTimeMS", 12.5), + (6, "prevDeviceStepTimeMS", 1.25), + (7, "schedulerMode", "overlap"), + (8, "gpuForwardTimeMS", 4.25), + (9, "kvTransferOutcomes", {"success": 3, "error": 2, "timeout": 1}), + (10, "executorLoop", loop), + ): + if field_count >= required_fields: + expected[key] = value + assert json.loads(BaseWorker._stats_serializer(legacy)) == expected + + def test_tagged_per_rank_dictionary_preserves_wire_fields(self): + stats = { + "iter": 9, + "rank": 3, + "kvCacheStats": {"maxNumBlocks": 10, "usedNumBlocks": 4}, + } + assert json.loads(BaseWorker._stats_serializer(("per_rank_dict", stats))) == stats + def test_serializer_without_kv_iter_stats(self): """Legacy 2-tuple and 3-tuple with None should produce same output.""" iter_stats = _make_mock_iteration_stats() @@ -289,6 +361,27 @@ def test_serializer_8_tuple_emits_new_timing_and_scheduler_mode(self): assert d["schedulerMode"] == "overlap" assert d["gpuForwardTimeMS"] == 4.25 + def test_loop_observation_keeps_its_identity_separate_from_the_batch(self): + iter_stats = _make_mock_iteration_stats() + loop = { + "kind": "loop", + "per_iter": { + "iter": 9, + "loopSequence": 12, + "globalRank": 4, + "numFetchedRequestsTotal": 17, + }, + } + result = BaseWorker._stats_serializer( + (iter_stats, None, None, None, 12.5, None, "overlap", 4.25, None, loop) + ) + data = json.loads(result) + assert data["executorLoop"] == loop + assert data["hostStepTimeMS"] == 12.5 + assert data["executorLoop"]["per_iter"]["iter"] == 9 + # Old tuple producers retain their original shape and do not gain fake loops. + assert "executorLoop" not in json.loads(BaseWorker._stats_serializer((iter_stats, None))) + def test_serializer_with_v2_pool_group_stats(self): """KV cache manager V2 stats should include pool group breakdown.""" iter_stats = _make_mock_iteration_stats() diff --git a/tests/unittest/metrics/test_collector.py b/tests/unittest/metrics/test_collector.py index 4f722d80df4d..b929641ada53 100644 --- a/tests/unittest/metrics/test_collector.py +++ b/tests/unittest/metrics/test_collector.py @@ -873,6 +873,84 @@ def test_empty_stats_returns_empty(self): assert process_req_perf_metrics(None, output_length=10) == {} +class TestCollectedRequestRecords: + @pytest.mark.parametrize("exposure", ["http", "openengine"]) + def test_completed_request_record_is_collected_once_and_exposed_directly( + self, collector, exposure + ): + import asyncio + from types import SimpleNamespace + from unittest.mock import Mock, patch + + from tensorrt_llm.executor.result import GenerationResultBase + from tensorrt_llm.metrics.perf import PerfMetricsCollector, PerfMetricsRecord + + result = SimpleNamespace( + finished=True, + candidate_metric_records=[], + candidate_metrics=[], + metrics_dict={}, + cached_tokens=0, + prompt_token_ids=[1, 2], + per_pos_drafted=[4, 2], + per_pos_accepted=[3, 1], + ) + output = SimpleNamespace( + finish_reason="end_id", + token_ids=[3, 4], + request_perf_metrics=None, + prompt_logprobs=None, + length=2, + cumulative_logprob=None, + logprobs=None, + ) + stats = { + RequestEventTiming.ARRIVAL_TIME: 10.0, + RequestEventTiming.FIRST_TOKEN_TIME: 11.0, + RequestEventTiming.LAST_TOKEN_TIME: 12.0, + } + with ( + patch.object( + PerfMetricsCollector, "collect_request", wraps=PerfMetricsCollector.collect_request + ) as collect, + patch.object( + PerfMetricsRecord, "request_metrics", side_effect=AssertionError("unexpected thaw") + ), + ): + GenerationResultBase.record_stats(result, output, stats) + result.per_pos_accepted[0] = 99 + result.per_pos_drafted[0] = 99 + assert result.candidate_metrics[0][MetricNames.SPEC_DEC_ACCEPTED_PER_POS] == [3, 1] + assert result.candidate_metrics[0][MetricNames.SPEC_DEC_DRAFTED_PER_POS] == [4, 2] + # Compatibility dictionaries stay mutable without changing the collected observation. + result.candidate_metrics[0][MetricNames.E2E] = 999 + result.candidate_metrics[0][MetricNames.SPEC_DEC_ACCEPTED_PER_POS][0] = 999 + if exposure == "http": + from tensorrt_llm.serve.openai_server import OpenAIServer + + server = SimpleNamespace( + metrics_collector=collector, + _collect_perf_metrics=False, + _iteration_stats_collector_task=None, + ) + asyncio.run(OpenAIServer._extract_metrics(server, result, None)) + else: + from tensorrt_llm.grpc.openengine.metrics import OpenEngineMetrics + + metrics = SimpleNamespace( + collector=collector, _collect_requests=True, _record_error=Mock() + ) + OpenEngineMetrics.record_result(metrics, result) + metrics._record_error.assert_not_called() + assert collect.call_count == 1 + assert _get_counter_value(collector, "counter_generation_tokens") == 2 + assert _get_histogram_count(collector, "histogram_e2e_time_request") == 1 + assert _get_histogram_sum(collector, "histogram_e2e_time_request") == 2.0 + assert result.candidate_metric_records[0].per_req[ + MetricNames.SPEC_DEC_ACCEPTED_PER_POS + ] == (3, 1) + + class TestCustomHistogramBuckets: """Tests for configurable Prometheus histogram bucket boundaries.""" diff --git a/tests/unittest/metrics/test_perf.py b/tests/unittest/metrics/test_perf.py new file mode 100644 index 000000000000..45c0d5dce2f7 --- /dev/null +++ b/tests/unittest/metrics/test_perf.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Collection/exposure parity without a GPU, model, source drain or logger scrape.""" + +import json +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from prometheus_client import CollectorRegistry, generate_latest + +from tensorrt_llm.metrics.collector import MetricsCollector, PrometheusExposure +from tensorrt_llm.metrics.enums import MetricNames +from tensorrt_llm.metrics.logging import LoggerExposure +from tensorrt_llm.metrics.perf import PerfMetricsCollector, PerfMetricsRecord, kv_cache_utilization + +pytestmark = pytest.mark.cpu_only + + +def _loop(iteration: int = 0, fetched: int = 3) -> dict: + return { + "iter": iteration, + "rank": 0, + "globalRank": 4, + "numScheduledRequests": 2, + "adpDummyCtxTokens": 8, + "adpDummyGenTokens": 1, + "numFetchedRequestsTotal": fetched, + "numFetchedRequestsAllRanks": 10, + "hostStepTimeMS": 2.5, + "prevDeviceStepTimeMS": None, + "states": "decode", + "numActiveRequests": 3, + "numDummyRequests": 1, + "numUnscheduledRequests": 1, + "timestamp": 1000.0, + } + + +def _prometheus() -> MetricsCollector: + return MetricsCollector({"model_name": "test"}, registry=CollectorRegistry()) + + +def test_direct_record_construction_owns_mutable_category_values() -> None: + values = {"kvCacheStats": {"maxNumBlocks": 10, "usedNumBlocks": 3}} + record = PerfMetricsRecord("rank", per_rank=values) + values["kvCacheStats"]["usedNumBlocks"] = 9 + assert record.per_rank["kvCacheStats"]["usedNumBlocks"] == 3 + + +def test_categories_own_the_source_and_preserve_legacy_wire_shape() -> None: + source = { + "iter": 7, + "attentionDpRank": 2, + "hostStepTimeMS": 2.0, + "futureMeasurement": [1, 2], + "requestStats": [{"id": 9, "numGeneratedTokens": 4}], + "kvCacheStats": {"usedNumBlocks": 2, "maxNumBlocks": 10}, + "kvCacheIterationStatsByPoolGroup": [{"slotSize": [64]}], + } + record = PerfMetricsCollector.collect_iteration(source) + assert record.iteration_stats() == source + assert json.loads(record.iteration_json()) == source + with pytest.raises(TypeError): + record.iteration_view()["kvCacheStats"]["usedNumBlocks"] = 99 + assert "requestStats" in record.per_req + assert "hostStepTimeMS" in record.per_iter + assert "kvCacheStats" in record.per_rank + assert record.per_iter["attentionDpRank"] == 2 + source["kvCacheStats"]["usedNumBlocks"] = 9 + source["kvCacheIterationStatsByPoolGroup"][0]["slotSize"].append(128) + source["requestStats"][0]["numGeneratedTokens"] = 99 + assert record.per_rank["kvCacheStats"]["usedNumBlocks"] == 2 + assert record.per_rank["kvCacheIterationStatsByPoolGroup"][0]["slotSize"] == (64,) + assert record.per_req["requestStats"][0]["numGeneratedTokens"] == 4 + with pytest.raises(TypeError): + record.per_rank["kvCacheStats"]["usedNumBlocks"] = 20 + wire = record.iteration_stats() + wire["kvCacheStats"]["usedNumBlocks"] = 20 + assert record.per_rank["kvCacheStats"]["usedNumBlocks"] == 2 + + +def test_logger_and_prometheus_share_one_collection_without_extra_source_reads() -> None: + collector = PerfMetricsCollector() + reader = Mock(return_value=SimpleNamespace(max_num_blocks=10, free_num_blocks=4)) + record = collector.collect_loop(_loop(), read_kv_cache=reader) + reader.assert_called_once_with() + messages = [] + logger = LoggerExposure(messages.append) + prom = _prometheus() + logger.emit(record) + prom.emit(record) + reader.assert_called_once_with() + assert "kv_cache_util = 0.600" in messages[0] + assert "num_fetched = 3" in messages[0] + assert "prev_device_step_time = N/A" in messages[0] + assert json.loads(messages[1])["per_rank"]["captureIter"] == 0 + assert kv_cache_utilization(record.per_rank["kvCacheStats"]) == pytest.approx(0.6) + assert prom.kv_cache_utilization.labels( + **prom.labels, dp_rank="0" + )._value.get() == pytest.approx(0.6) + logger.emit(record) + prom.emit(record) + # Cached rank snapshots are not re-exposed by the logger. + assert len(messages) == 3 + + +def test_batch_snapshots_replace_logger_polling_and_retain_capture_identity() -> None: + collector = PerfMetricsCollector() + rank = collector.collect_rank( + {"kvCacheStats": {"maxNumBlocks": 8, "freeNumBlocks": 2, "usedNumBlocks": 6}}, + capture_iter=7, + ) + loop = collector.collect_loop(_loop(9)) + assert loop.per_rank is rank.per_rank + assert loop.per_rank["captureIter"] == 7 + assert loop.per_iter["iter"] == 9 + assert loop.per_iter["loopSequence"] == 0 + messages = [] + exposure = LoggerExposure(messages.append) + exposure.emit(rank) + exposure.emit(loop) + assert len(messages) == 2 + assert "kv_cache_util = 0.750" in messages[1] + + +def test_logger_only_kv_refresh_uses_loop_sequence_when_iteration_counter_stalls() -> None: + collector = PerfMetricsCollector() + reader = Mock(return_value=SimpleNamespace(max_num_blocks=10, free_num_blocks=10)) + for _ in range(17): + collector.collect_loop(_loop(0), read_kv_cache=reader, kv_refresh_interval=16) + assert reader.call_count == 2 + assert collector.latest_loop.per_iter["iter"] == 0 + assert collector.latest_loop.per_iter["loopSequence"] == 16 + + +def test_request_candidates_keep_enum_keys_and_diagnostic_requests_are_not_completions() -> None: + source = { + "finished_reason": "stop", + MetricNames.REQUEST_QUEUE_TIME: 0.0, + MetricNames.SPEC_DEC_DRAFTED_PER_POS: [2, 1], + } + request = PerfMetricsCollector.collect_request(source) + assert request.request_metrics() == source + source[MetricNames.SPEC_DEC_DRAFTED_PER_POS].append(99) + assert request.per_req[MetricNames.SPEC_DEC_DRAFTED_PER_POS] == (2, 1) + messages = [] + LoggerExposure(messages.append).emit(request) + assert json.loads(messages[0])["per_req"]["spec_dec_drafted_per_pos"] == [2, 1] + prom = _prometheus() + diagnostic = PerfMetricsCollector.collect_iteration({"requestStats": [{"id": 9}]}) + prom.emit(diagnostic) + assert prom.counter_request_success.collect()[0].samples == [] + prom.emit(request) + assert prom.histogram_queue_time_request.labels(**prom.labels)._sum.get() == 0 + assert ( + prom.counter_request_success.labels(**prom.labels, finished_reason="stop")._value.get() == 1 + ) + + +def test_loop_identity_is_independent_of_delayed_batch_and_fanout_rank() -> None: + runtime = PerfMetricsCollector() + loop = runtime.collect_loop(_loop(9, fetched=8)) + batch = PerfMetricsCollector.collect_iteration( + { + "iter": 7, + "attentionDpRank": 0, + "executorLoop": {"kind": "loop", "per_iter": dict(loop.per_iter)}, + } + ) + prom = _prometheus() + prom.emit(batch) + assert batch.per_iter["iter"] == 7 + assert batch.per_iter["executorLoop"]["per_iter"]["iter"] == 9 + # A peer row without a measured loop must not create a rank-1 zero placeholder. + prom.emit( + PerfMetricsCollector.collect_iteration({"attentionDpRank": 1, "numActiveRequests": 99}) + ) + assert prom.num_requests_running.collect()[0].samples == [] + + +def test_prometheus_endpoint_uses_collected_state_without_a_source_or_logger() -> None: + prom = _prometheus() + assert isinstance(prom, PrometheusExposure) + prom.emit(PerfMetricsCollector.collect_iteration({"numActiveRequests": 3})) + first = generate_latest(prom._registry) + second = generate_latest(prom._registry) + assert b'trtllm_num_requests_running{model_name="test"} 3.0' in first + assert first == second + + +def test_cached_rank_observations_are_not_replayed_by_multiple_loops() -> None: + runtime = PerfMetricsCollector() + rank = runtime.collect_rank( + { + "rank": 0, + "globalRank": 4, + "kvCacheStats": {"maxNumBlocks": 10, "usedNumBlocks": 4}, + }, + capture_iter=0, + ) + prom = _prometheus() + observe = Mock(wraps=prom._expose_iteration_stats) + prom._expose_iteration_stats = observe + prom.emit(rank) + for iteration in range(3): + prom.emit(runtime.collect_loop(_loop(iteration))) + rank_calls = [call for call in observe.call_args_list if "kvCacheStats" in call.args[0]] + assert len(rank_calls) == 1 + assert prom.kv_cache_utilization.labels( + **prom.labels, dp_rank="0" + )._value.get() == pytest.approx(0.4) From 368bb68ab7fa9a3d0f8947397061f3a1c8342877 Mon Sep 17 00:00:00 2001 From: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:12:31 +0000 Subject: [PATCH 2/5] [None][refactor] Centralize iteration stats serialization Signed-off-by: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> --- .../_torch/pyexecutor/iteration_stats.py | 39 ++++++++++++++ tensorrt_llm/_torch/pyexecutor/py_executor.py | 46 +++++----------- tensorrt_llm/executor/base_worker.py | 53 +------------------ tests/unittest/metrics/test_perf.py | 50 +++++++++++++++++ 4 files changed, 104 insertions(+), 84 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/iteration_stats.py b/tensorrt_llm/_torch/pyexecutor/iteration_stats.py index 03c1d20b7e43..afd2af184d88 100644 --- a/tensorrt_llm/_torch/pyexecutor/iteration_stats.py +++ b/tensorrt_llm/_torch/pyexecutor/iteration_stats.py @@ -4,8 +4,11 @@ from __future__ import annotations +import json from typing import TYPE_CHECKING, Literal, NamedTuple +from .kv_cache_stats import append_kv_cache_iteration_stats + if TYPE_CHECKING: from tensorrt_llm.bindings.executor import IterationStats, RequestStats @@ -36,3 +39,39 @@ def from_tuple(cls, stats: tuple) -> IterationStatsRecord: if isinstance(stats, cls): return stats return cls(*stats[: len(cls._fields)]) + + def to_dict(self, *, tp_rank: int | None = None) -> dict: + """Serialize observations using the executor's existing JSON field names. + + Gathered rank rows carry ``rank``; worker rows carry ``attentionDpRank`` + (defaulting to zero for consumers that require it). + """ + stats = json.loads(self.iteration_stats.to_json_str()) + if tp_rank is None: + stats["attentionDpRank"] = ( + 0 if self.attention_dp_rank is None else self.attention_dp_rank + ) + else: + stats["rank"] = tp_rank + if self.request_stats: + stats["requestStats"] = [json.loads(req.to_json_str()) for req in self.request_stats] + append_kv_cache_iteration_stats(stats, self.kv_iteration_stats) + + # Host time covers one loop; the ping-pong device time lags it by one + # loop. GPU forward time belongs to this batch. Scheduler mode tells + # consumers whether iterLatencyMS includes overlapping loops. + optional_fields = ( + ("hostStepTimeMS", self.host_step_time_ms), + ("prevDeviceStepTimeMS", self.prev_device_step_time_ms), + ("gpuForwardTimeMS", self.gpu_forward_time_ms), + ("schedulerMode", self.scheduler_mode), + ("executorLoop", self.executor_loop), + ) + for name, value in optional_fields: + if value is not None: + stats[name] = value + if self.kv_transfer_outcomes is not None: + stats["kvTransferOutcomes"] = dict( + zip(("success", "error", "timeout"), self.kv_transfer_outcomes) + ) + return stats diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 755ef55d3a2c..6fdfaa19ef4e 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -3621,30 +3621,23 @@ def _append_iter_stats(self, if loop is not None: loop_metrics = {"kind": "loop", "per_iter": dict(loop.per_iter)} + entry = IterationStatsRecord( + iteration_stats=stats, + request_stats=req_stats, + kv_iteration_stats=kv_iter_stats, + attention_dp_rank=attention_dp_rank, + host_step_time_ms=host_step_time_ms, + prev_device_step_time_ms=prev_device_step_time_ms, + scheduler_mode=scheduler_mode, + gpu_forward_time_ms=gpu_forward_time_ms, + kv_transfer_outcomes=kv_transfer_outcomes, + executor_loop=loop_metrics) + tp_size = getattr(self.dist, "tp_size", 1) gather_all_ranks = os.environ.get("TLLM_METRICS_ALL_RANKS", "0") == "1" if (gather_all_ranks and self.enable_iter_perf_stats and tp_size > 1 and self.enable_attention_dp and attention_dp_rank is None): - import json as _json - local_dict = _json.loads(stats.to_json_str()) - if req_stats: - local_dict["requestStats"] = [ - _json.loads(r.to_json_str()) for r in req_stats - ] - append_kv_cache_iteration_stats(local_dict, kv_iter_stats) - if host_step_time_ms is not None: - local_dict["hostStepTimeMS"] = host_step_time_ms - if prev_device_step_time_ms is not None: - local_dict["prevDeviceStepTimeMS"] = prev_device_step_time_ms - if gpu_forward_time_ms is not None: - local_dict["gpuForwardTimeMS"] = gpu_forward_time_ms - if kv_transfer_outcomes is not None: - local_dict["kvTransferOutcomes"] = dict( - zip(("success", "error", "timeout"), kv_transfer_outcomes)) - local_dict["schedulerMode"] = scheduler_mode - local_dict["rank"] = self.dist.tp_rank - if loop_metrics is not None: - local_dict["executorLoop"] = loop_metrics + local_dict = entry.to_dict(tp_rank=self.dist.tp_rank) # Buffer for _flush_iter_stats_synced; in-place tp_allgather would # desync (reached from per-rank-divergent gates). self._pending_iter_stats_dict = local_dict @@ -3654,18 +3647,7 @@ def _append_iter_stats(self, if (not _stats_buffer_is_unbounded(self.max_stats_len) and len(self.stats) > self.max_stats_len): self.stats.pop(0) - self.stats.append( - IterationStatsRecord( - iteration_stats=stats, - request_stats=req_stats, - kv_iteration_stats=kv_iter_stats, - attention_dp_rank=attention_dp_rank, - host_step_time_ms=host_step_time_ms, - prev_device_step_time_ms=prev_device_step_time_ms, - scheduler_mode=scheduler_mode, - gpu_forward_time_ms=gpu_forward_time_ms, - kv_transfer_outcomes=kv_transfer_outcomes, - executor_loop=loop_metrics)) + self.stats.append(entry) def _process_iter_stats( self, diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index f5b8559f1c22..d142f93574bf 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -33,7 +33,6 @@ from .._torch.peft.lora.manager import LoraManager from .._torch.pyexecutor.iteration_stats import IterationStatsRecord -from .._torch.pyexecutor.kv_cache_stats import append_kv_cache_iteration_stats from .._torch.pyexecutor.llm_request import LlmResponse from .._torch.pyexecutor.request_inputs import (ExecutorRequest, PythonRequestInputs) @@ -1296,57 +1295,7 @@ def _stats_serializer(stats: tuple) -> str: entry = IterationStatsRecord.from_tuple(stats) - stats_dict = json.loads(entry.iteration_stats.to_json_str()) - # Always tag the row so Dynamo's adapter can read - # stat["attentionDpRank"] without a missing-key branch. Non-ADP stats - # default to rank 0; ADP stats carry the rank supplied by PyExecutor. - stats_dict["attentionDpRank"] = (0 if entry.attention_dp_rank is None - else entry.attention_dp_rank) - - if entry.request_stats is not None and len(entry.request_stats) > 0: - stats_dict["requestStats"] = [] - for req_stat in entry.request_stats: - stats_dict["requestStats"].append( - json.loads(req_stat.to_json_str())) - - append_kv_cache_iteration_stats(stats_dict, entry.kv_iteration_stats) - - # Per-loop CPU wall captured by profile_step() — always a clean - # single-loop measurement, matching the log line's `host_step_time`. - # Prefer this over iterLatencyMS when you need absolute per-loop - # CPU cost, especially under the overlap scheduler where - # iterLatencyMS measures the batch's full lifecycle (~2 loops). - if entry.host_step_time_ms is not None: - stats_dict["hostStepTimeMS"] = entry.host_step_time_ms - # GPU forward time read via the ping-pong CUDA event pair in - # profile_step(). Note the "prev" in the name: under steady state - # the value lags its sibling host_step_time on the same record by - # one loop (the event-pair being read corresponds to the loop - # before the one host_step_time describes). See the ping-pong - # comment in PyExecutor._profiler for the design rationale. - if entry.prev_device_step_time_ms is not None: - stats_dict["prevDeviceStepTimeMS"] = entry.prev_device_step_time_ms - # Batch-matched GPU forward time measured from the CUDA events around - # this record's _forward_step. This is the preferred field for - # ForwardPassMetrics wall_time. - if entry.gpu_forward_time_ms is not None: - stats_dict["gpuForwardTimeMS"] = entry.gpu_forward_time_ms - # Scheduler mode for this record. "overlap" means iterLatencyMS - # spans ~2 loops (use hostStepTimeMS for clean per-loop cost); - # "non_overlap" means iterLatencyMS is itself the clean per-loop - # CPU wall. Set per-record so consumers do not need server config. - if entry.scheduler_mode is not None: - stats_dict["schedulerMode"] = entry.scheduler_mode - - if entry.kv_transfer_outcomes is not None: - stats_dict["kvTransferOutcomes"] = dict( - zip(("success", "error", "timeout"), - entry.kv_transfer_outcomes)) - - if entry.executor_loop is not None: - stats_dict["executorLoop"] = entry.executor_loop - # Preserve the public JSON shape while collecting categories before exposure. - record = PerfMetricsCollector.collect_iteration(stats_dict) + record = PerfMetricsCollector.collect_iteration(entry.to_dict()) return record.iteration_json() @staticmethod diff --git a/tests/unittest/metrics/test_perf.py b/tests/unittest/metrics/test_perf.py index 45c0d5dce2f7..f7987743eb14 100644 --- a/tests/unittest/metrics/test_perf.py +++ b/tests/unittest/metrics/test_perf.py @@ -9,6 +9,7 @@ import pytest from prometheus_client import CollectorRegistry, generate_latest +from tensorrt_llm._torch.pyexecutor.iteration_stats import IterationStatsRecord from tensorrt_llm.metrics.collector import MetricsCollector, PrometheusExposure from tensorrt_llm.metrics.enums import MetricNames from tensorrt_llm.metrics.logging import LoggerExposure @@ -210,3 +211,52 @@ def test_cached_rank_observations_are_not_replayed_by_multiple_loops() -> None: assert prom.kv_cache_utilization.labels( **prom.labels, dp_rank="0" )._value.get() == pytest.approx(0.4) + + +@pytest.mark.parametrize("tp_rank", [None, 3]) +def test_iteration_record_preserves_transport_fields(tp_rank: int | None) -> None: + stats = Mock() + stats.to_json_str.return_value = '{"iter": 7, "futureField": {"value": 1}}' + request = Mock() + request.to_json_str.return_value = '{"id": 9}' + loop = {"kind": "loop", "per_iter": {"iter": 7}} + entry = IterationStatsRecord( + iteration_stats=stats, + request_stats=[request], + attention_dp_rank=2, + host_step_time_ms=0.0, + prev_device_step_time_ms=1.0, + scheduler_mode="overlap", + gpu_forward_time_ms=2.0, + kv_transfer_outcomes=(0, 1, 2), + executor_loop=loop, + ) + expected = { + "iter": 7, + "futureField": {"value": 1}, + "requestStats": [{"id": 9}], + "hostStepTimeMS": 0.0, + "prevDeviceStepTimeMS": 1.0, + "gpuForwardTimeMS": 2.0, + "schedulerMode": "overlap", + "kvTransferOutcomes": {"success": 0, "error": 1, "timeout": 2}, + "executorLoop": loop, + } + if tp_rank is None: + expected["attentionDpRank"] = 2 + else: + expected["rank"] = tp_rank + assert entry.to_dict(tp_rank=tp_rank) == expected + assert json.loads(PerfMetricsCollector.collect_iteration(expected).iteration_json()) == expected + + +@pytest.mark.parametrize("width", range(1, 11)) +def test_iteration_record_accepts_historical_tuple_widths(width: int) -> None: + stats = Mock() + stats.to_json_str.return_value = '{"iter": 7}' + legacy = (stats,) + (None,) * (width - 1) + entry = IterationStatsRecord.from_tuple(legacy) + assert entry.to_dict() == {"iter": 7, "attentionDpRank": 0} + assert entry.to_dict(tp_rank=3) == {"iter": 7, "rank": 3} + assert IterationStatsRecord.from_tuple(entry) is entry + assert IterationStatsRecord.from_tuple(legacy + (None,) * 10).to_dict() == entry.to_dict() From a12d8916d38d5eb650ac21f6fee582b8ab7b7232 Mon Sep 17 00:00:00 2001 From: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:19:56 +0000 Subject: [PATCH 3/5] [None][feat] Include Grafana dashboard setup in serving examples Signed-off-by: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> --- .../performance-metrics-collection.md | 45 + examples/serve/README.md | 13 + examples/serve/grafana/README.md | 178 + examples/serve/grafana/dashboard.py | 952 ++ examples/serve/grafana/example.config.json | 126 + examples/serve/grafana/templates/logs.json | 1770 +++ examples/serve/grafana/templates/metrics.json | 10065 ++++++++++++++++ .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../metrics/test_grafana_dashboard.py | 518 + 9 files changed, 13668 insertions(+) create mode 100644 examples/serve/grafana/README.md create mode 100644 examples/serve/grafana/dashboard.py create mode 100644 examples/serve/grafana/example.config.json create mode 100644 examples/serve/grafana/templates/logs.json create mode 100644 examples/serve/grafana/templates/metrics.json create mode 100644 tests/unittest/metrics/test_grafana_dashboard.py diff --git a/docs/source/commands/trtllm-serve/performance-metrics-collection.md b/docs/source/commands/trtllm-serve/performance-metrics-collection.md index f4c1dc9091ec..3968852d06ba 100644 --- a/docs/source/commands/trtllm-serve/performance-metrics-collection.md +++ b/docs/source/commands/trtllm-serve/performance-metrics-collection.md @@ -57,3 +57,48 @@ work. It introduces no CUDA events or synchronization beyond the existing collection/timing settings. The existing device-step profiler timing policy is preserved. CPU and end-to-end overhead of this refactor have not been benchmarked. + +## Create a Grafana dashboard + +The [Grafana setup example](https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/serve/grafana) +provides a standard-library Python renderer and publisher for Kubernetes serving. +It discovers engine workers and has E2E, Prefill, Decode and All views. Configure +your workload pod/job patterns, datasource UIDs, endpoint/container labels and +Grafana's supported schema-v2 API version in a copy of `example.config.json`. +The default configuration generates metrics only; Loki, scrape-resource creation +and trace-PVC panels are optional. + +```sh +cp examples/serve/grafana/example.config.json /tmp/trtllm-dashboard.json +# Edit the copied configuration for your deployment. +python3 examples/serve/grafana/dashboard.py render \ + --config /tmp/trtllm-dashboard.json --output /tmp/trtllm-dashboard-rendered +python3 examples/serve/grafana/dashboard.py deploy \ + --config /tmp/trtllm-dashboard.json --output /tmp/trtllm-dashboard-rendered --dry-run +``` + +Rendering is offline. The deployment dry-run validates the selected Grafana API +and enabled Kubernetes resources without publishing them. Run `deploy` without +`--dry-run` to publish. Use API persistence for API-owned dashboards and +`configmap-v2` for existing file-sidecar ownership; applying its generated ConfigMap +alone is incomplete. The tool guards resource versions and reads back saved +content. ConfigMap checkpoint plus native finalization is not a cross-system +transaction; the README documents `--finalize-only` recovery. + +The 80 default metric templates include physical tier states/resident occupancy, +restore/eviction turnover, final drops, sampled restore latency/skips and +completed-request transfer duration. Native ratios use logical byte capacity and +remain separated by level/tier and reporting rank. Unsupported observations hide +optional panels. GPU active/available block charts retain their separate meaning: +available blocks can contain reclaimable prefixes. These panels cannot prove +which conversation prefix was evicted. + +Select one application scrape job path to avoid duplicate engine observations. +Existing `irate` charts use the last two samples in `counter_lookback`; new tier +rate/turnover charts average the window, and sampled latency uses a 5-minute +histogram window. Missing metrics, sparse timing samples, rank coverage and native +build requirements still apply; dashboard creation does not enable instrumentation. + +The template includes optional KV-cache tier and transfer panels whose added +instrumentation is provided separately by [PR #22083](https://github.com/NVIDIA/TensorRT-LLM/pull/22083). +This metrics refactor does not introduce those metric families. diff --git a/examples/serve/README.md b/examples/serve/README.md index 35b0e0627f29..88071f43221a 100644 --- a/examples/serve/README.md +++ b/examples/serve/README.md @@ -1,3 +1,16 @@ + + # Online Serving Examples with `trtllm-serve` We provide a CLI command, `trtllm-serve`, to launch a FastAPI server compatible with OpenAI APIs, here are some client examples to query the server, you can check the source code here or refer to the [command documentation](https://nvidia.github.io/TensorRT-LLM/commands/trtllm-serve.html) and [examples](https://nvidia.github.io/TensorRT-LLM/examples/trtllm_serve_examples.html) for detailed information and usage guidelines. + + +## Grafana monitoring + +The [Grafana setup example](grafana/README.md) renders and publishes role-aware +Kubernetes inference dashboards, including KV-cache pressure, transfer and +executor performance views. Start with an offline render and the neutral example +configuration; Grafana/Prometheus deployment and inference settings remain separate. diff --git a/examples/serve/grafana/README.md b/examples/serve/grafana/README.md new file mode 100644 index 000000000000..395d31e5e74e --- /dev/null +++ b/examples/serve/grafana/README.md @@ -0,0 +1,178 @@ + + +# Grafana dashboards for Kubernetes serving + +Render and publish role-aware TensorRT-LLM monitoring dashboards using Python 3.10+ +and its standard library. Copy this whole directory, including `templates/`, to +use it separately from the TensorRT-LLM checkout. Rendering does not import +TensorRT-LLM, resolve credentials, contact Grafana, or invoke Kubernetes. + +The metrics dashboard has E2E, Prefill, Decode and All views. Engine workers are +discovered from current scrape targets on refresh. The 80 default panel templates +include request/queue latency, throughput, rank-local KV reuse/load, transfer +outcomes, executor timing and physical tier pressure. Optional trace-PVC charts +add two templates. Optional Loki support adds a separate 16-panel Logs dashboard. +Auto-grid panels without observations are hidden; missing native tier/timing +measurements do not become zero-valued metrics. + +## Requirements + +- A Grafana server exposing the **dashboard schema v2** API and dynamic layouts. + `grafana.api_version` selects `v2` (the example) or `v2beta1`. This is not the + Classic dashboard JSON API. Confirm the selected version in your server's + Swagger and run the validation command before publication. The `12.3.1` plugin + versions in templates do not establish server API compatibility: Grafana's + [v12.3.1 schema snapshots](https://github.com/grafana/grafana/tree/v12.3.1/pkg/tests/apis/openapi_snapshots) + expose `v2alpha1`, which this example does not target. The current + [Grafana dashboard API documentation](https://grafana.com/docs/grafana/latest/developer-resources/api-reference/http-api/dashboard/) + recommends `v2`. +- A Prometheus datasource and application series with `namespace`, `pod`, `job` + and `endpoint` scrape labels. Set `metric_job_regex` to **one application scrape + path**, especially when both frontend and engine endpoints expose overlapping + counters. Selectors are deployment inputs, not inferred from the current cluster. +- kube-state-metrics for deployment topology; DCGM with `exported_namespace` and + `exported_pod` labels for GPU charts. Dynamo frontend metrics enable the E2E + charts. Missing optional exporters do not establish workload health. +- `kubectl` is needed for ConfigMap publication, Kubernetes Secret authentication, + or optional Service/ServiceMonitor creation. API publication with environment + authentication and scrape setup disabled makes no Kubernetes calls. +- Loki is optional and disabled in the example. Its charts expect the Dynamo/TRTLLM + iteration/trace log fields represented in `templates/logs.json`; this tool does + not install a log collector. Optional distinct-session queries count observed + trace IDs, not online connections, and can encounter Loki cardinality limits. + +## Configure and render + +From the repository root: + +```sh +cp examples/serve/grafana/example.config.json /tmp/trtllm-dashboard.json +# Edit /tmp/trtllm-dashboard.json for your deployment and datasource UIDs. +python3 examples/serve/grafana/dashboard.py render \ + --config /tmp/trtllm-dashboard.json --output /tmp/trtllm-dashboard-rendered +``` + +Rendering produces `metrics.json`, optionally `logs.json`, +`dashboards.configmap.json`, and `scrape-resources.json`. These are local artifacts; +rendering alone does not create a dashboard. Keep deployment configs and rendered +resource identities outside the distributable example. + +| Configuration | Meaning | +| --- | --- | +| `kubernetes.context`, `namespace` | Explicit cluster context and workload namespace; no current-context fallback. | +| `grafana.url`, `api_namespace`, `api_version` | Grafana endpoint, API namespace and supported schema-v2 API version. | +| `grafana.namespace`, `configmap`, `sidecar_labels` | Monitoring namespace and existing sidecar configuration for file publication. | +| `grafana.publication` | `api` or `configmap-v2`, according to who owns the dashboard. | +| `grafana.dashboard_keys` | Existing metrics/logs JSON filenames in the ConfigMap; preserve them when updating a file-owned dashboard. | +| `dashboard.*_uid`, `title`, `refresh` | Stable dashboard identities, display title and refresh period. | +| `dashboard.include_logs` | Enable the optional Logs dashboard; defaults to false in this example. | +| `datasources.prometheus`, `loki` | Grafana datasource UIDs; the Loki value is unused when Logs is disabled. | +| `workload.*_pod_regex`, `metric_job_regex` | Workload selection and the application scrape path. | +| `workload.metric_role_label`, `decode_metric_role_value` | Exporter role label and decode value; ServiceMonitors can inject this label. | +| `workload.engine_endpoint`, `engine_container`, `frontend_container` | Actual endpoint and container labels. | +| `workload.log_*`, `frontend_container` | Loki job, role and container contract, when Logs is enabled. | +| `workload.trace_pvc` | Optional trace volume; null omits both PVC charts. | +| `scrape.enabled` | Generate/apply scoped Services and ServiceMonitors; false by default. | + +Use the supported RE2-compatible subset: ordinary groups and `[.]` for literal +periods; no lookaround, inline flags, backslashes, quotes, colons or commas. Use +JSON booleans, not strings such as `"false"`. + +`dashboard.counter_lookback` defaults to `1m`; choose at least two actual scrape +intervals. Existing `irate` panels use the last two scrapes within that lookup +window. New tier turnover/drop panels use `rate`, averaging resets-aware deltas +across the window. Migration/transfer latency percentiles use a fixed 5-minute +bucket-rate window. Refresh cadence, scrape cadence and statistics collection +cadence are independent. + +## Authentication and publication + +`grafana.auth` accepts references, not inline credentials: + +```json +{"token_env": "GRAFANA_TOKEN"} +``` + +Or use `username_env` / `password_env`, or an existing Kubernetes Secret reference: + +```json +{"secret": "grafana-auth", "username_key": "admin-user", "password_key": "admin-password"} +``` + +Supply values through your environment or secret manager. Use HTTPS or a loopback +administrative tunnel. The client rejects redirects and omits credentials, raw +HTTP error bodies and Kubernetes stderr from output. + +Validate the configured server/API and any enabled Kubernetes resources: + +```sh +python3 examples/serve/grafana/dashboard.py deploy \ + --config /tmp/trtllm-dashboard.json --output /tmp/trtllm-dashboard-rendered --dry-run +``` + +This uses server-side `dryRun=All` / Kubernetes dry-run and writes local artifacts; +it does not publish dashboards or scrape resources. Publication is explicit: + +```sh +python3 examples/serve/grafana/dashboard.py deploy \ + --config /tmp/trtllm-dashboard.json --output /tmp/trtllm-dashboard-rendered +``` + +`api` saves directly in Grafana. Use `configmap-v2` when an existing file +sidecar owns these UIDs; an API-only update can be overwritten by that provisioner. +The latter mode preserves unrelated ConfigMap entries/metadata, applies the +checkpoint, waits for its source checksum, then finalizes and verifies the native +v2 spec. Applying the rendered ConfigMap alone does not complete this workflow. + +The script checks ConfigMap data/resource versions and dashboard resource versions +before writes, including after a sidecar checkpoint. Final native PUTs retain +`resourceVersion` for optimistic concurrency. This is not a transaction spanning +Kubernetes and Grafana: partial checkpoints can exist after an error. Resume only +with the same config/rendered checkpoint: + +```sh +python3 examples/serve/grafana/dashboard.py deploy \ + --config /tmp/trtllm-dashboard.json --output /tmp/trtllm-dashboard-rendered --finalize-only +``` + +A successful publication reads back layouts, queries, variables, links, title and +time settings and writes `receipt.json` with `verified: true`. Each invocation +removes an old receipt first. Always check the current command's exit status. +Publication changes only the configured dashboard/scrape resources; the example +neither installs Grafana/Prometheus nor changes inference configuration. + +## Interpreting KV-cache panels + +Resident occupancy includes retained evictable prefixes. Combine it with turnover, +final capacity/priority drops and recent misses; high occupancy alone does not +prove thrashing. Capacity ratios weight pool groups by logical slot bytes. Legacy +GPU active/available charts are not physical tier occupancy, and available blocks +can contain useful prefixes. No host-only occupancy is inferred from aggregated +secondary-tier statistics. + +Restore/evict turnover is `rate(moved_bytes) / capacity` in turnovers per second; +it can exceed one per second. Capacity/priority drops measure final prefix loss, +not ordinary request release. Native counters count accepted submissions and do +not confirm successful asynchronous DMA. Detailed tier rank coverage can be limited. + +Sampled migration latency is disabled unless +`TRTLLM_KV_CACHE_TIMING_SAMPLE_INTERVAL` is positive before manager construction. +It measures batch stream intervals, including codec/staging, and is distinct from +completed-request KV-transfer duration. Neither is necessarily a request's restore +stall. Compare source/destination paths and skipped timing samples; sparse or +absent samples do not mean zero latency. Ratios and latencies are not stacked. + +## Standalone tests + +From the repository root, the utility tests need Python, `pytest` for the CI marker, +and no TensorRT-LLM runtime import. They mock publication; they do not deploy: + +```sh +python3 -m unittest discover -s tests/unittest/metrics -p test_grafana_dashboard.py -v +``` + +A successful offline render or mocked publisher test is not a live Grafana, +Prometheus-query or native KV-instrumentation integration test. diff --git a/examples/serve/grafana/dashboard.py b/examples/serve/grafana/dashboard.py new file mode 100644 index 000000000000..4a39868615fb --- /dev/null +++ b/examples/serve/grafana/dashboard.py @@ -0,0 +1,952 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Render and publish role-aware inference dashboards for an explicit K8s target. + +Python standard library only. Credentials are resolved at runtime and never saved. +""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from copy import deepcopy +from pathlib import Path +from typing import Callable, TypeAlias + +JsonValue: TypeAlias = str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"] + +HERE = Path(__file__).resolve().parent + + +class SetupError(Exception): + """Invalid configuration, failed transport, or concurrent modification.""" + + +def require(condition: object, message: str) -> None: + """Raise a setup diagnostic when a validated condition is false.""" + if not condition: + raise SetupError(message) + + +def load_config(path: Path) -> dict: + """Load and validate an explicit, credential-reference-only configuration.""" + c = json.loads(Path(path).read_text(encoding="utf-8-sig")) + require(isinstance(c, dict), "Configuration must be a JSON object") + require(c.get("schema_version") == 1, "Expected schema_version 1") + for section in ("kubernetes", "grafana", "dashboard", "datasources", "workload", "scrape"): + require(isinstance(c.get(section), dict), "Missing config section: " + section) + k, g, d, w = (c[x] for x in ("kubernetes", "grafana", "dashboard", "workload")) + require( + isinstance(k.get("context"), str) + and bool(k["context"]) + and not k["context"].startswith("-"), + "An explicit Kubernetes context is required", + ) + for value in (k.get("namespace"), g.get("namespace"), g.get("configmap")): + require( + isinstance(value, str) and re.fullmatch(r"[a-z0-9][a-z0-9.-]*", value), + "Invalid Kubernetes resource name", + ) + for uid in (d.get("metrics_uid"), d.get("logs_uid")): + require( + isinstance(uid, str) and re.fullmatch(r"[A-Za-z0-9_-]{1,40}", uid), + "Invalid dashboard UID", + ) + require(d["metrics_uid"] != d["logs_uid"], "Dashboard UIDs must differ") + require( + g.get("publication") in ("api", "configmap-v2"), "publication must be api or configmap-v2" + ) + keys = g.get("dashboard_keys", {"metrics": "metrics.json", "logs": "logs.json"}) + require(isinstance(keys, dict), "dashboard_keys must be a JSON object") + require( + set(keys) == {"metrics", "logs"} + and len(set(keys.values())) == 2 + and all( + isinstance(x, str) and re.fullmatch(r"[A-Za-z0-9_.-]+", x) and x.endswith(".json") + for x in keys.values() + ), + "dashboard_keys must contain distinct metrics/logs JSON filenames", + ) + parsed = urllib.parse.urlsplit(g.get("url", "")) + require( + parsed.scheme in ("https", "http") + and bool(parsed.hostname) + and not parsed.username + and not parsed.password + and not parsed.query + and not parsed.fragment, + "Grafana URL must have no embedded credentials, query, or fragment", + ) + require( + parsed.scheme == "https" or parsed.hostname in ("localhost", "127.0.0.1", "::1"), + "Use HTTPS or a loopback administrative tunnel", + ) + require( + g.get("api_version", "v2beta1") in ("v2beta1", "v2"), "api_version must be v2beta1 or v2" + ) + for section, key in ((d, "include_logs"), (c["scrape"], "enabled")): + require(isinstance(section.get(key, False), bool), key + " must be a JSON boolean") + for key in ("prometheus", "loki"): + require( + isinstance(c["datasources"].get(key), str) + and re.fullmatch(r"[A-Za-z0-9_-]{1,40}", c["datasources"][key]), + "Invalid datasource UID", + ) + auth = g.get("auth", {}) + require(isinstance(auth, dict), "grafana.auth must be a JSON object") + require( + set(auth) + in ( + {"token_env"}, + {"username_env", "password_env"}, + {"secret", "username_key", "password_key"}, + ), + "Use token_env, username_env/password_env, or a Kubernetes Secret reference for Grafana auth", + ) + require( + all(isinstance(v, str) and v for v in auth.values()), + "Authentication references must be nonempty strings", + ) + if "secret" in auth: + require( + bool(re.fullmatch(r"[a-z0-9][a-z0-9.-]*", auth["secret"])), + "Invalid Kubernetes Secret name", + ) + for key in ( + "prefill_pod_regex", + "decode_pod_regex", + "frontend_pod_regex", + "all_pod_regex", + "metric_job_regex", + ): + value = w.get(key) + require( + isinstance(value, str) + and value + and not any(x in value for x in ('"', "\\", "\n", "\r", ",", ":", "@@")), + key + + ( + " must be a nonempty RE2-compatible regex without quotes, commas, colons or " + "backslashes (use [.] for a literal dot)" + ), + ) + require( + "(?" not in value, key + " supports ordinary groups, not lookaround or inline flags" + ) + try: + re.compile(value) + except re.error: + raise SetupError("Invalid regex: " + key) from None + for key in ("metric_role_label", "log_role_label"): + require( + bool(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", w.get(key, ""))), + "Invalid metric/log label name", + ) + require( + isinstance(w.get("log_containers"), list) and bool(w["log_containers"]), + "log_containers must be a nonempty list", + ) + for value in w["log_containers"]: + require( + isinstance(value, str) and bool(re.fullmatch(r"[a-z0-9][a-z0-9-]*", value)), + "Invalid container name", + ) + for duration in ( + d.get("counter_lookback"), + d.get("refresh"), + c["scrape"].get("interval"), + c["scrape"].get("timeout"), + ): + require( + isinstance(duration, str) and bool(re.fullmatch(r"[1-9][0-9]*[smhd]", duration)), + "Durations must be positive integers followed by s/m/h/d", + ) + return c + + +def walk(value: JsonValue, fn: Callable[[str], str]) -> JsonValue: + """Replace strings recursively in a JSON value.""" + if isinstance(value, str): + return fn(value) + if isinstance(value, list): + return [walk(x, fn) for x in value] + if isinstance(value, dict): + return {fn(k): walk(v, fn) for k, v in value.items()} + return value + + +def add_unique_sessions(spec: dict, c: dict) -> None: + """Optional exact rolling distinct session count, sourced from frontend traces.""" + settings = c["dashboard"].get("unique_sessions", {}) + if not settings.get("enabled", False): + return + windows = settings.get("windows", ["1m", "5m", "15m", "30m", "60m"]) + default = settings.get("default_window", "30m") + require( + isinstance(windows, list) + and windows + and all( + isinstance(x, str) and re.fullmatch(r"[1-9][0-9]*m", x) and int(x[:-1]) <= 60 + for x in windows + ), + "Session windows must be minute durations up to 60m", + ) + require( + default in windows and len(set(windows)) == len(windows), + "Session default must be one of the distinct window choices", + ) + spec["variables"] = [v for v in spec["variables"] if v["spec"]["name"] != "session_window"] + spec["variables"].append( + { + "kind": "CustomVariable", + "spec": { + "name": "session_window", + "label": "Session window", + "query": ",".join(windows), + "current": {"text": default, "value": default}, + "options": [{"text": x, "value": x, "selected": x == default} for x in windows], + "multi": False, + "includeAll": False, + "allowCustomValue": False, + "hide": "dontHide", + "skipUrlSync": False, + "description": "Rolling session observation window ending at the dashboard time-range end.", + }, + } + ) + w = c["workload"] + selector = ( + '{namespace="$namespace",job=' + + json.dumps(w["log_job"]) + + ",pod=~" + + json.dumps(w["frontend_pod_regex"]) + + ",container=" + + json.dumps(w["frontend_container"]) + + "}" + ) + # Disjoint partitions avoid Loki's per-query series ceiling. Deduplication + # still uses the complete session ID, so hash collisions do not merge IDs. + partition = ( + "{{ $s := .session_id }}{{ $h := 0 }}{{ range $i := len $s }}{{ $h = mod (add " + "(mul $h 31) (index $s $i)) 8 }}{{ end }}{{ $h }}" + ) + queries = [] + for i in range(8): + ref = chr(65 + i) + pipeline = ( + selector + + ( + ' |= "dynamo.request.trace.v1" | json session_id="agent_context.session_id", ' + 'trace_schema="schema" | __error__="" | trace_schema="dynamo.request.trace.v1" ' + '| session_id!="" | label_format session_partition=`' + ) + + partition + + '` | session_partition="' + + str(i) + + '" | keep session_id' + ) + expr = ( + "count(sum by (session_id) (count_over_time(" + + pipeline + + " [${session_window:raw}]))) or vector(0)" + ) + query = { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": {"name": c["datasources"]["loki"]}, + "spec": { + "expr": expr, + "queryType": "instant", + "instant": True, + "range": False, + "refId": ref, + "legendFormat": "partition " + str(i), + }, + } + queries.append( + {"kind": "PanelQuery", "spec": {"query": query, "refId": ref, "hidden": False}} + ) + queries.append( + { + "kind": "PanelQuery", + "spec": { + "refId": "TOTAL", + "hidden": False, + "query": { + "kind": "DataQuery", + "group": "__expr__", + "version": "v0", + "datasource": {"name": "__expr__"}, + "spec": { + "type": "math", + "expression": " + ".join("$" + chr(65 + i) for i in range(8)), + "refId": "TOTAL", + }, + }, + }, + } + ) + key = "session-unique-observed" + panel_id = ( + spec["elements"] + .get(key, {}) + .get("spec", {}) + .get("id", max(e["spec"]["id"] for e in spec["elements"].values()) + 1) + ) + spec["elements"][key] = { + "kind": "Panel", + "spec": { + "id": panel_id, + "title": "Observed unique sessions - last ${session_window}", + "description": ( + "Distinct nonempty agent_context.session_id values in frontend " + "dynamo.request.trace.v1 logs during the selected Session window, ending at the" + " dashboard time-range end. Repeated requests/events for one session count " + "once. Observed sessions, not online connections, people or agent instances. A " + "session appears only after a trace event is logged; request-end traces do not " + "expose an unfinished first request. Independent of log search, container, " + "worker and rank filters. Missing IDs are excluded. Empty successful queries " + "return zero; query errors remain errors. Exact partitioned deduplication; " + "excessive cardinality can still exceed Loki limits." + ), + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": queries, + "transformations": [ + { + "kind": "filterByRefId", + "spec": {"id": "filterByRefId", "options": {"include": "TOTAL"}}, + } + ], + "queryOptions": {}, + }, + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.3.1", + "spec": { + "options": { + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "graphMode": "none", + "textMode": "value", + "colorMode": "value", + "justifyMode": "center", + }, + "fieldConfig": { + "defaults": {"unit": "short", "decimals": 0, "min": 0, "noValue": "—"}, + "overrides": [], + }, + }, + }, + }, + } + roles = next(v["spec"] for v in spec["variables"] if v["spec"]["name"] == "role") + all_role = next(o["value"] for o in roles["options"] if o["text"] == "All") + rows = spec["layout"]["spec"]["rows"] + rows[:] = [r for r in rows if r["spec"].get("title") != "E2E - Session activity"] + rows.insert( + 0, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "E2E - Session activity", + "collapse": False, + "hideHeader": False, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": {"variable": "role", "operator": "equals", "value": v}, + } + for v in ("e2e", all_role) + ], + }, + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 4, + "element": {"kind": "ElementReference", "name": key}, + }, + } + ] + }, + }, + }, + }, + ) + + +def render(c: dict) -> tuple[dict, dict, list[dict]]: + """Render portable dashboard specs and optional scrape resources offline.""" + w, d, g = c["workload"], c["dashboard"], c["grafana"] + engine = "(" + w["prefill_pod_regex"] + "|" + w["decode_pod_regex"] + ")" + values = { + "NAMESPACE": c["kubernetes"]["namespace"], + "TITLE": d["title"], + "METRICS_UID": d["metrics_uid"], + "LOGS_UID": d["logs_uid"], + "PREFILL_PODS": w["prefill_pod_regex"], + "DECODE_PODS": w["decode_pod_regex"], + "ENGINE_PODS": engine, + "FRONTEND_PODS": w["frontend_pod_regex"], + "ALL_PODS": w["all_pod_regex"], + "METRIC_JOBS": w["metric_job_regex"], + "METRIC_ROLE_LABEL": w["metric_role_label"], + "DECODE_ROLE_VALUE": w["decode_metric_role_value"], + "ENGINE_ENDPOINT": w["engine_endpoint"], + "ENGINE_CONTAINER": w["engine_container"], + "FRONTEND_CONTAINER": w["frontend_container"], + "LOG_JOB": w["log_job"], + "LOG_ROLE_LABEL": w["log_role_label"], + "PREFILL_LOG_ROLE": w["prefill_log_role_value"], + "DECODE_LOG_ROLE": w["decode_log_role_value"], + "CONTAINERS": ",".join(w["log_containers"]), + "TRACE_PVC": w.get("trace_pvc") or "", + "PROMETHEUS_UID": c["datasources"]["prometheus"], + "LOKI_UID": c["datasources"]["loki"], + } + for key in ( + "DECODE_ROLE_VALUE", + "ENGINE_ENDPOINT", + "ENGINE_CONTAINER", + "FRONTEND_CONTAINER", + "LOG_JOB", + "PREFILL_LOG_ROLE", + "DECODE_LOG_ROLE", + "TRACE_PVC", + ): + values[key] = json.dumps(values[key], ensure_ascii=False)[1:-1] + + def replace(text: str) -> str: + return re.sub(r"@@([A-Z_]+)@@", lambda m: values[m[1]], text) + + result = {} + for name in ("metrics", "logs"): + if name == "logs" and not d.get("include_logs", True): + continue + spec = walk( + json.loads((HERE / "templates" / f"{name}.json").read_text(encoding="utf-8")), replace + ) + spec["timeSettings"]["autoRefresh"] = d["refresh"] + for panel in spec["elements"].values(): + for q in panel["spec"]["data"]["spec"]["queries"]: + s = q["spec"]["query"]["spec"] + if q["spec"]["query"]["group"] == "prometheus" and "expr" in s: + s["expr"] = s["expr"].replace("[1m]", "[" + d["counter_lookback"] + "]") + if "kubelet_volume_stats_" in s["expr"]: + s["expr"] = s["expr"].replace( + "persistentvolumeclaim=", + 'namespace="$namespace",persistentvolumeclaim=', + ) + if not w.get("trace_pvc"): + removed = { + k for k, v in spec["elements"].items() if "kubelet_volume_stats_" in json.dumps(v) + } + spec["elements"] = {k: v for k, v in spec["elements"].items() if k not in removed} + for row in spec["layout"]["spec"]["rows"]: + items = row["spec"]["layout"]["spec"]["items"] + row["spec"]["layout"]["spec"]["items"] = [ + x for x in items if x["spec"]["element"]["name"] not in removed + ] + # Repack shared fixed-grid rows after dropping the optional PVC charts. + for row in spec["layout"]["spec"]["rows"]: + layout = row["spec"]["layout"] + if layout["kind"] != "GridLayout": + continue + x = y = height = 0 + for item in layout["spec"]["items"]: + s = item["spec"] + if x + s["width"] > 24: + y += height + x = height = 0 + s.update(x=x, y=y) + x += s["width"] + height = max(height, s["height"]) + if not d.get("include_logs", True): + spec["links"] = [x for x in spec["links"] if x.get("url") != "/d/" + d["logs_uid"]] + if name == "logs": + add_unique_sessions(spec, c) + require("@@" not in json.dumps(spec), "Unresolved dashboard placeholder") + refs = [ + i["spec"]["element"]["name"] + for r in spec["layout"]["spec"]["rows"] + for i in r["spec"]["layout"]["spec"]["items"] + ] + require( + len(refs) == len(set(refs)) == len(spec["elements"]), + "Dashboard layout references are inconsistent", + ) + result[name] = spec + cm = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": g["configmap"], + "namespace": g["namespace"], + "labels": g.get("sidecar_labels", {}), + }, + # The legacy file provisioner needs uid even though native v2 spec omits it. + "data": { + g.get("dashboard_keys", {}).get(name, name + ".json"): json.dumps( + dict(spec, uid=d[name + "_uid"]), separators=(",", ":") + ) + for name, spec in result.items() + }, + } + return result, cm, monitors(c) + + +def monitors(c: dict) -> list[dict]: + """Build Services and ServiceMonitors scoped to the configured workload.""" + scrape = c["scrape"] + if not scrape.get("enabled", False): + return [] + result = [] + for role in scrape["roles"]: + require(role.get("selector"), "Each scrape role needs an explicit Service selector") + name = c["grafana"]["configmap"] + "-" + role["name"] + "-metrics" + require( + bool(re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", name)), + "Generated Service name is invalid or too long", + ) + label = {"app.kubernetes.io/name": name} + endpoints = role["endpoints"] + require(bool(endpoints), "Each scrape role needs an endpoint") + for e in endpoints: + require(isinstance(e["port"], int) and 0 < e["port"] < 65536, "Invalid metrics port") + result.append( + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": name, + "namespace": c["kubernetes"]["namespace"], + "labels": label, + }, + "spec": { + "selector": role["selector"], + "ports": [ + { + "name": e["name"], + "port": e["port"], + "targetPort": e.get("target_port", e["port"]), + } + for e in endpoints + ], + }, + } + ) + result.append( + { + "apiVersion": "monitoring.coreos.com/v1", + "kind": "ServiceMonitor", + "metadata": { + "name": name, + "namespace": scrape["monitor_namespace"], + "labels": scrape.get("service_monitor_labels", {}), + }, + "spec": { + "selector": {"matchLabels": label}, + "namespaceSelector": {"matchNames": [c["kubernetes"]["namespace"]]}, + "endpoints": [ + { + "port": e["name"], + "path": e["path"], + "interval": scrape["interval"], + "scrapeTimeout": scrape["timeout"], + "relabelings": [ + { + "targetLabel": c["workload"]["metric_role_label"], + "replacement": role["metric_role_value"], + } + ], + } + for e in endpoints + ], + }, + } + ) + return result + + +def write_outputs(output: Path, dashboards: dict, cm: dict, resources: list[dict]) -> None: + """Write rendered JSON artifacts without authentication material.""" + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + values = {name + ".json": spec for name, spec in dashboards.items()} + values.update( + { + "dashboards.configmap.json": cm, + "scrape-resources.json": {"apiVersion": "v1", "kind": "List", "items": resources}, + } + ) + for name, value in values.items(): + (output / name).write_text( + json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + + +class Client: + """Explicitly scoped Grafana and Kubernetes client with runtime authentication.""" + + def __init__(self, config: dict) -> None: + """Resolve configured credential references in memory.""" + self.c = config + auth = config["grafana"]["auth"] + if "token_env" in auth: + token = os.environ.get(auth["token_env"], "") + require(bool(token), "Grafana token environment variable is unset") + self.authorization = "Bearer " + token + else: + if "secret" in auth: + secret = self.kube(config["grafana"]["namespace"], "get", "secret", auth["secret"])[ + "data" + ] + try: + user, password = ( + base64.b64decode(secret[auth[k]], validate=True).decode() + for k in ("username_key", "password_key") + ) + except (KeyError, ValueError, UnicodeError): + raise SetupError( + "Grafana Secret is missing valid authentication keys" + ) from None + else: + user, password = ( + os.environ.get(auth[k], "") for k in ("username_env", "password_env") + ) + require(bool(user) and bool(password), "Grafana authentication is unset") + self.authorization = ( + "Basic " + base64.b64encode((user + ":" + password).encode()).decode() + ) + + def kube( + self, namespace: str, *args: str, payload: dict | None = None, missing_ok: bool = False + ) -> dict | None: + command = [ + self.c["kubernetes"].get("kubectl", "kubectl"), + "--context", + self.c["kubernetes"]["context"], + "-n", + namespace, + *args, + ] + if payload is None: + command += ["-o", "json"] + p = subprocess.run( + command, + input=json.dumps(payload) if payload is not None else None, + capture_output=True, + text=True, + timeout=120, + ) + if missing_ok and p.returncode and "NotFound" in p.stderr: + return None + require( + p.returncode == 0, + "Kubernetes operation failed (exit " + + str(p.returncode) + + "); check context, access and resource names", + ) + return json.loads(p.stdout) if p.stdout.strip() else None + + def api( + self, + path: str, + payload: dict | None = None, + method: str | None = None, + missing_ok: bool = False, + ) -> dict | None: + req = urllib.request.Request( + self.c["grafana"]["url"].rstrip("/") + path, + method=method, + headers={"Authorization": self.authorization, "Content-Type": "application/json"}, + data=json.dumps(payload).encode() if payload is not None else None, + ) + + # Credentials may not follow a redirect to another URL. + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request( + self, + req: urllib.request.Request, + fp: object, + code: int, + msg: str, + headers: object, + newurl: str, + ) -> None: + return None + + try: + with urllib.request.build_opener(NoRedirect()).open(req, timeout=60) as response: + return json.load(response) + except urllib.error.HTTPError as error: + if missing_ok and error.code == 404: + return None + raise SetupError( + "Grafana request failed (HTTP " + str(error.code) + "); response body omitted" + ) from None + except urllib.error.URLError: + raise SetupError("Grafana connection failed; check the configured URL/tunnel") from None + + +def endpoint(c: dict, uid: str = "") -> str: + """Return the selected native dashboard API endpoint.""" + ns = urllib.parse.quote(c["grafana"].get("api_namespace", "default"), safe="") + return ( + "/apis/dashboard.grafana.app/" + + c["grafana"].get("api_version", "v2beta1") + + "/namespaces/" + + ns + + "/dashboards" + + ("/" + uid if uid else "") + ) + + +def resource(c: dict, uid: str, desired: dict, current: dict | None) -> dict: + """Build a native dashboard resource retaining optimistic-concurrency metadata.""" + metadata = ( + deepcopy(current["metadata"]) + if current + else {"name": uid, "namespace": c["grafana"].get("api_namespace", "default")} + ) + return { + "apiVersion": "dashboard.grafana.app/" + c["grafana"].get("api_version", "v2beta1"), + "kind": "Dashboard", + "metadata": metadata, + "spec": desired, + } + + +def normalized(value: dict) -> dict: + """Normalize Grafana defaults before comparing persisted dashboard content.""" + result = deepcopy(value) + for e in result["elements"].values(): + defaults = e["spec"]["vizConfig"]["spec"].get("fieldConfig", {}).get("defaults", {}) + for step in defaults.get("thresholds", {}).get("steps", []): + if step.get("value") is None: + step["value"] = 0 + for link in result.get("links", []): + link.setdefault("tooltip", "") + return result + + +def _revision(value: dict | None) -> str | None: + """Return the opaque resource version used for optimistic-concurrency checks.""" + return (value or {}).get("metadata", {}).get("resourceVersion") + + +def deploy( + c: dict, + dashboards: dict, + cm: dict, + scrape: list[dict], + client: Client, + dry_run: bool = False, + finalize_only: bool = False, +) -> dict: + """Validate, publish, and verify dashboards using the selected persistence mode.""" + g = c["grafana"] + names = {name: c["dashboard"][name + "_uid"] for name in dashboards} + initial = {} + # Check native-schema API support and all desired dashboards before mutation. + for name, uid in names.items(): + current = client.api(endpoint(c, uid), missing_ok=True) + initial[name] = current + path = endpoint(c, uid) if current else endpoint(c) + client.api( + path + "?dryRun=All", + resource(c, uid, dashboards[name], current), + "PUT" if current else "POST", + ) + checkpoint = None + baseline = None + if g["publication"] == "configmap-v2": + baseline = client.kube(g["namespace"], "get", "configmap", g["configmap"], missing_ok=True) + checkpoint = deepcopy(cm) + if baseline: + # Preserve unrelated dashboards and metadata in a shared ConfigMap. + checkpoint["data"] = dict(baseline.get("data", {}), **cm["data"]) + checkpoint["metadata"]["labels"] = dict( + baseline["metadata"].get("labels", {}), **checkpoint["metadata"].get("labels", {}) + ) + annotations = { + k: v + for k, v in baseline["metadata"].get("annotations", {}).items() + if k != "kubectl.kubernetes.io/last-applied-configuration" + } + if annotations: + checkpoint["metadata"]["annotations"] = annotations + if "ownerReferences" in baseline["metadata"]: + checkpoint["metadata"]["ownerReferences"] = baseline["metadata"]["ownerReferences"] + checkpoint["metadata"]["resourceVersion"] = baseline["metadata"]["resourceVersion"] + client.kube( + g["namespace"], "apply", "--dry-run=server", "-f", "-", "-o", "json", payload=checkpoint + ) + for item in scrape: + client.kube( + item["metadata"]["namespace"], + "apply", + "--dry-run=server", + "-f", + "-", + "-o", + "json", + payload=item, + ) + if dry_run: + return { + "dry_run": True, + "dashboards": list(names.values()), + "scrape_resources": len(scrape), + } + provisioned = {} + if checkpoint: + if not finalize_only: + for name, uid in names.items(): + current = client.api(endpoint(c, uid), missing_ok=True) + require( + _revision(current) == _revision(initial[name]), + "Dashboard changed before the file checkpoint; retry from fresh state", + ) + latest = client.kube(g["namespace"], "get", "configmap", g["configmap"], missing_ok=True) + require( + _revision(latest) == _revision(baseline) + and (latest or {}).get("data", {}) == (baseline or {}).get("data", {}), + "Dashboard ConfigMap changed during validation; retry from fresh state", + ) + if finalize_only: + require( + latest is not None + and all(latest.get("data", {}).get(k) == v for k, v in cm["data"].items()), + "Resume requires the exact rendered ConfigMap checkpoint", + ) + else: + client.kube(g["namespace"], "apply", "-f", "-", "-o", "json", payload=checkpoint) + deadline = time.monotonic() + 90 + while True: + matches = [] + for name, uid in names.items(): + current = client.api(endpoint(c, uid), missing_ok=True) + key = g.get("dashboard_keys", {}).get(name, name + ".json") + checksum = hashlib.md5(cm["data"][key].encode(), usedforsecurity=False).hexdigest() + provisioned[name] = current + matches.append( + bool(current) + and current["metadata"].get("annotations", {}).get("grafana.app/sourceChecksum") + == checksum + ) + if all(matches): + break + require( + time.monotonic() < deadline, + "File provisioner did not acknowledge the source; check sidecar labels and resume with --finalize-only", + ) + time.sleep(2) + for item in scrape: + client.kube(item["metadata"]["namespace"], "apply", "-f", "-", "-o", "json", payload=item) + for name, uid in names.items(): + current = client.api(endpoint(c, uid), missing_ok=True) + expected = initial[name] if g["publication"] == "api" else provisioned[name] + require( + _revision(current) == _revision(expected), + "Dashboard changed during validation; retry from fresh state", + ) + client.api( + endpoint(c, uid) if current else endpoint(c), + resource(c, uid, dashboards[name], current), + "PUT" if current else "POST", + ) + saved = client.api(endpoint(c, uid)) + require( + not saved.get("status", {}).get("conversion", {}).get("failed"), + "Grafana saved an incompatible dashboard schema", + ) + for field in ("elements", "layout", "variables", "links", "title", "timeSettings"): + require( + normalized(saved["spec"]).get(field) == normalized(dashboards[name]).get(field), + "Saved dashboard verification failed: " + field, + ) + return { + "dry_run": False, + "dashboards": list(names.values()), + "scrape_resources": len(scrape), + "verified": True, + } + + +def main(argv: list[str] | None = None) -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("command", choices=("render", "deploy")) + p.add_argument("--config", required=True, type=Path) + p.add_argument("--output", required=True, type=Path) + p.add_argument("--dry-run", action="store_true") + p.add_argument("--finalize-only", action="store_true") + args = p.parse_args(argv) + (args.output / "receipt.json").unlink(missing_ok=True) + require( + args.command == "deploy" or not (args.dry_run or args.finalize_only), + "--dry-run and --finalize-only require deploy", + ) + c = load_config(args.config) + dashboards, cm, scrape = render(c) + write_outputs(args.output, dashboards, cm, scrape) + if args.command == "render": + print("Rendered " + str(len(dashboards)) + " dashboards; no cluster access") + else: + require( + not args.finalize_only or c["grafana"]["publication"] == "configmap-v2", + "--finalize-only requires configmap-v2 publication", + ) + result = deploy(c, dashboards, cm, scrape, Client(c), args.dry_run, args.finalize_only) + (args.output / "receipt.json").write_text( + json.dumps(result, indent=2) + "\n", encoding="utf-8" + ) + print("Dry run passed" if args.dry_run else "Dashboards published and verified") + + +if __name__ == "__main__": + try: + main() + except ( + SetupError, + KeyError, + OSError, + ValueError, + TypeError, + subprocess.SubprocessError, + ) as error: + # Do not echo kube stderr, HTTP bodies, config values or authentication. + print( + str(error) + if isinstance(error, SetupError) + else "Invalid/incomplete configuration or unavailable dependency", + file=sys.stderr, + ) + sys.exit(1) diff --git a/examples/serve/grafana/example.config.json b/examples/serve/grafana/example.config.json new file mode 100644 index 000000000000..e6bf2cdc31d1 --- /dev/null +++ b/examples/serve/grafana/example.config.json @@ -0,0 +1,126 @@ +{ + "schema_version": 1, + "kubernetes": { + "context": "example-context", + "namespace": "inference", + "kubectl": "kubectl" + }, + "grafana": { + "url": "http://127.0.0.1:3000", + "namespace": "monitoring", + "api_namespace": "default", + "auth": { + "token_env": "GRAFANA_TOKEN" + }, + "publication": "api", + "configmap": "inference-dashboards", + "dashboard_keys": { + "metrics": "metrics.json", + "logs": "logs.json" + }, + "sidecar_labels": { + "grafana_dashboard": "1" + }, + "api_version": "v2" + }, + "dashboard": { + "title": "Inference", + "metrics_uid": "inference-metrics", + "logs_uid": "inference-logs", + "refresh": "1m", + "counter_lookback": "1m", + "include_logs": false, + "unique_sessions": { + "enabled": false, + "default_window": "30m", + "windows": [ + "1m", + "5m", + "15m", + "30m", + "60m" + ] + } + }, + "datasources": { + "prometheus": "prometheus", + "loki": "loki" + }, + "workload": { + "prefill_pod_regex": "inference-prefill-.*", + "decode_pod_regex": "inference-decode-.*", + "frontend_pod_regex": "inference-frontend-.*", + "all_pod_regex": "inference-.*", + "metric_job_regex": "inference-.*-metrics", + "metric_role_label": "workload_role", + "decode_metric_role_value": "decode", + "engine_endpoint": "engine", + "engine_container": "engine", + "frontend_container": "frontend", + "log_job": "inference", + "log_role_label": "role", + "prefill_log_role_value": "prefill", + "decode_log_role_value": "decode", + "log_containers": [ + "engine", + "frontend" + ], + "trace_pvc": null + }, + "scrape": { + "enabled": false, + "monitor_namespace": "monitoring", + "service_monitor_labels": { + "release": "prometheus" + }, + "interval": "15s", + "timeout": "10s", + "roles": [ + { + "name": "prefill", + "selector": { + "app": "inference", + "component": "prefill" + }, + "metric_role_value": "prefill", + "endpoints": [ + { + "name": "engine", + "port": 8001, + "path": "/prometheus/metrics" + } + ] + }, + { + "name": "decode", + "selector": { + "app": "inference", + "component": "decode" + }, + "metric_role_value": "decode", + "endpoints": [ + { + "name": "engine", + "port": 8001, + "path": "/prometheus/metrics" + } + ] + }, + { + "name": "frontend", + "selector": { + "app": "inference", + "component": "frontend" + }, + "metric_role_value": "frontend", + "endpoints": [ + { + "name": "frontend", + "port": 8000, + "path": "/metrics" + } + ] + } + ] + } +} diff --git a/examples/serve/grafana/templates/logs.json b/examples/serve/grafana/templates/logs.json new file mode 100644 index 000000000000..28e9c049d7db --- /dev/null +++ b/examples/serve/grafana/templates/logs.json @@ -0,0 +1,1770 @@ +{ + "annotations": [], + "cursorSync": "Off", + "description": "Role-aware inference monitoring. Workers are discovered at refresh. Gauges show snapshots; rates use consecutive-scrape deltas. Additive counts use stacked lines; ratios and latency stay separate.", + "editable": true, + "elements": { + "panel-124": { + "kind": "Panel", + "spec": { + "id": 124, + "title": "Prefill - TRTLLM context logs (prefill)", + "description": "Scoped to prefill; the Role selector filters this section.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "{namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",@@LOG_ROLE_LABEL@@=\"@@PREFILL_LOG_ROLE@@\",container=\"@@ENGINE_CONTAINER@@\"} |~ \"$search\"", + "legendFormat": "{{pod}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "logs", + "version": "12.3.1", + "spec": { + "options": { + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + } + } + }, + "panel-125": { + "kind": "Panel", + "spec": { + "id": 125, + "title": "Prefill - TRTLLM raw iteration logs", + "description": "Scoped to prefill; the Role selector filters this section.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "{namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" |~ \"$search\" | regexp \"global_rank = (?P[0-9]+),\" | rank=~\"${mpi_rank:raw}\"", + "legendFormat": "{{pod}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "logs", + "version": "12.3.1", + "spec": { + "options": { + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + } + } + }, + "panel-15": { + "kind": "Panel", + "spec": { + "id": 15, + "title": "Instance logs by role and container", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "{namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"@@ENGINE_PODS@@\",pod=~\".*\",container=~\"${container:regex}\"} |~ \"$search\"", + "legendFormat": "" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "logs", + "version": "12.3.1", + "spec": { + "options": { + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + } + } + }, + "panel-152": { + "kind": "Panel", + "spec": { + "id": 152, + "title": "Decode - TRTLLM generation logs (decode)", + "description": "Scoped to decode; the Role selector filters this section.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "{namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",@@LOG_ROLE_LABEL@@=\"@@DECODE_LOG_ROLE@@\",container=\"@@ENGINE_CONTAINER@@\"} |~ \"$search\"", + "legendFormat": "{{pod}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "logs", + "version": "12.3.1", + "spec": { + "options": { + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + } + } + }, + "panel-153": { + "kind": "Panel", + "spec": { + "id": 153, + "title": "Decode - TRTLLM raw iteration logs", + "description": "Scoped to decode; the Role selector filters this section.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "{namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" |~ \"$search\" | regexp \"global_rank = (?P[0-9]+),\" | rank=~\"${mpi_rank:raw}\"", + "legendFormat": "{{pod}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "logs", + "version": "12.3.1", + "spec": { + "options": { + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "id": 16, + "title": "Per-request Dynamo traces", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "{namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"@@FRONTEND_PODS@@\",container=\"@@FRONTEND_CONTAINER@@\"} |= \"dynamo.request.trace.v1\"", + "legendFormat": "" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "logs", + "version": "12.3.1", + "spec": { + "options": { + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + } + } + }, + "panel-119": { + "kind": "Panel", + "spec": { + "id": 119, + "title": "Prefill - Iteration ADP dummy tokens per iteration (latest logged) [log-only]", + "description": "Actual latest numeric iteration sample for each reporting MPI rank in the plotted $__interval time bin. No window sum, maximum, averaged rate or log-window quantile. Missing/NA and idle bins remain no data. Per-iteration token/request counts are counts in that iteration, not tokens/s or lifetime totals. No exact native exporter equivalent is currently available; retained as a log-only diagnostic.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*adp_dummy_ctx_tokens = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}} context" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*adp_dummy_gen_tokens = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}} generation" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-120": { + "kind": "Panel", + "spec": { + "id": 120, + "title": "Prefill - Iteration requests fetched per iteration (latest logged) per rank [log-only]", + "description": "Actual latest numeric iteration sample for each reporting MPI rank in the plotted $__interval time bin. No window sum, maximum, averaged rate or log-window quantile. Missing/NA and idle bins remain no data. Per-iteration token/request counts are counts in that iteration, not tokens/s or lifetime totals. No exact native exporter equivalent is currently available; retained as a log-only diagnostic.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_fetched = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-121": { + "kind": "Panel", + "spec": { + "id": 121, + "title": "Prefill - Active requests per rank (latest logged) [log-only]", + "description": "Actual latest numeric iteration sample for each reporting MPI rank in the plotted $__interval time bin. No window sum, maximum, averaged rate or log-window quantile. Missing/NA and idle bins remain no data. Per-iteration token/request counts are counts in that iteration, not tokens/s or lifetime totals. No exact native exporter equivalent is currently available; retained as a log-only diagnostic.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_active = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-122": { + "kind": "Panel", + "spec": { + "id": 122, + "title": "Prefill - Unscheduled requests per rank (latest logged) [log-only]", + "description": "Actual latest numeric iteration sample for each reporting MPI rank in the plotted $__interval time bin. No window sum, maximum, averaged rate or log-window quantile. Missing/NA and idle bins remain no data. Per-iteration token/request counts are counts in that iteration, not tokens/s or lifetime totals. No exact native exporter equivalent is currently available; retained as a log-only diagnostic.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_unscheduled = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-123": { + "kind": "Panel", + "spec": { + "id": 123, + "title": "Prefill - ADP dummy requests per rank (latest logged) [log-only]", + "description": "Actual latest numeric iteration sample for each reporting MPI rank in the plotted $__interval time bin. No window sum, maximum, averaged rate or log-window quantile. Missing/NA and idle bins remain no data. Per-iteration token/request counts are counts in that iteration, not tokens/s or lifetime totals. No exact native exporter equivalent is currently available; retained as a log-only diagnostic.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_dummy = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-147": { + "kind": "Panel", + "spec": { + "id": 147, + "title": "Decode - Iteration ADP dummy tokens per iteration (latest logged) [log-only]", + "description": "Actual latest numeric iteration sample for each reporting MPI rank in the plotted $__interval time bin. No window sum, maximum, averaged rate or log-window quantile. Missing/NA and idle bins remain no data. Per-iteration token/request counts are counts in that iteration, not tokens/s or lifetime totals. No exact native exporter equivalent is currently available; retained as a log-only diagnostic.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*adp_dummy_ctx_tokens = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}} context" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*adp_dummy_gen_tokens = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}} generation" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-148": { + "kind": "Panel", + "spec": { + "id": 148, + "title": "Decode - Iteration requests fetched per iteration (latest logged) per rank [log-only]", + "description": "Actual latest numeric iteration sample for each reporting MPI rank in the plotted $__interval time bin. No window sum, maximum, averaged rate or log-window quantile. Missing/NA and idle bins remain no data. Per-iteration token/request counts are counts in that iteration, not tokens/s or lifetime totals. No exact native exporter equivalent is currently available; retained as a log-only diagnostic.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_fetched = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-149": { + "kind": "Panel", + "spec": { + "id": 149, + "title": "Decode - Active requests per rank (latest logged) [log-only]", + "description": "Actual latest numeric iteration sample for each reporting MPI rank in the plotted $__interval time bin. No window sum, maximum, averaged rate or log-window quantile. Missing/NA and idle bins remain no data. Per-iteration token/request counts are counts in that iteration, not tokens/s or lifetime totals. No exact native exporter equivalent is currently available; retained as a log-only diagnostic.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_active = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-150": { + "kind": "Panel", + "spec": { + "id": 150, + "title": "Decode - Unscheduled requests per rank (latest logged) [log-only]", + "description": "Actual latest numeric iteration sample for each reporting MPI rank in the plotted $__interval time bin. No window sum, maximum, averaged rate or log-window quantile. Missing/NA and idle bins remain no data. Per-iteration token/request counts are counts in that iteration, not tokens/s or lifetime totals. No exact native exporter equivalent is currently available; retained as a log-only diagnostic.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_unscheduled = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + }, + "panel-151": { + "kind": "Panel", + "spec": { + "id": 151, + "title": "Decode - ADP dummy requests per rank (latest logged) [log-only]", + "description": "Actual latest numeric iteration sample for each reporting MPI rank in the plotted $__interval time bin. No window sum, maximum, averaged rate or log-window quantile. Missing/NA and idle bins remain no data. Per-iteration token/request counts are counts in that iteration, not tokens/s or lifetime totals. No exact native exporter equivalent is currently available; retained as a log-only diagnostic.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "loki", + "version": "v0", + "datasource": { + "name": "@@LOKI_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_dummy = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", + "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + } + }, + "overrides": [] + } + } + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Prefill - engine logs — ${prefill_worker:text}", + "collapse": false, + "hideHeader": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@PREFILL_PODS@@" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-124" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 10, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-125" + } + } + } + ] + } + }, + "repeat": { + "mode": "variable", + "value": "prefill_worker" + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Decode - engine logs — ${decode_worker:text}", + "collapse": false, + "hideHeader": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@DECODE_PODS@@" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-152" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 10, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-153" + } + } + } + ] + } + }, + "repeat": { + "mode": "variable", + "value": "decode_worker" + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "E2E - Selected role logs and frontend request traces", + "collapse": false, + "hideHeader": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-15" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 10, + "width": 24, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-16" + } + } + } + ] + } + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "e2e" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Prefill - log-only diagnostics by MPI rank — ${prefill_worker:text}", + "collapse": false, + "hideHeader": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@PREFILL_PODS@@" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-119" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-120" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-121" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-122" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 16, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-123" + } + } + } + ] + } + }, + "repeat": { + "mode": "variable", + "value": "prefill_worker" + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Decode - log-only diagnostics by MPI rank — ${decode_worker:text}", + "collapse": false, + "hideHeader": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@DECODE_PODS@@" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-147" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-148" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-149" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-150" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 16, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-151" + } + } + } + ] + } + }, + "repeat": { + "mode": "variable", + "value": "decode_worker" + } + } + } + ] + } + }, + "links": [ + { + "title": "Performance and metrics", + "type": "link", + "icon": "external link", + "tooltip": "", + "url": "/d/@@METRICS_UID@@", + "tags": [], + "asDropdown": false, + "targetBlank": false, + "includeVars": true, + "keepTime": true + } + ], + "preload": false, + "tags": [ + "inference", + "dynamo", + "trtllm", + "logs" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30m", + "to": "now", + "autoRefresh": "1m", + "autoRefreshIntervals": [ + "15s", + "30s", + "1m", + "5m", + "15m" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "title": "@@TITLE@@ - Logs", + "variables": [ + { + "kind": "ConstantVariable", + "spec": { + "name": "namespace", + "query": "@@NAMESPACE@@", + "current": { + "text": "@@NAMESPACE@@", + "value": "@@NAMESPACE@@" + }, + "hide": "hideVariable", + "skipUrlSync": false + } + }, + { + "kind": "CustomVariable", + "spec": { + "name": "role", + "query": "E2E : e2e,Prefill : @@PREFILL_PODS@@,Decode : @@DECODE_PODS@@,All : @@ENGINE_PODS@@", + "current": { + "text": "All", + "value": "@@ENGINE_PODS@@" + }, + "options": [ + { + "text": "E2E", + "value": "e2e", + "selected": false + }, + { + "text": "Prefill", + "value": "@@PREFILL_PODS@@", + "selected": false + }, + { + "text": "Decode", + "value": "@@DECODE_PODS@@", + "selected": false + }, + { + "text": "All", + "value": "@@ENGINE_PODS@@", + "selected": true + } + ], + "multi": false, + "includeAll": false, + "label": "Role", + "hide": "dontHide", + "skipUrlSync": false, + "allowCustomValue": false + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "worker", + "label": "Engine worker", + "hide": "dontHide", + "skipUrlSync": false, + "refresh": "onTimeRangeChanged", + "sort": "alphabeticalAsc", + "regex": "/pod=\"([^\"]+)\"/", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": false, + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "query": "query_result(up{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",endpoint=\"@@ENGINE_ENDPOINT@@\",pod=~\"${role:raw}\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + } + }, + "description": "Discovered native-engine scrape targets. Updated on load/time-range refresh; no fixed worker count or pod suffix.", + "allValue": ".*" + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "prefill_worker", + "label": "Prefill worker", + "hide": "hideVariable", + "skipUrlSync": true, + "refresh": "onTimeRangeChanged", + "sort": "alphabeticalAsc", + "regex": "/pod=\"([^\"]+)\"/", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": false, + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "query": "query_result(up{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",endpoint=\"@@ENGINE_ENDPOINT@@\",pod=~\"${role:raw}\",pod=~\"@@PREFILL_PODS@@\",pod=~\"${worker:regex}\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + } + }, + "description": "Discovered native-engine scrape targets. Updated on load/time-range refresh; no fixed worker count or pod suffix." + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "decode_worker", + "label": "Decode worker", + "hide": "hideVariable", + "skipUrlSync": true, + "refresh": "onTimeRangeChanged", + "sort": "alphabeticalAsc", + "regex": "/pod=\"([^\"]+)\"/", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": false, + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "query": "query_result(up{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",endpoint=\"@@ENGINE_ENDPOINT@@\",pod=~\"${role:raw}\",pod=~\"@@DECODE_PODS@@\",pod=~\"${worker:regex}\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + } + }, + "description": "Discovered native-engine scrape targets. Updated on load/time-range refresh; no fixed worker count or pod suffix." + } + }, + { + "kind": "TextVariable", + "spec": { + "name": "mpi_rank", + "label": "Iteration MPI rank (regex)", + "query": ".*", + "current": { + "text": ".*", + "value": ".*" + }, + "hide": "dontHide", + "skipUrlSync": false, + "description": "Local MPI rank within each selected worker, e.g. 0 or 0|1. .* shows every reported rank. No fixed rank count; separate workers can each have rank 0." + } + }, + { + "kind": "CustomVariable", + "spec": { + "name": "container", + "query": "@@CONTAINERS@@", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [], + "multi": false, + "includeAll": true, + "allValue": ".*", + "label": "Log container", + "hide": "dontHide", + "skipUrlSync": false, + "allowCustomValue": false + } + }, + { + "kind": "TextVariable", + "spec": { + "name": "search", + "current": { + "text": "", + "value": "" + }, + "query": "", + "label": "Log search", + "hide": "dontHide", + "skipUrlSync": false + } + } + ] +} diff --git a/examples/serve/grafana/templates/metrics.json b/examples/serve/grafana/templates/metrics.json new file mode 100644 index 000000000000..d0b6a2f44e65 --- /dev/null +++ b/examples/serve/grafana/templates/metrics.json @@ -0,0 +1,10065 @@ +{ + "annotations": [], + "cursorSync": "Off", + "description": "Role-aware inference monitoring. Workers are discovered at refresh. Gauges show snapshots; rates use consecutive-scrape deltas. Additive counts use stacked lines; ratios and latency stay separate.", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "Scrape health", + "description": "Current instance scrape targets, discovered from live labels. Expected counts depend on the actual number of engine workers; no fixed six-target baseline.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "up{namespace=\"$namespace\",pod=~\"@@ALL_PODS@@\",job=~\"@@METRIC_JOBS@@\"}", + "instant": true, + "legendFormat": "{{pod}} / {{service}} / {{endpoint}}", + "queryType": "instant", + "range": false + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "reduceOptions": { + "calcs": [ + "last" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + } + } + }, + "panel-105": { + "kind": "Panel", + "spec": { + "id": 105, + "title": "Prefill - Request queue time p50/p95", + "description": "Per-request wait before first scheduling in this engine instance, from the native queue-time histogram; separate from instance processing time. p50/p95 are histogram estimates for observations recorded between the latest consecutive scrapes, with reset handling. These completion-accounted observations are not timers for currently unfinished requests. No observations means no percentile; coarse buckets limit precision. Quantiles are computed separately per pod, never summed or stacked.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,@@METRIC_ROLE_LABEL@@,pod)(irate(trtllm_request_queue_time_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,@@METRIC_ROLE_LABEL@@,pod)(irate(trtllm_request_queue_time_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p95" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-107": { + "kind": "Panel", + "spec": { + "id": 107, + "title": "Prefill - GPU utilization", + "description": "Scoped to prefill; the Role selector filters this section.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "DCGM_FI_DEV_GPU_UTIL{exported_namespace=\"$namespace\",exported_pod=~\"${role:raw}\",exported_pod=~\"${prefill_worker:regex}\",exported_pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{exported_pod}} / GPU {{gpu}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-108": { + "kind": "Panel", + "spec": { + "id": 108, + "title": "Prefill - GPU memory used", + "description": "Scoped to prefill; the Role selector filters this section.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "DCGM_FI_DEV_FB_USED{exported_namespace=\"$namespace\",exported_pod=~\"${role:raw}\",exported_pod=~\"${prefill_worker:regex}\",exported_pod=~\"@@PREFILL_PODS@@\"} * 1048576", + "legendFormat": "{{exported_pod}} / GPU {{gpu}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-110": { + "kind": "Panel", + "spec": { + "id": 110, + "title": "Prefill - TRTLLM running / waiting / scheduled requests", + "description": "Scoped to prefill; the Role selector filters this section. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_num_requests_running{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / running", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_num_requests_waiting{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / waiting", + "queryType": "range" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_num_scheduled_requests{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / rank {{dp_rank}} / scheduled", + "queryType": "range" + } + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "B", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "C", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-111": { + "kind": "Panel", + "spec": { + "id": 111, + "title": "Prefill - KV block hit ratio (sample interval)", + "description": "For each point: reused block delta / (reused + missed block deltas) over matching consecutive scrape samples, by DP rank. Excludes lifetime history. An idle zero denominator is undefined, not zero. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "sum by(pod,dp_rank)(irate(trtllm_kv_cache_reused_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])) / (sum by(pod,dp_rank)(irate(trtllm_kv_cache_reused_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])) + sum by(pod,dp_rank)(irate(trtllm_kv_cache_missed_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / rank {{dp_rank}}", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-112": { + "kind": "Panel", + "spec": { + "id": 112, + "title": "Prefill - GPU KV active-block utilization", + "description": "Reported GPU active-used / maximum blocks by reporting DP rank. Available/free slots can contain evictable retained prefixes, so this is not total retained-cache occupancy. Host/SSD occupancy is not provided by this chart; the old secondary utilization line was not a reliable measure of separately retained host cache.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_kv_cache_utilization{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / rank {{dp_rank}} / GPU", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-113": { + "kind": "Panel", + "spec": { + "id": 113, + "title": "Prefill - GPU KV active / available blocks", + "description": "Reported GPU active-used and available blocks. Available slots include evictable cached prefixes; they are not necessarily empty memory. Separate query categories have separate stack groups, so active and available are not added to each other. Does not report host or SSD occupancy.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_kv_cache_used_blocks{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / rank {{dp_rank}} / used", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_kv_cache_free_blocks{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / rank {{dp_rank}} / free", + "queryType": "range" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "B", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-114": { + "kind": "Panel", + "spec": { + "id": 114, + "title": "Prefill - Instance processing time p50/p95", + "description": "Per-request wall-clock processing duration inside this engine instance: last_token_time - first_scheduled_time, from the native inference-time histogram. Excludes waiting before first scheduling and frontend end-to-end time; includes any stalls after scheduling. This is not pure GPU compute time. p50/p95 are histogram estimates for observations recorded between the latest consecutive scrapes, with reset handling. These completion-accounted observations are not timers for currently unfinished requests. No observations means no percentile; coarse buckets limit precision. Quantiles are computed separately per pod, never summed or stacked.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum by(le,@@METRIC_ROLE_LABEL@@,pod)(irate(trtllm_request_inference_time_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p50", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "histogram_quantile(0.95,sum by(le,@@METRIC_ROLE_LABEL@@,pod)(irate(trtllm_request_inference_time_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p95", + "queryType": "range" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-130": { + "kind": "Panel", + "spec": { + "id": 130, + "title": "Decode - Request queue time p50/p95", + "description": "Per-request wait before first scheduling in this engine instance, from the native queue-time histogram; separate from instance processing time. p50/p95 are histogram estimates for observations recorded between the latest consecutive scrapes, with reset handling. These completion-accounted observations are not timers for currently unfinished requests. No observations means no percentile; coarse buckets limit precision. Quantiles are computed separately per pod, never summed or stacked.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,@@METRIC_ROLE_LABEL@@,pod)(irate(trtllm_request_queue_time_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,@@METRIC_ROLE_LABEL@@,pod)(irate(trtllm_request_queue_time_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p95" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-131": { + "kind": "Panel", + "spec": { + "id": 131, + "title": "Decode - MTP average acceptance length (reported iteration)", + "description": "Reported per-iteration average tokens produced per speculative step, including the target token. This gauge may retain the most recent iteration while idle; it is not a lifetime ratio.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_spec_decode_acceptance_length{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",@@METRIC_ROLE_LABEL@@=\"@@DECODE_ROLE_VALUE@@\"}", + "legendFormat": "{{pod}} / average acceptance length" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 2, + "min": 0, + "max": 4, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-133": { + "kind": "Panel", + "spec": { + "id": 133, + "title": "Decode - GPU utilization", + "description": "Scoped to decode; the Role selector filters this section.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "DCGM_FI_DEV_GPU_UTIL{exported_namespace=\"$namespace\",exported_pod=~\"${role:raw}\",exported_pod=~\"${decode_worker:regex}\",exported_pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{exported_pod}} / GPU {{gpu}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-134": { + "kind": "Panel", + "spec": { + "id": 134, + "title": "Decode - GPU memory used", + "description": "Scoped to decode; the Role selector filters this section.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "DCGM_FI_DEV_FB_USED{exported_namespace=\"$namespace\",exported_pod=~\"${role:raw}\",exported_pod=~\"${decode_worker:regex}\",exported_pod=~\"@@DECODE_PODS@@\"} * 1048576", + "legendFormat": "{{exported_pod}} / GPU {{gpu}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-136": { + "kind": "Panel", + "spec": { + "id": 136, + "title": "Decode - TRTLLM running / waiting / scheduled requests", + "description": "Scoped to decode; the Role selector filters this section. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_num_requests_running{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / running", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_num_requests_waiting{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / waiting", + "queryType": "range" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_num_scheduled_requests{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / rank {{dp_rank}} / scheduled", + "queryType": "range" + } + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "B", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "C", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-138": { + "kind": "Panel", + "spec": { + "id": 138, + "title": "Decode - GPU KV active-block utilization", + "description": "Reported GPU active-used / maximum blocks by reporting DP rank. Available/free slots can contain evictable retained prefixes, so this is not total retained-cache occupancy. Host/SSD occupancy is not provided by this chart; the old secondary utilization line was not a reliable measure of separately retained host cache.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_kv_cache_utilization{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / rank {{dp_rank}} / GPU", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-139": { + "kind": "Panel", + "spec": { + "id": 139, + "title": "Decode - GPU KV active / available blocks", + "description": "Reported GPU active-used and available blocks. Available slots include evictable cached prefixes; they are not necessarily empty memory. Separate query categories have separate stack groups, so active and available are not added to each other. Does not report host or SSD occupancy.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_kv_cache_used_blocks{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / rank {{dp_rank}} / used", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_kv_cache_free_blocks{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / rank {{dp_rank}} / free", + "queryType": "range" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "B", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-140": { + "kind": "Panel", + "spec": { + "id": 140, + "title": "Decode - Instance processing time p50/p95", + "description": "Per-request wall-clock processing duration inside this engine instance: last_token_time - first_scheduled_time, from the native inference-time histogram. Excludes waiting before first scheduling and frontend end-to-end time; includes any stalls after scheduling. This is not pure GPU compute time. p50/p95 are histogram estimates for observations recorded between the latest consecutive scrapes, with reset handling. These completion-accounted observations are not timers for currently unfinished requests. No observations means no percentile; coarse buckets limit precision. Quantiles are computed separately per pod, never summed or stacked.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum by(le,@@METRIC_ROLE_LABEL@@,pod)(irate(trtllm_request_inference_time_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p50", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "histogram_quantile(0.95,sum by(le,@@METRIC_ROLE_LABEL@@,pod)(irate(trtllm_request_inference_time_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p95", + "queryType": "range" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-141": { + "kind": "Panel", + "spec": { + "id": 141, + "title": "Decode - TRTLLM time per output token (TPOT) p50/p95", + "description": "Engine-native TPOT: decode time per generated token, from the same request histograms as the prefill/decode panel. Context produces no output tokens, so only generation ranks report it; an empty p50/p95 with live traffic points at a scrape or Dashboard bug, not at context. Scoped to decode; the Role selector filters this section. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Percentiles describe observations recorded in that scrape interval; no observations gives no percentile.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum by(le,@@METRIC_ROLE_LABEL@@,pod)(irate(trtllm_time_per_output_token_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / p50", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "histogram_quantile(0.95,sum by(le,@@METRIC_ROLE_LABEL@@,pod)(irate(trtllm_time_per_output_token_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / {{@@METRIC_ROLE_LABEL@@}} / p95", + "queryType": "range" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-142": { + "kind": "Panel", + "spec": { + "id": 142, + "title": "Decode - runtime batch / token limits", + "description": "Scoped to decode; the Role selector filters this section.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_max_batch_size_runtime{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",@@METRIC_ROLE_LABEL@@=\"@@DECODE_ROLE_VALUE@@\"}", + "legendFormat": "{{pod}} / batch", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "trtllm_max_num_tokens_runtime{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",@@METRIC_ROLE_LABEL@@=\"@@DECODE_ROLE_VALUE@@\"}", + "legendFormat": "{{pod}} / tokens", + "queryType": "range" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-159": { + "kind": "Panel", + "spec": { + "id": 159, + "title": "Prefill - KV transfer outcomes / s", + "description": "Whole-request transfer outcomes per reporting MPI rank. Sender and receiver observations are separate; do not sum across roles as unique client requests. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_transfer_requests_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / {{outcome}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-160": { + "kind": "Panel", + "spec": { + "id": 160, + "title": "Prefill - OpenEngine RPC outcomes / s", + "description": "Generate RPC outcomes, independent of output-candidate completion metrics. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_openengine_rpc_outcomes_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / {{phase}} / {{outcome}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-161": { + "kind": "Panel", + "spec": { + "id": 161, + "title": "Prefill - KV block allocation / reuse / miss rates", + "description": "Native counters converted to blocks per second; allocation counters record positive deltas. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_allocated_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / allocated" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_new_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / new" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_reused_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / reused" + } + }, + "refId": "C", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_missed_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / missed" + } + }, + "refId": "D", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_iter_missed_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / iteration missed" + } + }, + "refId": "E", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "B", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "C", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "D", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "E" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "E", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-162": { + "kind": "Panel", + "spec": { + "id": 162, + "title": "Prefill - KV reclaimable blocks", + "description": "Reported reclaimable primary blocks at each sampled timestamp. Historical high-water marks are excluded. Reclaimable prefixes occupy storage but can be evicted; rank coverage may be limited. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_kv_cache_evictable_blocks{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / reclaimable" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-163": { + "kind": "Panel", + "spec": { + "id": 163, + "title": "Prefill - Iteration block reuse ratio (sample interval)", + "description": "Matching consecutive-scrape reused/missed iteration counter deltas, for reporting ranks only. Avoids the stale conditionally updated ratio gauge; missing counters or idle denominators remain undefined. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod,dp_rank)(irate(trtllm_kv_cache_iter_reused_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])) / (sum by(pod,dp_rank)(irate(trtllm_kv_cache_iter_reused_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])) + sum by(pod,dp_rank)(irate(trtllm_kv_cache_iter_missed_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / rank {{dp_rank}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-164": { + "kind": "Panel", + "spec": { + "id": 164, + "title": "Prefill - Scheduled KV context tokens", + "description": "Existing KV context tokens of scheduled or paused requests; not new prompt throughput. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_context_kv_tokens{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / prefill" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_generation_kv_tokens{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / decode" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_paused_kv_tokens{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / paused" + } + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "B", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "C", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-165": { + "kind": "Panel", + "spec": { + "id": 165, + "title": "Prefill - Native host / device / GPU forward step time", + "description": "Latest rank-0 samples. Device time is lagged; GPU forward time is batch-matched. These are not all-rank aggregates.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_host_step_time_seconds{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / host" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_prev_device_step_time_seconds{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / previous device" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_gpu_forward_time_seconds{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / GPU forward" + } + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-166": { + "kind": "Panel", + "spec": { + "id": 166, + "title": "Prefill - GPU forward duration p50/p95", + "description": "Rank-0 batch-matched GPU forward time histogram from exported iteration samples. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Percentiles describe observations recorded in that scrape interval; no observations gives no percentile.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,pod)(irate(trtllm_gpu_forward_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,pod)(irate(trtllm_gpu_forward_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p95" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-167": { + "kind": "Panel", + "spec": { + "id": 167, + "title": "Prefill - Host step duration p50/p95", + "description": "Rank-0 executor-loop wall-time histogram from exported iteration samples. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Percentiles describe observations recorded in that scrape interval; no observations gives no percentile.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,pod)(irate(trtllm_host_step_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,pod)(irate(trtllm_host_step_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p95" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-168": { + "kind": "Panel", + "spec": { + "id": 168, + "title": "Prefill - Prompt cache throughput", + "description": "Process-lifetime cached prompt token counter converted to tokens per second; cache flushes do not reset it. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_prompt_cached_tokens_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / cached tokens" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "tps", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-169": { + "kind": "Panel", + "spec": { + "id": 169, + "title": "Prefill - Cached prompt tokens per request p50/p95", + "description": "Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Percentiles describe observations recorded in that scrape interval; no observations gives no percentile.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,pod)(irate(trtllm_prompt_cached_tokens_per_request_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,pod)(irate(trtllm_prompt_cached_tokens_per_request_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p95" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-170": { + "kind": "Panel", + "spec": { + "id": 170, + "title": "Prefill - Prompt cache hits by tier and level / s", + "description": "Process-lifetime prompt cache hits by configured tier and level, converted to tokens per second. Connector hits are counted when loading is accepted, before asynchronous transfer completion; their presence does not establish Store enablement. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_prompt_cache_hit_tokens_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / {{cache_tier}} / level {{cache_level}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "tps", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-171": { + "kind": "Panel", + "spec": { + "id": 171, + "title": "Prefill - Iteration KV block reuse / s", + "description": "Native iteration reuse counters in blocks per second; full reuse is a subset of full plus partial. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_iter_reused_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / full + partial" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_iter_full_reused_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / full" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "B", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-172": { + "kind": "Panel", + "spec": { + "id": 172, + "title": "Prefill - Host KV discarded bytes / s", + "description": "Host KV-cache bytes discarded from reported iteration deltas; not host/network transfer bandwidth. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_host_dropped_bytes_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / discarded" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-173": { + "kind": "Panel", + "spec": { + "id": 173, + "title": "Prefill - Prefill batch occupancy", + "description": "Fraction of maximum active slots occupied by context requests.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_prefill_batch_occupancy{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / occupancy" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-174": { + "kind": "Panel", + "spec": { + "id": 174, + "title": "Prefill - Prefill batch tokens p50/p95", + "description": "Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Percentiles describe observations recorded in that scrape interval; no observations gives no percentile.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,pod)(irate(trtllm_prefill_batch_tokens_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,pod)(irate(trtllm_prefill_batch_tokens_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p95" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-175": { + "kind": "Panel", + "spec": { + "id": 175, + "title": "Decode - KV transfer outcomes / s", + "description": "Whole-request transfer outcomes per reporting MPI rank. Sender and receiver observations are separate; do not sum across roles as unique client requests. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_transfer_requests_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / {{outcome}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-176": { + "kind": "Panel", + "spec": { + "id": 176, + "title": "Decode - OpenEngine RPC outcomes / s", + "description": "Generate RPC outcomes, independent of output-candidate completion metrics. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_openengine_rpc_outcomes_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / {{phase}} / {{outcome}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-177": { + "kind": "Panel", + "spec": { + "id": 177, + "title": "Decode - KV block allocation / reuse / miss rates", + "description": "Native counters converted to blocks per second; allocation counters record positive deltas. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_allocated_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / allocated" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_new_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / new" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_reused_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / reused" + } + }, + "refId": "C", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_missed_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / missed" + } + }, + "refId": "D", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_iter_missed_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / iteration missed" + } + }, + "refId": "E", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "B", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "C", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "D", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "E" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "E", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-178": { + "kind": "Panel", + "spec": { + "id": 178, + "title": "Decode - KV reclaimable blocks", + "description": "Reported reclaimable primary blocks at each sampled timestamp. Historical high-water marks are excluded. Reclaimable prefixes occupy storage but can be evicted; rank coverage may be limited. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_kv_cache_evictable_blocks{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / reclaimable" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-180": { + "kind": "Panel", + "spec": { + "id": 180, + "title": "Decode - Scheduled KV context tokens", + "description": "Existing KV context tokens of scheduled or paused requests; not new prompt throughput. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_context_kv_tokens{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / prefill" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_generation_kv_tokens{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / decode" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_paused_kv_tokens{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / paused" + } + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "B", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "C", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-181": { + "kind": "Panel", + "spec": { + "id": 181, + "title": "Decode - Native host / device / GPU forward step time", + "description": "Latest rank-0 samples. Device time is lagged; GPU forward time is batch-matched. These are not all-rank aggregates.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_host_step_time_seconds{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / host" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_prev_device_step_time_seconds{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / previous device" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_gpu_forward_time_seconds{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}", + "legendFormat": "{{pod}} / rank {{dp_rank}} / GPU forward" + } + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-182": { + "kind": "Panel", + "spec": { + "id": 182, + "title": "Decode - GPU forward duration p50/p95", + "description": "Rank-0 batch-matched GPU forward time histogram from exported iteration samples. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Percentiles describe observations recorded in that scrape interval; no observations gives no percentile.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,pod)(irate(trtllm_gpu_forward_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,pod)(irate(trtllm_gpu_forward_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p95" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-183": { + "kind": "Panel", + "spec": { + "id": 183, + "title": "Decode - Host step duration p50/p95", + "description": "Rank-0 executor-loop wall-time histogram from exported iteration samples. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Percentiles describe observations recorded in that scrape interval; no observations gives no percentile.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,pod)(irate(trtllm_host_step_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,pod)(irate(trtllm_host_step_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p95" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-184": { + "kind": "Panel", + "spec": { + "id": 184, + "title": "Decode - Prompt cache throughput", + "description": "Process-lifetime cached prompt token counter converted to tokens per second; cache flushes do not reset it. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_prompt_cached_tokens_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / cached tokens" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "tps", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-185": { + "kind": "Panel", + "spec": { + "id": 185, + "title": "Decode - Cached prompt tokens per request p50/p95", + "description": "Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Percentiles describe observations recorded in that scrape interval; no observations gives no percentile.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,pod)(irate(trtllm_prompt_cached_tokens_per_request_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,pod)(irate(trtllm_prompt_cached_tokens_per_request_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])))", + "legendFormat": "{{pod}} / p95" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-186": { + "kind": "Panel", + "spec": { + "id": 186, + "title": "Decode - Generation KV allocation / s", + "description": "Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_gen_alloc_blocks_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / generation allocated" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-187": { + "kind": "Panel", + "spec": { + "id": 187, + "title": "Decode - Completed requests / s", + "description": "Requests completed across reported iterations; separate from frontend and RPC outcome counters. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_num_requests_completed_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / completed" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-188": { + "kind": "Panel", + "spec": { + "id": 188, + "title": "Decode - KV transfer throughput", + "description": "Bytes reported by completed-request performance metrics per second; not a NIC bandwidth counter. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "irate(trtllm_kv_cache_transfer_bytes_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])", + "legendFormat": "{{pod}} / rank {{dp_rank}} / completed-request KV bytes" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-189": { + "kind": "Panel", + "spec": { + "id": 189, + "title": "Decode - MTP acceptance by draft position", + "description": "Accepted/drafted ratio by draft position; undefined when no drafts occur. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(token_position,pod)(irate(trtllm_spec_decode_accepted_tokens_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m])) / sum by(token_position,pod)(irate(trtllm_spec_decode_drafted_tokens_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m]))", + "legendFormat": "{{pod}} / position {{token_position}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-190": { + "kind": "Panel", + "spec": { + "id": 190, + "title": "Live engine topology", + "description": "Current worker pod, node and exported TP/EP/PP/GPU configuration. Each row is an independent native-engine endpoint. Updated from live metrics; topology is not inferred from the dashboard title.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "trtllm_parallel_config_info{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",endpoint=\"@@ENGINE_ENDPOINT@@\",pod=~\"@@ENGINE_PODS@@\",pod=~\".*\"} * on(namespace,pod) group_left(node) (max by(namespace,pod,node)(kube_pod_info{namespace=\"$namespace\",pod=~\"@@ENGINE_PODS@@\",pod=~\".*\"}))", + "format": "table", + "instant": true, + "legendFormat": "{{pod}}", + "queryType": "instant", + "range": false + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [ + { + "kind": "organize", + "spec": { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "context_parallel_size": true, + "endpoint": true, + "engine_type": true, + "instance": true, + "job": true, + "namespace": true, + "service": true + }, + "renameByName": { + "expert_parallel_size": "EP", + "@@METRIC_ROLE_LABEL@@": "Role", + "gpu_count": "GPUs", + "model_name": "Model", + "node": "Node", + "pipeline_parallel_size": "PP", + "pod": "Worker", + "tensor_parallel_size": "TP" + } + } + } + } + ], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "table", + "version": "12.3.1", + "spec": { + "options": { + "showHeader": true, + "cellHeight": "sm", + "enablePagination": false + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—", + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": false, + "minWidth": 80 + } + }, + "overrides": [] + } + } + } + } + }, + "panel-191": { + "kind": "Panel", + "spec": { + "id": 191, + "title": "Discovered engine endpoints", + "description": "Actual discovered native-engine endpoints by role, including down scrape targets. No hardcoded replica count.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "count by(@@METRIC_ROLE_LABEL@@)(up{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",endpoint=\"@@ENGINE_ENDPOINT@@\",pod=~\"@@ENGINE_PODS@@\",pod=~\".*\"})", + "format": "time_series", + "instant": true, + "legendFormat": "{{@@METRIC_ROLE_LABEL@@}}", + "queryType": "instant", + "range": false + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.3.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—" + }, + "overrides": [] + } + } + } + } + }, + "panel-192": { + "kind": "Panel", + "spec": { + "id": 192, + "title": "Reported DP groups per engine", + "description": "Count of currently reported DP-rank series per engine. This is reporting coverage, not the MPI world size or TP size.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "count by(pod)(trtllm_kv_cache_utilization{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",endpoint=\"@@ENGINE_ENDPOINT@@\",pod=~\"@@ENGINE_PODS@@\",pod=~\".*\"})", + "format": "time_series", + "instant": true, + "legendFormat": "{{pod}}", + "queryType": "instant", + "range": false + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.3.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—" + }, + "overrides": [] + } + } + } + } + }, + "panel-193": { + "kind": "Panel", + "spec": { + "id": 193, + "title": "Running engines missing telemetry", + "description": "Running Kubernetes pods with a native-engine container but no healthy instance engine scrape. Independent of the worker picker, so an absent target cannot silently disappear from discovery.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "count((max by(namespace,pod)(kube_pod_init_container_info{namespace=\"$namespace\",pod=~\"@@ENGINE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} or kube_pod_container_info{namespace=\"$namespace\",pod=~\"@@ENGINE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"}) and on(namespace,pod) max by(namespace,pod)(kube_pod_status_phase{namespace=\"$namespace\",pod=~\"@@ENGINE_PODS@@\",phase=\"Running\"} == 1)) unless on(namespace,pod) (up{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",endpoint=\"@@ENGINE_ENDPOINT@@\",pod=~\"@@ENGINE_PODS@@\"} == 1)) or vector(0)", + "format": "time_series", + "instant": true, + "legendFormat": "missing", + "queryType": "instant", + "range": false + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.3.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] + }, + "noValue": "—" + }, + "overrides": [] + } + } + } + } + }, + "panel-194": { + "kind": "Panel", + "spec": { + "id": 194, + "title": "Prefill - Input tokens per second (request accounting)", + "description": "Per-instance input token throughput from the native worker counter, using its latest two scrapes. The installed exporter increments this counter when a request completes. It is completion-accounted TPS, not in-flight token emission or a lifetime total. E2E streaming output TPS is a separate frontend measurement. No rolling average is used. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod)(irate(trtllm_prompt_tokens_total{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",pod=~\"${role:raw}\",pod=~\"@@PREFILL_PODS@@\",pod=~\"${prefill_worker:regex}\"}[1m]))", + "instant": false, + "legendFormat": "{{pod}}", + "queryType": "range", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "tps", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-195": { + "kind": "Panel", + "spec": { + "id": 195, + "title": "Decode - Output tokens per second (request accounting)", + "description": "Per-instance output token throughput from the native worker counter, using its latest two scrapes. The installed exporter increments this counter when a request completes. It is completion-accounted TPS, not in-flight token emission or a lifetime total. E2E streaming output TPS is a separate frontend measurement. No rolling average is used. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod)(irate(trtllm_generation_tokens_total{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",pod=~\"${role:raw}\",pod=~\"@@DECODE_PODS@@\",pod=~\"${decode_worker:regex}\"}[1m]))", + "instant": false, + "legendFormat": "{{pod}}", + "queryType": "range", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "tps", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Frontend requests per second", + "description": "Frontend metrics cover the complete request path and remain visible for every Role selection. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod)(irate(dynamo_frontend_requests_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"}[1m]))", + "legendFormat": "{{pod}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-26": { + "kind": "Panel", + "spec": { + "id": 26, + "title": "Frontend inter-token latency", + "description": "Frontend metrics cover the complete request path and remain visible for every Role selection. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Percentiles describe observations recorded in that scrape interval; no observations gives no percentile.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "histogram_quantile(0.5,sum by(le)(irate(dynamo_frontend_inter_token_latency_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"}[1m])))", + "legendFormat": "p50", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "histogram_quantile(0.95,sum by(le)(irate(dynamo_frontend_inter_token_latency_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"}[1m])))", + "legendFormat": "p95", + "queryType": "range" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-27": { + "kind": "Panel", + "spec": { + "id": 27, + "title": "Frontend failures by reason / s", + "description": "Failure rate by reason, per second. Cumulative totals are shown separately. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "sum by (pod,reason)(irate(dynamo_frontend_failures_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"}[1m]))", + "legendFormat": "{{pod}} / {{reason}}", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Frontend TTFT - p50/p95 estimates", + "description": "Estimated percentiles of frontend time-to-first-token observations between the latest two scrapes. These are not individual request latencies. The source histogram has coarse buckets: a sparse set near the bottom of a bucket can yield a percentile estimate near its upper bound. The highest finite boundary can also cap estimates for overflow observations. Better percentile precision requires finer source buckets; latest-request values or averages are different statistics and are not substitutes.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le)(irate(dynamo_frontend_time_to_first_token_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"}[1m])))", + "legendFormat": "p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le)(irate(dynamo_frontend_time_to_first_token_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"}[1m])))", + "legendFormat": "p95" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-35": { + "kind": "Panel", + "spec": { + "id": 35, + "title": "Request-trace volume used / free", + "description": "Kubelet-reported disk usage for the request-trace PVC holding frontend gzip trace segments. Segments rotate at 64 MiB but the sink never prunes old files, so used bytes only grow; a steady slope is time until the claim fills. Archive and verify before deleting, then prune.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "kubelet_volume_stats_used_bytes{persistentvolumeclaim=\"@@TRACE_PVC@@\"}", + "legendFormat": "used", + "queryType": "range" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "kubelet_volume_stats_available_bytes{persistentvolumeclaim=\"@@TRACE_PVC@@\"}", + "legendFormat": "free", + "queryType": "range" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "max(kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=\"@@TRACE_PVC@@\"})", + "legendFormat": "capacity", + "queryType": "range" + } + }, + "refId": "C", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-39": { + "kind": "Panel", + "spec": { + "id": 39, + "title": "Healthy scrape targets", + "description": "Current instance scrape targets, discovered from live labels. Expected counts depend on the actual number of engine workers; no fixed six-target baseline.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "sum(up{namespace=\"$namespace\",pod=~\"@@ALL_PODS@@\",job=~\"@@METRIC_JOBS@@\"})", + "instant": true, + "legendFormat": "healthy", + "queryType": "instant", + "range": false + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.3.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—" + }, + "overrides": [] + } + } + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "id": 4, + "title": "Frontend end-to-end latency", + "description": "Frontend metrics cover the complete request path and remain visible for every Role selection. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Percentiles describe observations recorded in that scrape interval; no observations gives no percentile.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le)(irate(dynamo_frontend_request_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"}[1m])))", + "legendFormat": "p50" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le)(irate(dynamo_frontend_request_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"}[1m])))", + "legendFormat": "p95" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + }, + "panel-40": { + "kind": "Panel", + "spec": { + "id": 40, + "title": "Model ready", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "max(dynamo_frontend_model_ready{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"})", + "instant": true, + "legendFormat": "ready", + "queryType": "instant", + "range": false + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.3.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—" + }, + "overrides": [] + } + } + } + } + }, + "panel-41": { + "kind": "Panel", + "spec": { + "id": 41, + "title": "GPUs with telemetry", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "count(DCGM_FI_DEV_FB_USED{exported_namespace=\"$namespace\",exported_pod=~\"@@ENGINE_PODS@@\",exported_pod=~\".*\"})", + "instant": true, + "legendFormat": "GPUs", + "queryType": "instant", + "range": false + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.3.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "—" + }, + "overrides": [] + } + } + } + } + }, + "panel-42": { + "kind": "Panel", + "spec": { + "id": 42, + "title": "Frontend requests per second", + "description": "Latest request counter change divided by elapsed time between the last two scrapes. Not a selected-range total or rolling average. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "sum(irate(dynamo_frontend_requests_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"}[1m]))", + "instant": true, + "legendFormat": "requests", + "queryType": "instant", + "range": false + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.3.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + } + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "noValue": "—" + }, + "overrides": [] + } + } + } + } + }, + "panel-43": { + "kind": "Panel", + "spec": { + "id": 43, + "title": "Request-trace volume free", + "description": "Unused fraction of the 20-GiB request-trace PVC. Red means the primary acceptance dump is at risk: the file sink drops records once the claim fills. The used/free trend panel shows the same claim, so archive and verify before it fills.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "editorMode": "code", + "expr": "min(kubelet_volume_stats_available_bytes{persistentvolumeclaim=\"@@TRACE_PVC@@\"} / kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=\"@@TRACE_PVC@@\"})", + "instant": true, + "legendFormat": "free", + "queryType": "instant", + "range": false + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "12.3.1", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 2, + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "red" + }, + { + "value": 0.15, + "color": "yellow" + }, + { + "value": 0.3, + "color": "green" + } + ] + }, + "noValue": "—" + }, + "overrides": [] + } + } + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "id": 5, + "title": "Output tokens per second", + "description": "Live aggregate output token rate from the streaming counter, averaged over the selected rate interval. The counter updates during generation, avoiding completion-time output-length spikes. Each point uses the latest consecutive-scrape counter deltas, with reset handling, before combining series. The 1m range only finds two samples; it does not average them across a minute. The actual elapsed scrape interval determines the per-second value. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod)(irate(dynamo_frontend_output_tokens_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"}[1m]))", + "legendFormat": "{{pod}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "id": 6, + "title": "Frontend active requests", + "description": "Frontend metrics cover the complete request path and remain visible for every Role selection. Normal line stacking adds instance/rank contributions at the same timestamp; it does not accumulate over time. Different query categories use separate stack groups to avoid adding overlapping quantities.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "dynamo_frontend_active_requests{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"@@FRONTEND_PODS@@\"}", + "legendFormat": "{{pod}} / {{model}}" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + } + } + } + } + }, + "observability-196": { + "kind": "Panel", + "spec": { + "id": 196, + "title": "Prefill - Physical tier storage states", + "description": "Logical slot bytes, disjoint free/evictable/unavailable states. Retained evictable prefixes occupy storage and may be useful to a later request. Pool groups are byte-weighted; this is not device allocator usage. Reporting-rank coverage can be limited.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod,dp_rank,cache_level,cache_tier,state)(trtllm_kv_cache_tier_bytes{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"})", + "legendFormat": "{{pod}} / rank {{dp_rank}} / level {{cache_level}} / {{cache_tier}} / {{state}}", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-197": { + "kind": "Panel", + "spec": { + "id": 197, + "title": "Prefill - Tier resident occupancy", + "description": "Resident = (unavailable + evictable) / total logical slot bytes at the current sample. High occupancy alone is not proof of thrashing; correlate churn, final drops and recent misses. Missing/zero-capacity observations are undefined, not zero.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod,dp_rank,cache_level,cache_tier)(trtllm_kv_cache_tier_bytes{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",state=~\"unavailable|evictable\"}) / sum by(pod,dp_rank,cache_level,cache_tier)(trtllm_kv_cache_tier_bytes{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"})", + "legendFormat": "{{pod}} / rank {{dp_rank}} / level {{cache_level}} / {{cache_tier}}", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-198": { + "kind": "Panel", + "spec": { + "id": 198, + "title": "Prefill - Tier restore / eviction turnover", + "description": "rate(moved bytes) / current capacity, by attributed level. This is capacity turnovers per second and may exceed 1/s. Restore is attributed to the hotter destination; evict to its departing source. Counts reflect accepted submissions, not confirmed DMA.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod,dp_rank,cache_level,cache_tier,operation)(rate(trtllm_kv_cache_tier_moved_bytes_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",operation=~\"restore|evict\"}[1m])) / on(pod,dp_rank,cache_level,cache_tier) group_left sum by(pod,dp_rank,cache_level,cache_tier)(trtllm_kv_cache_tier_bytes{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"})", + "legendFormat": "{{pod}} / rank {{dp_rank}} / level {{cache_level}} / {{cache_tier}} / {{operation}}", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "suffix:/s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-199": { + "kind": "Panel", + "spec": { + "id": 199, + "title": "Prefill - Tier capacity / priority drops", + "description": "Final prefix-loss bytes per second, separated by capacity and priority policy. Ordinary request release and explicit cache clearing are excluded. These rate() queries average counter deltas over the configured lookback.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod,dp_rank,cache_level,cache_tier,operation)(rate(trtllm_kv_cache_tier_moved_bytes_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",operation=~\"drop_capacity|drop_priority\"}[1m]))", + "legendFormat": "{{pod}} / rank {{dp_rank}} / level {{cache_level}} / {{cache_tier}} / {{operation}}", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-200": { + "kind": "Panel", + "spec": { + "id": 200, + "title": "Prefill - Sampled restore latency p50/p95", + "description": "Opt-in sampled migration-batch CUDA stream intervals over 5m, bucket-merged across pools per source/destination path. Not per-request restore stall, exclusive GPU busy time, or all-rank latency. Sparse samples limit percentiles. Disabled or pending samples are not zero latency.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,pod,dp_rank,source_cache_level,destination_cache_level)(rate(trtllm_kv_cache_tier_migration_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",operation=\"restore\"}[5m])))", + "legendFormat": "{{pod}} / rank {{dp_rank}} / {{source_cache_level}} -> {{destination_cache_level}} / p50", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,pod,dp_rank,source_cache_level,destination_cache_level)(rate(trtllm_kv_cache_tier_migration_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",operation=\"restore\"}[5m])))", + "legendFormat": "{{pod}} / rank {{dp_rank}} / {{source_cache_level}} -> {{destination_cache_level}} / p95", + "refId": "B" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-201": { + "kind": "Panel", + "spec": { + "id": 201, + "title": "Prefill - Migration timing samples skipped / s", + "description": "Eligible timing samples skipped while the bounded pending-event budget is full. Zero means no measured skips; absent does not prove that timing is enabled. Permanently pending events can exhaust the sampling budget until teardown.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod,dp_rank)(rate(trtllm_kv_cache_migration_timing_skipped_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[1m]))", + "legendFormat": "{{pod}} / rank {{dp_rank}} / skipped", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-202": { + "kind": "Panel", + "spec": { + "id": 202, + "title": "Prefill - Request KV transfer duration p50/p95", + "description": "Completed-request KV-transfer start/end duration, counted once across output candidates. Includes only supplied valid measurements. This is not NIC utilization, sampled migration-batch latency, or necessarily request stall; prefill and decode observations are separate.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,pod)(rate(trtllm_kv_cache_transfer_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[5m])))", + "legendFormat": "{{pod}} / p50", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,pod)(rate(trtllm_kv_cache_transfer_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\"}[5m])))", + "legendFormat": "{{pod}} / p95", + "refId": "B" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-203": { + "kind": "Panel", + "spec": { + "id": 203, + "title": "Decode - Physical tier storage states", + "description": "Logical slot bytes, disjoint free/evictable/unavailable states. Retained evictable prefixes occupy storage and may be useful to a later request. Pool groups are byte-weighted; this is not device allocator usage. Reporting-rank coverage can be limited.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod,dp_rank,cache_level,cache_tier,state)(trtllm_kv_cache_tier_bytes{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"})", + "legendFormat": "{{pod}} / rank {{dp_rank}} / level {{cache_level}} / {{cache_tier}} / {{state}}", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-204": { + "kind": "Panel", + "spec": { + "id": 204, + "title": "Decode - Tier resident occupancy", + "description": "Resident = (unavailable + evictable) / total logical slot bytes at the current sample. High occupancy alone is not proof of thrashing; correlate churn, final drops and recent misses. Missing/zero-capacity observations are undefined, not zero.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod,dp_rank,cache_level,cache_tier)(trtllm_kv_cache_tier_bytes{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",state=~\"unavailable|evictable\"}) / sum by(pod,dp_rank,cache_level,cache_tier)(trtllm_kv_cache_tier_bytes{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"})", + "legendFormat": "{{pod}} / rank {{dp_rank}} / level {{cache_level}} / {{cache_tier}}", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-205": { + "kind": "Panel", + "spec": { + "id": 205, + "title": "Decode - Tier restore / eviction turnover", + "description": "rate(moved bytes) / current capacity, by attributed level. This is capacity turnovers per second and may exceed 1/s. Restore is attributed to the hotter destination; evict to its departing source. Counts reflect accepted submissions, not confirmed DMA.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod,dp_rank,cache_level,cache_tier,operation)(rate(trtllm_kv_cache_tier_moved_bytes_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",operation=~\"restore|evict\"}[1m])) / on(pod,dp_rank,cache_level,cache_tier) group_left sum by(pod,dp_rank,cache_level,cache_tier)(trtllm_kv_cache_tier_bytes{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"})", + "legendFormat": "{{pod}} / rank {{dp_rank}} / level {{cache_level}} / {{cache_tier}} / {{operation}}", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "suffix:/s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-206": { + "kind": "Panel", + "spec": { + "id": 206, + "title": "Decode - Tier capacity / priority drops", + "description": "Final prefix-loss bytes per second, separated by capacity and priority policy. Ordinary request release and explicit cache clearing are excluded. These rate() queries average counter deltas over the configured lookback.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod,dp_rank,cache_level,cache_tier,operation)(rate(trtllm_kv_cache_tier_moved_bytes_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",operation=~\"drop_capacity|drop_priority\"}[1m]))", + "legendFormat": "{{pod}} / rank {{dp_rank}} / level {{cache_level}} / {{cache_tier}} / {{operation}}", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-207": { + "kind": "Panel", + "spec": { + "id": 207, + "title": "Decode - Sampled restore latency p50/p95", + "description": "Opt-in sampled migration-batch CUDA stream intervals over 5m, bucket-merged across pools per source/destination path. Not per-request restore stall, exclusive GPU busy time, or all-rank latency. Sparse samples limit percentiles. Disabled or pending samples are not zero latency.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,pod,dp_rank,source_cache_level,destination_cache_level)(rate(trtllm_kv_cache_tier_migration_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",operation=\"restore\"}[5m])))", + "legendFormat": "{{pod}} / rank {{dp_rank}} / {{source_cache_level}} -> {{destination_cache_level}} / p50", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,pod,dp_rank,source_cache_level,destination_cache_level)(rate(trtllm_kv_cache_tier_migration_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",operation=\"restore\"}[5m])))", + "legendFormat": "{{pod}} / rank {{dp_rank}} / {{source_cache_level}} -> {{destination_cache_level}} / p95", + "refId": "B" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-208": { + "kind": "Panel", + "spec": { + "id": 208, + "title": "Decode - Migration timing samples skipped / s", + "description": "Eligible timing samples skipped while the bounded pending-event budget is full. Zero means no measured skips; absent does not prove that timing is enabled. Permanently pending events can exhaust the sampling budget until teardown.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "sum by(pod,dp_rank)(rate(trtllm_kv_cache_migration_timing_skipped_total{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[1m]))", + "legendFormat": "{{pod}} / rank {{dp_rank}} / skipped", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + }, + "observability-209": { + "kind": "Panel", + "spec": { + "id": 209, + "title": "Decode - Request KV transfer duration p50/p95", + "description": "Completed-request KV-transfer start/end duration, counted once across output candidates. Includes only supplied valid measurements. This is not NIC utilization, sampled migration-batch latency, or necessarily request stall; prefill and decode observations are separate.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.5,sum by(le,pod)(rate(trtllm_kv_cache_transfer_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[5m])))", + "legendFormat": "{{pod}} / p50", + "refId": "A" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "expr": "histogram_quantile(0.95,sum by(le,pod)(rate(trtllm_kv_cache_transfer_duration_seconds_bucket{job=~\"@@METRIC_JOBS@@\",namespace=\"$namespace\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\"}[5m])))", + "legendFormat": "{{pod}} / p95", + "refId": "B" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "12.3.1", + "spec": { + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "—", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + } + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "title": "", + "collapse": false, + "hideHeader": true, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "e2e" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-39" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 4, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-40" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-41" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-42" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-43" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 20, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "E2E - Live deployment topology", + "collapse": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "e2e" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 7, + "element": { + "kind": "ElementReference", + "name": "panel-190" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 7, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-191" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 7, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-192" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 7, + "width": 8, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-193" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "E2E - Frontend serving and shared health", + "collapse": false, + "hideHeader": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "e2e" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-27" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 0, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 8, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 16, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-5" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 16, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-6" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 24, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-26" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 24, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-35" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Prefill - input throughput by instance (stacked TPS)", + "collapse": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@PREFILL_PODS@@" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 7, + "element": { + "kind": "ElementReference", + "name": "panel-194" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Prefill - additive counts across instances", + "collapse": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@PREFILL_PODS@@" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "layout": { + "kind": "AutoGridLayout", + "spec": { + "maxColumnCount": 2, + "columnWidthMode": "custom", + "columnWidth": 600, + "rowHeightMode": "custom", + "rowHeight": 300, + "fillScreen": false, + "items": [ + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-159" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-160" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-161" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-162" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-164" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-168" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-170" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-171" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-172" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-110" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-113" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Decode - output throughput by instance (stacked TPS)", + "collapse": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@DECODE_PODS@@" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 7, + "element": { + "kind": "ElementReference", + "name": "panel-195" + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Decode - additive counts across instances", + "collapse": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@DECODE_PODS@@" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "layout": { + "kind": "AutoGridLayout", + "spec": { + "maxColumnCount": 2, + "columnWidthMode": "custom", + "columnWidth": 600, + "rowHeightMode": "custom", + "rowHeight": 300, + "fillScreen": false, + "items": [ + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-175" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-176" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-177" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-178" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-180" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-184" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-186" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-187" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-188" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-136" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-139" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Prefill - engine, KV cache and GPUs — ${prefill_worker:text}", + "collapse": false, + "hideHeader": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@PREFILL_PODS@@" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "repeat": { + "mode": "variable", + "value": "prefill_worker" + }, + "layout": { + "kind": "AutoGridLayout", + "spec": { + "maxColumnCount": 2, + "columnWidthMode": "custom", + "columnWidth": 600, + "rowHeightMode": "custom", + "rowHeight": 300, + "fillScreen": false, + "items": [ + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-163" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-165" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-166" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-167" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-169" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-173" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-174" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-105" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-107" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-108" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-111" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-112" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-114" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-196" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-197" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-198" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-199" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-200" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-201" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-202" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + } + ] + } + } + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Decode - engine, KV cache and GPUs — ${decode_worker:text}", + "collapse": false, + "hideHeader": false, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "or", + "items": [ + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@DECODE_PODS@@" + } + }, + { + "kind": "ConditionalRenderingVariable", + "spec": { + "variable": "role", + "operator": "equals", + "value": "@@ENGINE_PODS@@" + } + } + ] + } + }, + "repeat": { + "mode": "variable", + "value": "decode_worker" + }, + "layout": { + "kind": "AutoGridLayout", + "spec": { + "maxColumnCount": 2, + "columnWidthMode": "custom", + "columnWidth": 600, + "rowHeightMode": "custom", + "rowHeight": 300, + "fillScreen": false, + "items": [ + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-181" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-182" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-183" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-185" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-189" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-130" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-131" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-133" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-134" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-138" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-140" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-141" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-142" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-203" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-204" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-205" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-206" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-207" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-208" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + }, + { + "kind": "AutoGridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "observability-209" + }, + "conditionalRendering": { + "kind": "ConditionalRenderingGroup", + "spec": { + "visibility": "show", + "condition": "and", + "items": [ + { + "kind": "ConditionalRenderingData", + "spec": { + "value": true + } + } + ] + } + } + } + } + ] + } + } + } + } + ] + } + }, + "links": [ + { + "title": "Logs and request traces", + "type": "link", + "icon": "external link", + "tooltip": "", + "url": "/d/@@LOGS_UID@@", + "tags": [], + "asDropdown": false, + "targetBlank": false, + "includeVars": true, + "keepTime": true + } + ], + "preload": false, + "tags": [ + "inference", + "dynamo", + "trtllm" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-30m", + "to": "now", + "autoRefresh": "1m", + "autoRefreshIntervals": [ + "15s", + "30s", + "1m", + "5m", + "15m" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "title": "@@TITLE@@ - Metrics", + "variables": [ + { + "kind": "ConstantVariable", + "spec": { + "name": "namespace", + "query": "@@NAMESPACE@@", + "current": { + "text": "@@NAMESPACE@@", + "value": "@@NAMESPACE@@" + }, + "hide": "hideVariable", + "skipUrlSync": false + } + }, + { + "kind": "CustomVariable", + "spec": { + "name": "role", + "query": "E2E : e2e,Prefill : @@PREFILL_PODS@@,Decode : @@DECODE_PODS@@,All : @@ENGINE_PODS@@", + "current": { + "text": "All", + "value": "@@ENGINE_PODS@@" + }, + "options": [ + { + "selected": false, + "text": "E2E", + "value": "e2e" + }, + { + "selected": false, + "text": "Prefill", + "value": "@@PREFILL_PODS@@" + }, + { + "selected": false, + "text": "Decode", + "value": "@@DECODE_PODS@@" + }, + { + "selected": true, + "text": "All", + "value": "@@ENGINE_PODS@@" + } + ], + "multi": false, + "includeAll": false, + "label": "Role", + "hide": "dontHide", + "skipUrlSync": false, + "allowCustomValue": false + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "worker", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Engine worker", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "description": "Discovered native-engine scrape targets. Updated on load/time-range refresh; no fixed worker count or pod suffix.", + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "query": "query_result(up{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",endpoint=\"@@ENGINE_ENDPOINT@@\",pod=~\"${role:raw}\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + } + }, + "regex": "/pod=\"([^\"]+)\"/", + "sort": "alphabeticalAsc", + "options": [], + "multi": true, + "includeAll": true, + "allValue": ".*", + "allowCustomValue": false + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "prefill_worker", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Prefill worker", + "hide": "hideVariable", + "refresh": "onTimeRangeChanged", + "skipUrlSync": true, + "description": "Discovered native-engine scrape targets. Updated on load/time-range refresh; no fixed worker count or pod suffix.", + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "query": "query_result(up{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",endpoint=\"@@ENGINE_ENDPOINT@@\",pod=~\"${role:raw}\",pod=~\"@@PREFILL_PODS@@\",pod=~\"${worker:regex}\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + } + }, + "regex": "/pod=\"([^\"]+)\"/", + "sort": "alphabeticalAsc", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": false + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "decode_worker", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "Decode worker", + "hide": "hideVariable", + "refresh": "onTimeRangeChanged", + "skipUrlSync": true, + "description": "Discovered native-engine scrape targets. Updated on load/time-range refresh; no fixed worker count or pod suffix.", + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "@@PROMETHEUS_UID@@" + }, + "spec": { + "query": "query_result(up{namespace=\"$namespace\",job=~\"@@METRIC_JOBS@@\",endpoint=\"@@ENGINE_ENDPOINT@@\",pod=~\"${role:raw}\",pod=~\"@@DECODE_PODS@@\",pod=~\"${worker:regex}\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + } + }, + "regex": "/pod=\"([^\"]+)\"/", + "sort": "alphabeticalAsc", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": false + } + } + ] +} diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 95d3041933e3..fa807f709c5b 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -416,6 +416,7 @@ l0_cpu: - unittest/llmapi/test_trtllm_serve_config_schemas.py - unittest/llmapi/test_utils.py - unittest/llmapi/test_visual_gen_startup_shutdown.py + - unittest/metrics/test_grafana_dashboard.py - unittest/metrics/test_collector.py - unittest/metrics/test_perf.py - unittest/models/test_quant_config_utils.py diff --git a/tests/unittest/metrics/test_grafana_dashboard.py b/tests/unittest/metrics/test_grafana_dashboard.py new file mode 100644 index 000000000000..6b65a762f404 --- /dev/null +++ b/tests/unittest/metrics/test_grafana_dashboard.py @@ -0,0 +1,518 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Portability, safe publication, and authentication boundary tests.""" + +import hashlib +import importlib.util +import io +import json +import os +import subprocess +import tempfile +import unittest +import urllib.error +from copy import deepcopy +from pathlib import Path +from unittest.mock import patch + +import pytest + +pytestmark = pytest.mark.cpu_only +ROOT = Path(__file__).resolve().parents[3] +DASHBOARD_DIR = ROOT / "examples/serve/grafana" +_spec = importlib.util.spec_from_file_location( + "trtllm_grafana_dashboard", DASHBOARD_DIR / "dashboard.py" +) +assert _spec is not None and _spec.loader is not None +app = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(app) + + +def config() -> dict: + c = app.load_config(DASHBOARD_DIR / "example.config.json") + c["dashboard"]["include_logs"] = True + return c + + +class SessionPanelTests(unittest.TestCase): + def test_optional_logs_only_and_idempotent(self) -> None: + c = config() + baseline, _, _ = app.render(c) + c["dashboard"]["unique_sessions"] = {"enabled": True} + dashboards, _, _ = app.render(c) + self.assertEqual(baseline["metrics"], dashboards["metrics"]) + logs = dashboards["logs"] + self.assertEqual(len(logs["elements"]), len(baseline["logs"]["elements"]) + 1) + before = deepcopy(logs) + app.add_unique_sessions(logs, c) + self.assertEqual(before, logs) + panel = logs["elements"]["session-unique-observed"]["spec"] + queries = panel["data"]["spec"]["queries"] + self.assertEqual(len(queries), 9) + for i, q in enumerate(queries[:8]): + expr = q["spec"]["query"]["spec"]["expr"] + self.assertIn("sum by (session_id)", expr) + self.assertIn('session_partition="' + str(i) + '"', expr) + self.assertNotIn("$search", expr) + self.assertIn("[${session_window:raw}]", expr) + self.assertEqual( + queries[-1]["spec"]["query"]["spec"]["expression"], + "$A + $B + $C + $D + $E + $F + $G + $H", + ) + values = [ + x["spec"]["value"] + for x in logs["layout"]["spec"]["rows"][0]["spec"]["conditionalRendering"]["spec"][ + "items" + ] + ] + self.assertIn("e2e", values) + + def test_invalid_window_rejected(self) -> None: + c = config() + c["dashboard"]["unique_sessions"] = { + "enabled": True, + "windows": ["120m"], + "default_window": "120m", + } + with self.assertRaises(app.SetupError): + app.render(c) + + +class FakeClient: + def __init__(self, c: dict) -> None: + self.c = c + self.calls = [] + self.saved = {} + self.cm = None + self.edit_on_get = False + + def api( + self, + path: str, + payload: dict | None = None, + method: str | None = None, + missing_ok: bool = False, + ) -> dict | None: + self.calls.append(("api", path, method)) + uid = path.split("?")[0].rsplit("/", 1)[-1] + if payload is not None: + if "dryRun=All" in path: + return deepcopy(payload) + uid = payload["metadata"]["name"] + self.saved[uid] = deepcopy(payload) + self.saved[uid]["metadata"]["resourceVersion"] = "2" + return deepcopy(self.saved[uid]) + result = deepcopy(self.saved.get(uid)) + if self.edit_on_get and result: + result["metadata"]["resourceVersion"] = str(len(self.calls)) + return result + + def kube( + self, namespace: str, *args: str, payload: dict | None = None, missing_ok: bool = False + ) -> dict | None: + self.calls.append(("kube", namespace, args)) + if args[0] == "get": + return deepcopy(self.cm) + if "--dry-run=server" not in args: + self.cm = deepcopy(payload) + self.cm["metadata"]["resourceVersion"] = "2" + for name in ("metrics", "logs"): + if name + ".json" not in self.cm["data"]: + continue + uid = self.c["dashboard"][name + "_uid"] + self.saved[uid] = { + "metadata": { + "name": uid, + "resourceVersion": "1", + "annotations": { + "grafana.app/sourceChecksum": hashlib.md5( + self.cm["data"][name + ".json"].encode(), usedforsecurity=False + ).hexdigest() + }, + }, + "spec": {}, + } + return deepcopy(payload) + + +class DashboardTests(unittest.TestCase): + def test_offline_render_has_no_auth_or_kubernetes_access(self) -> None: + with patch("subprocess.run", side_effect=AssertionError("Cluster access during render")): + dashboards, cm, scrape = app.render(config()) + self.assertEqual(len(dashboards["metrics"]["elements"]), 80) + panels = dashboards["metrics"]["elements"] + self.assertTrue({"panel-106", "panel-132", "panel-104", "panel-129"}.isdisjoint(panels)) + for key in ("panel-114", "panel-140"): + queries = panels[key]["spec"]["data"]["spec"]["queries"] + self.assertEqual(len(queries), 2) + for q, percentile in zip(queries, ("0.5", "0.95")): + expr = q["spec"]["query"]["spec"]["expr"] + self.assertIn("histogram_quantile(" + percentile + ",", expr) + self.assertIn("trtllm_request_inference_time_seconds_bucket", expr) + self.assertIn("sum by(le,workload_role,pod)", expr) + self.assertEqual(len(dashboards["logs"]["elements"]), 16) + self.assertEqual(scrape, []) + self.assertNotIn("auth", cm) + self.assertEqual(json.loads(cm["data"]["metrics.json"])["uid"], "inference-metrics") + self.assertNotIn("uid", dashboards["metrics"]) + + def test_second_cluster_uses_only_its_own_inputs(self) -> None: + c = config() + c["kubernetes"].update(context="second-context", namespace="model-team") + c["dashboard"].update( + metrics_uid="model-metrics", logs_uid="model-logs", title="Model team" + ) + c["datasources"] = {"prometheus": "metrics-other", "loki": "logs-other"} + w = c["workload"] + w.update( + prefill_pod_regex="context-worker-.*", + decode_pod_regex="generation-worker-.*", + frontend_pod_regex="api-worker-.*", + all_pod_regex=".*-worker-.*", + metric_job_regex="team-model-.*", + metric_role_label="engine_role", + trace_pvc="trace-data", + ) + dashboards, cm, _ = app.render(c) + text = json.dumps(dashboards) + for unwanted in ( + "inference-prefill-", + "inference-decode-", + "inference-frontend-", + "inference-metrics", + "workload_role", + "@@", + ): + self.assertNotIn(unwanted, text) + self.assertEqual(len(dashboards["metrics"]["elements"]), 82) + self.assertIn("/d/model-logs", text) + self.assertIn("engine_role", text) + for panel in dashboards["metrics"]["elements"].values(): + for q in panel["spec"]["data"]["spec"]["queries"]: + self.assertEqual(q["spec"]["query"]["datasource"]["name"], "metrics-other") + expr = q["spec"]["query"]["spec"].get("expr", "") + if "kubelet_volume_stats_" in expr: + self.assertIn('namespace="$namespace",persistentvolumeclaim="trace-data"', expr) + self.assertNotIn("second-context", json.dumps(cm)) + + def test_no_logs_option_removes_navigation_without_deleting_other_dashboards(self) -> None: + c = config() + c["dashboard"]["include_logs"] = False + dashboards, cm, _ = app.render(c) + self.assertEqual(set(dashboards), {"metrics"}) + self.assertNotIn("/d/inference-logs", json.dumps(dashboards)) + self.assertEqual(set(cm["data"]), {"metrics.json"}) + + def test_existing_configmap_file_keys_can_be_retained(self) -> None: + c = config() + c["grafana"]["dashboard_keys"] = { + "metrics": "owned-metrics.json", + "logs": "owned-logs.json", + } + _, cm, _ = app.render(c) + self.assertEqual(set(cm["data"]), {"owned-metrics.json", "owned-logs.json"}) + + def test_optional_monitors_have_scoped_selectors_and_labels(self) -> None: + c = config() + c["scrape"]["enabled"] = True + _, _, resources = app.render(c) + self.assertEqual(len(resources), 6) + for service, monitor in zip(resources[::2], resources[1::2]): + self.assertEqual(service["metadata"]["namespace"], "inference") + self.assertEqual(monitor["metadata"]["namespace"], "monitoring") + self.assertEqual(monitor["spec"]["namespaceSelector"], {"matchNames": ["inference"]}) + self.assertEqual( + monitor["spec"]["selector"]["matchLabels"], service["metadata"]["labels"] + ) + + def test_dry_run_never_publishes(self) -> None: + c = config() + c["grafana"]["publication"] = "configmap-v2" + dashboards, cm, scrape = app.render(c) + client = FakeClient(c) + receipt = app.deploy(c, dashboards, cm, scrape, client, dry_run=True) + self.assertTrue(receipt["dry_run"]) + self.assertFalse(client.saved) + self.assertIsNone(client.cm) + for call in client.calls: + if call[0] == "api" and call[2]: + self.assertIn("dryRun=All", call[1]) + if call[0] == "kube" and call[2][0] == "apply": + self.assertIn("--dry-run=server", call[2]) + + def test_api_creation_and_postsave_verification(self) -> None: + c = config() + ds, cm, s = app.render(c) + client = FakeClient(c) + result = app.deploy(c, ds, cm, s, client) + self.assertTrue(result["verified"]) + self.assertEqual(len(client.saved), 2) + self.assertIsNone(client.cm) + + def test_shared_configmap_preserves_other_dashboards_and_labels(self) -> None: + c = config() + c["grafana"]["publication"] = "configmap-v2" + ds, cm, s = app.render(c) + client = FakeClient(c) + client.cm = { + "metadata": { + "resourceVersion": "1", + "labels": {"keep": "yes"}, + "annotations": {"folder": "keep"}, + }, + "data": {"unrelated.json": "untouched"}, + } + app.deploy(c, ds, cm, s, client) + self.assertEqual(client.cm["data"]["unrelated.json"], "untouched") + self.assertEqual(client.cm["metadata"]["labels"]["keep"], "yes") + self.assertEqual(client.cm["metadata"]["annotations"]["folder"], "keep") + + def test_concurrent_dashboard_change_stops_publication(self) -> None: + c = config() + ds, cm, s = app.render(c) + client = FakeClient(c) + for name, spec in ds.items(): + uid = c["dashboard"][name + "_uid"] + client.saved[uid] = app.resource(c, uid, spec, None) + client.saved[uid]["metadata"]["resourceVersion"] = "1" + client.edit_on_get = True + with self.assertRaisesRegex(app.SetupError, "changed during validation"): + app.deploy(c, ds, cm, s, client) + self.assertFalse( + any(x[0] == "api" and x[2] and "dryRun=All" not in x[1] for x in client.calls) + ) + + def test_auth_errors_do_not_echo_server_body_or_secret(self) -> None: + c = config() + secret = "private-sentinel" + with patch.dict(os.environ, {"GRAFANA_TOKEN": secret}): + client = app.Client(c) + error = urllib.error.HTTPError( + "https://internal.invalid", 403, "denied", {}, io.BytesIO(secret.encode()) + ) + with patch("urllib.request.OpenerDirector.open", side_effect=error): + with self.assertRaises(app.SetupError) as caught: + client.api("/test") + self.assertNotIn(secret, str(caught.exception)) + self.assertNotIn("internal.invalid", str(caught.exception)) + + def test_kubernetes_context_and_namespace_are_explicit(self) -> None: + c = config() + with patch.dict(os.environ, {"GRAFANA_TOKEN": "test-only"}): + client = app.Client(c) + with patch( + "subprocess.run", return_value=subprocess.CompletedProcess([], 0, "{}", "") + ) as run: + client.kube("monitoring", "get", "configmap", "test") + command = run.call_args.args[0] + self.assertEqual( + command[:5], ["kubectl", "--context", "example-context", "-n", "monitoring"] + ) + self.assertTrue(run.call_args.kwargs["capture_output"]) + + def test_kubernetes_errors_do_not_expose_stderr(self) -> None: + c = config() + with patch.dict(os.environ, {"GRAFANA_TOKEN": "test-only"}): + client = app.Client(c) + with patch( + "subprocess.run", + return_value=subprocess.CompletedProcess([], 1, "", "private-error-sentinel"), + ): + with self.assertRaises(app.SetupError) as caught: + client.kube("monitoring", "get", "secret", "auth") + self.assertNotIn("private-error-sentinel", str(caught.exception)) + + def test_redirects_are_rejected_without_echoing_location(self) -> None: + c = config() + with patch.dict(os.environ, {"GRAFANA_TOKEN": "test-only"}): + client = app.Client(c) + error = urllib.error.HTTPError( + "https://redirect.invalid", + 302, + "redirect", + {"Location": "https://other.invalid"}, + io.BytesIO(b""), + ) + with patch("urllib.request.OpenerDirector.open", side_effect=error): + with self.assertRaisesRegex(app.SetupError, "HTTP 302"): + client.api("/test") + # The handler handed to urllib refuses even same-origin redirects. + with patch("urllib.request.build_opener") as build: + build.return_value.open.side_effect = error + with self.assertRaises(app.SetupError): + client.api("/test") + handler = build.call_args.args[0] + self.assertIsNone( + handler.redirect_request(None, None, 302, "redirect", {}, "https://other.invalid") + ) + + def test_missing_target_or_inline_credentials_are_rejected(self) -> None: + for update in ("context", "inline", "remote-http"): + c = config() + if update == "context": + c["kubernetes"]["context"] = "" + elif update == "inline": + c["grafana"]["auth"] = {"password": "do-not-allow"} + else: + c["grafana"]["url"] = "http://cluster.example.invalid" + with tempfile.TemporaryDirectory() as temp: + p = Path(temp) / "config.json" + p.write_text(json.dumps(c)) + with self.assertRaises(app.SetupError): + app.load_config(p) + + +class ReviewedPublicationTests(unittest.TestCase): + def test_default_example_requires_no_loki_and_render_is_offline(self) -> None: + c = app.load_config(DASHBOARD_DIR / "example.config.json") + with patch.object(app, "Client", side_effect=AssertionError("Network during render")): + dashboards, _, _ = app.render(c) + self.assertEqual(set(dashboards), {"metrics"}) + self.assertNotIn(c["dashboard"]["logs_uid"], json.dumps(dashboards)) + + def test_health_queries_use_configured_job_not_service(self) -> None: + c = config() + c["workload"]["metric_job_regex"] = "selected-metrics-job" + dashboards, _, _ = app.render(c) + for key in ("panel-1", "panel-39"): + query = dashboards["metrics"]["elements"][key]["spec"]["data"]["spec"]["queries"][0] + expr = query["spec"]["query"]["spec"]["expr"] + self.assertIn('job=~"selected-metrics-job"', expr) + self.assertNotIn("service=~", expr) + + def test_new_tier_panels_keep_scope_units_and_missing_data_rules(self) -> None: + c = config() + c["dashboard"]["counter_lookback"] = "2m" + dashboards, _, _ = app.render(c) + metrics = dashboards["metrics"] + panels = [p for key, p in metrics["elements"].items() if key.startswith("observability-")] + self.assertEqual(len(panels), 14) + for p in panels: + spec = p["spec"] + custom = spec["vizConfig"]["spec"]["fieldConfig"]["defaults"]["custom"] + self.assertEqual(custom["stacking"]["mode"], "none") + for query in spec["data"]["spec"]["queries"]: + expr = query["spec"]["query"]["spec"]["expr"] + self.assertIn('namespace="$namespace"', expr) + self.assertIn('job=~"inference-.*-metrics"', expr) + self.assertIn("pod=~", expr) + self.assertNotIn("or vector(0)", expr) + if "Tier restore / eviction turnover" in spec["title"]: + self.assertIn("rate(", expr) + self.assertIn("[2m]", expr) + self.assertIn("group_left", expr) + if "Sampled restore latency" in spec["title"]: + self.assertIn("[5m]", expr) + self.assertIn("source_cache_level,destination_cache_level", expr) + for row in metrics["layout"]["spec"]["rows"]: + for item in row["spec"]["layout"]["spec"]["items"]: + key = item["spec"]["element"]["name"] + if key.startswith("observability-"): + rule = item["spec"]["conditionalRendering"]["spec"] + self.assertEqual(rule["items"][0]["kind"], "ConditionalRenderingData") + + def test_configmap_native_edit_after_checkpoint_stops_final_put(self) -> None: + c = config() + c["grafana"]["publication"] = "configmap-v2" + dashboards, cm, scrape = app.render(c) + client = FakeClient(c) + original = client.api + reads = {} + + def api( + path: str, + payload: dict | None = None, + method: str | None = None, + missing_ok: bool = False, + ) -> dict | None: + value = original(path, payload, method, missing_ok) + if payload is None and client.cm is not None and value is not None: + uid = path.rsplit("/", 1)[-1] + reads[uid] = reads.get(uid, 0) + 1 + if reads[uid] == 2: + client.saved[uid]["metadata"]["resourceVersion"] = "edited" + client.saved[uid]["spec"]["title"] = "Concurrent user edit" + value = deepcopy(client.saved[uid]) + return value + + client.api = api + with self.assertRaisesRegex(app.SetupError, "changed during validation"): + app.deploy(c, dashboards, cm, scrape, client) + self.assertFalse( + any( + k == "api" and method and "dryRun=All" not in path + for k, path, method in client.calls + ) + ) + self.assertTrue( + any(v["spec"].get("title") == "Concurrent user edit" for v in client.saved.values()) + ) + + def test_configmap_metadata_edit_is_not_overwritten(self) -> None: + c = config() + c["grafana"]["publication"] = "configmap-v2" + dashboards, cm, scrape = app.render(c) + client = FakeClient(c) + client.cm = {"metadata": {"resourceVersion": "1"}, "data": {"unrelated.json": "keep"}} + original = client.kube + reads = 0 + + def kube( + namespace: str, *args: str, payload: dict | None = None, missing_ok: bool = False + ) -> dict | None: + nonlocal reads + if args[0] == "get": + reads += 1 + if reads == 2: + client.cm["metadata"]["resourceVersion"] = "2" + return original(namespace, *args, payload=payload, missing_ok=missing_ok) + + client.kube = kube + with self.assertRaisesRegex(app.SetupError, "ConfigMap changed"): + app.deploy(c, dashboards, cm, scrape, client) + self.assertEqual(client.cm["data"], {"unrelated.json": "keep"}) + + def test_api_version_can_be_selected_explicitly(self) -> None: + c = config() + for version in ("v2", "v2beta1"): + c["grafana"]["api_version"] = version + self.assertIn("/" + version + "/", app.endpoint(c)) + self.assertEqual( + app.resource(c, "example", {}, None)["apiVersion"], + "dashboard.grafana.app/" + version, + ) + + def test_invalid_bool_and_non_re2_groups_are_rejected(self) -> None: + for field in ("bool", "regex"): + c = config() + if field == "bool": + c["scrape"]["enabled"] = "false" + else: + c["workload"]["prefill_pod_regex"] = "(?=prefix).*" + with tempfile.TemporaryDirectory() as temp: + p = Path(temp) / "config.json" + p.write_text(json.dumps(c)) + with self.assertRaises(app.SetupError): + app.load_config(p) + + def test_secret_reference_is_a_resource_name_not_a_cli_option(self) -> None: + c = config() + c["grafana"]["auth"] = {"secret": "-A", "username_key": "user", "password_key": "pass"} + with tempfile.TemporaryDirectory() as temp: + p = Path(temp) / "config.json" + p.write_text(json.dumps(c)) + with self.assertRaisesRegex(app.SetupError, "Secret name"): + app.load_config(p) + + def test_failed_invocation_drops_stale_success_receipt(self) -> None: + with tempfile.TemporaryDirectory() as temp: + p = Path(temp) / "receipt.json" + p.write_text('{"verified":true}') + with patch.object(app, "load_config", side_effect=app.SetupError("invalid config")): + with self.assertRaises(app.SetupError): + app.main(["render", "--config", "missing.json", "--output", temp]) + self.assertFalse(p.exists()) + + +if __name__ == "__main__": + unittest.main() From 7e4450ab29165cb64249e7ca5263280bcd463717 Mon Sep 17 00:00:00 2001 From: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:39:49 +0000 Subject: [PATCH 4/5] [None][refactor] Defer KV transfer outcome transport to follow-up Signed-off-by: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> --- .../performance-metrics-collection.md | 25 ++++++++++ .../_torch/pyexecutor/iteration_stats.py | 5 -- tensorrt_llm/_torch/pyexecutor/py_executor.py | 6 +-- tensorrt_llm/metrics/perf.py | 7 ++- .../executor/test_stats_serializer.py | 8 +--- tests/unittest/metrics/test_perf.py | 47 +++++++++++++++++-- 6 files changed, 77 insertions(+), 21 deletions(-) diff --git a/docs/source/commands/trtllm-serve/performance-metrics-collection.md b/docs/source/commands/trtllm-serve/performance-metrics-collection.md index 3968852d06ba..953cf1a186f9 100644 --- a/docs/source/commands/trtllm-serve/performance-metrics-collection.md +++ b/docs/source/commands/trtllm-serve/performance-metrics-collection.md @@ -58,6 +58,31 @@ collection/timing settings. The existing device-step profiler timing policy is preserved. CPU and end-to-end overhead of this refactor have not been benchmarked. +## Transport cost and KV utilization semantics + +Worker serialization parses native JSON into a dictionary, partitions it into +owned immutable categories, and encodes the existing JSON transport shape. The +consumer still parses that JSON. Parsing the native JSON and encoding/decoding the +wire payload were already required; partitioning, recursive freezing and mapping +adaptation add CPU work. The worker's freeze runs on the transport thread. Source +snapshot freezing and loop-record construction also run on the executor thread. + +The pending overhead benchmark must cover both source collection and transport: +compare baseline and refactored CPU time/allocations per record and transport +throughput with request diagnostics disabled/enabled, varying request counts, +KV snapshot sizes and rank counts. Separately compare logger-only, Prometheus-only +and combined outputs in an inference workload. No measured overhead claim is +made before those checks run. + +KV utilization prefers an explicit `usedNumBlocks` value, including measured zero. +For basic snapshots without that field, it uses `maxNumBlocks - freeNumBlocks`. +Missing or non-positive capacity produces unavailable utilization (`N/A` in the +iteration log). This preserves the logger-only free-block calculation and its +zero-capacity `N/A` behavior. Prometheus retains its existing requirement for +explicit used blocks and positive capacity; it does not infer a utilization series +from a free-only snapshot. Reclaimable cached prefixes can remain in available +blocks, so utilization alone does not measure conversation-cache retention. + ## Create a Grafana dashboard The [Grafana setup example](https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/serve/grafana) diff --git a/tensorrt_llm/_torch/pyexecutor/iteration_stats.py b/tensorrt_llm/_torch/pyexecutor/iteration_stats.py index afd2af184d88..dc2af46d6044 100644 --- a/tensorrt_llm/_torch/pyexecutor/iteration_stats.py +++ b/tensorrt_llm/_torch/pyexecutor/iteration_stats.py @@ -30,7 +30,6 @@ class IterationStatsRecord(NamedTuple): prev_device_step_time_ms: float | None = None scheduler_mode: Literal["overlap", "non_overlap"] | None = None gpu_forward_time_ms: float | None = None - kv_transfer_outcomes: tuple[int, int, int] | None = None executor_loop: dict[str, object] | None = None @classmethod @@ -70,8 +69,4 @@ def to_dict(self, *, tp_rank: int | None = None) -> dict: for name, value in optional_fields: if value is not None: stats[name] = value - if self.kv_transfer_outcomes is not None: - stats["kvTransferOutcomes"] = dict( - zip(("success", "error", "timeout"), self.kv_transfer_outcomes) - ) return stats diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 6fdfaa19ef4e..1972a4b6af9f 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -3558,9 +3558,7 @@ def _append_iter_stats(self, attention_dp_rank: Optional[int] = None, host_step_time_ms: Optional[float] = None, prev_device_step_time_ms: Optional[float] = None, - gpu_forward_time_ms: Optional[float] = None, - kv_transfer_outcomes: Optional[tuple[int, int, - int]] = None): + gpu_forward_time_ms: Optional[float] = None): """Append one iteration's finalized stats to the export buffer. The normal Attention-DP path fans out rank-local rows before calling @@ -3586,7 +3584,6 @@ def _append_iter_stats(self, gpu_forward_time_ms: Batch-matched GPU forward time captured by the events surrounding this batch's ``_forward_step``. Surfaces as ``gpuForwardTimeMS`` in the /metrics JSON. - kv_transfer_outcomes: Rank-local cumulative request success/error/timeout counts. """ # Non-ADP appends immediately, so the latest KV stats belong to this # IterationStats. ADP appends later and passes the saved iter-matched @@ -3630,7 +3627,6 @@ def _append_iter_stats(self, prev_device_step_time_ms=prev_device_step_time_ms, scheduler_mode=scheduler_mode, gpu_forward_time_ms=gpu_forward_time_ms, - kv_transfer_outcomes=kv_transfer_outcomes, executor_loop=loop_metrics) tp_size = getattr(self.dist, "tp_size", 1) diff --git a/tensorrt_llm/metrics/perf.py b/tensorrt_llm/metrics/perf.py index 6f0d6380e053..02fb230ca08e 100644 --- a/tensorrt_llm/metrics/perf.py +++ b/tensorrt_llm/metrics/perf.py @@ -37,7 +37,6 @@ "kvCacheMigrationTimings", "kvCacheMigrationTimingsSkipped", "kvCacheLevelTiers", - "kvTransferOutcomes", "iterSuspendedRequests", "iterResumedRequests", "iterDiskPrefetchBlocks", @@ -77,7 +76,11 @@ def _thaw(value: MetricValue) -> object: def kv_cache_utilization(stats: Mapping) -> float | None: - """Return non-reclaimable utilization; missing/zero capacity is unavailable.""" + """Return non-reclaimable utilization, preferring an explicit used count. + + Basic snapshots can supply only capacity and free blocks. Missing or + non-positive capacity is unavailable, rather than measured zero utilization. + """ capacity = stats.get("maxNumBlocks", 0) if not isinstance(capacity, (int, float)) or capacity <= 0: return None diff --git a/tests/unittest/executor/test_stats_serializer.py b/tests/unittest/executor/test_stats_serializer.py index 7ae82644fbdd..bf952fb5a41a 100644 --- a/tests/unittest/executor/test_stats_serializer.py +++ b/tests/unittest/executor/test_stats_serializer.py @@ -136,7 +136,6 @@ def test_named_record_preserves_all_wire_fields(self): prev_device_step_time_ms=1.25, scheduler_mode="overlap", gpu_forward_time_ms=4.25, - kv_transfer_outcomes=(3, 2, 1), executor_loop=loop, ) expected = { @@ -147,13 +146,12 @@ def test_named_record_preserves_all_wire_fields(self): "prevDeviceStepTimeMS": 1.25, "schedulerMode": "overlap", "gpuForwardTimeMS": 4.25, - "kvTransferOutcomes": {"success": 3, "error": 2, "timeout": 1}, "executorLoop": loop, } assert json.loads(BaseWorker._stats_serializer(record)) == expected assert json.loads(BaseWorker._stats_serializer(tuple(record))) == expected - @pytest.mark.parametrize("field_count", range(2, 12)) + @pytest.mark.parametrize("field_count", range(2, 11)) def test_legacy_tuple_prefixes_and_future_trailing_fields(self, field_count): iter_stats = _make_mock_iteration_stats() loop = {"kind": "loop", "per_iter": {"iter": 9}} @@ -166,7 +164,6 @@ def test_legacy_tuple_prefixes_and_future_trailing_fields(self, field_count): 1.25, "overlap", 4.25, - (3, 2, 1), loop, "future field", )[:field_count] @@ -179,8 +176,7 @@ def test_legacy_tuple_prefixes_and_future_trailing_fields(self, field_count): (6, "prevDeviceStepTimeMS", 1.25), (7, "schedulerMode", "overlap"), (8, "gpuForwardTimeMS", 4.25), - (9, "kvTransferOutcomes", {"success": 3, "error": 2, "timeout": 1}), - (10, "executorLoop", loop), + (9, "executorLoop", loop), ): if field_count >= required_fields: expected[key] = value diff --git a/tests/unittest/metrics/test_perf.py b/tests/unittest/metrics/test_perf.py index f7987743eb14..8890dc80b336 100644 --- a/tests/unittest/metrics/test_perf.py +++ b/tests/unittest/metrics/test_perf.py @@ -228,7 +228,6 @@ def test_iteration_record_preserves_transport_fields(tp_rank: int | None) -> Non prev_device_step_time_ms=1.0, scheduler_mode="overlap", gpu_forward_time_ms=2.0, - kv_transfer_outcomes=(0, 1, 2), executor_loop=loop, ) expected = { @@ -239,7 +238,6 @@ def test_iteration_record_preserves_transport_fields(tp_rank: int | None) -> Non "prevDeviceStepTimeMS": 1.0, "gpuForwardTimeMS": 2.0, "schedulerMode": "overlap", - "kvTransferOutcomes": {"success": 0, "error": 1, "timeout": 2}, "executorLoop": loop, } if tp_rank is None: @@ -250,7 +248,7 @@ def test_iteration_record_preserves_transport_fields(tp_rank: int | None) -> Non assert json.loads(PerfMetricsCollector.collect_iteration(expected).iteration_json()) == expected -@pytest.mark.parametrize("width", range(1, 11)) +@pytest.mark.parametrize("width", range(1, len(IterationStatsRecord._fields) + 1)) def test_iteration_record_accepts_historical_tuple_widths(width: int) -> None: stats = Mock() stats.to_json_str.return_value = '{"iter": 7}' @@ -260,3 +258,46 @@ def test_iteration_record_accepts_historical_tuple_widths(width: int) -> None: assert entry.to_dict(tp_rank=3) == {"iter": 7, "rank": 3} assert IterationStatsRecord.from_tuple(entry) is entry assert IterationStatsRecord.from_tuple(legacy + (None,) * 10).to_dict() == entry.to_dict() + + +@pytest.mark.parametrize( + ("stats", "expected"), + [ + ({"maxNumBlocks": 10, "usedNumBlocks": 3, "freeNumBlocks": 1}, 0.3), + ({"maxNumBlocks": 10, "usedNumBlocks": 0, "freeNumBlocks": 1}, 0.0), + ({"maxNumBlocks": 10, "freeNumBlocks": 4}, 0.6), + ({"maxNumBlocks": 10, "freeNumBlocks": 10}, 0.0), + ({"maxNumBlocks": 0, "usedNumBlocks": 0}, None), + ({"maxNumBlocks": -1, "usedNumBlocks": 0}, None), + ({"maxNumBlocks": 10}, None), + ({"usedNumBlocks": 3}, None), + ({}, None), + ], +) +def test_kv_utilization_used_precedence_fallback_and_missing_capacity( + stats: dict, expected: float | None +) -> None: + assert kv_cache_utilization(stats) == expected + + +@pytest.mark.parametrize("capacity", [None, 0]) +def test_logger_reports_unavailable_kv_as_na(capacity: int | None) -> None: + collector = PerfMetricsCollector() + if capacity is not None: + collector.collect_rank( + {"kvCacheStats": {"maxNumBlocks": capacity, "usedNumBlocks": 0}}, + capture_iter=0, + ) + messages = [] + LoggerExposure(messages.append).emit(collector.collect_loop(_loop())) + assert "kv_cache_util = N/A" in messages[0] + + +def test_prometheus_missing_used_count_does_not_infer_utilization() -> None: + prom = _prometheus() + prom.emit( + PerfMetricsCollector.collect_iteration( + {"kvCacheStats": {"maxNumBlocks": 10, "freeNumBlocks": 4}} + ) + ) + assert prom.kv_cache_utilization.collect()[0].samples == [] From 076096d673d5b00b6068a2de6142413026d272e7 Mon Sep 17 00:00:00 2001 From: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> Date: Thu, 12 Nov 2026 04:59:56 +0000 Subject: [PATCH 5/5] [None][fix] Address metrics refactor and dashboard review findings Signed-off-by: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> --- examples/serve/grafana/dashboard.py | 10 +- examples/serve/grafana/templates/logs.json | 26 ++-- tensorrt_llm/_torch/pyexecutor/py_executor.py | 1 + tensorrt_llm/metrics/perf.py | 4 + .../test_executor_resource_rebind.py | 2 + .../pyexecutor/test_profile_endpoints.py | 86 ++++++++++++ .../executor/test_stats_serializer.py | 2 +- .../unittest/grpc/openengine/test_metrics.py | 21 +++ .../metrics/test_grafana_dashboard.py | 131 ++++++++++++++++++ tests/unittest/metrics/test_perf.py | 11 +- 10 files changed, 276 insertions(+), 18 deletions(-) diff --git a/examples/serve/grafana/dashboard.py b/examples/serve/grafana/dashboard.py index 4a39868615fb..9d08e62198ad 100644 --- a/examples/serve/grafana/dashboard.py +++ b/examples/serve/grafana/dashboard.py @@ -448,7 +448,7 @@ def replace(text: str) -> str: result = {} for name in ("metrics", "logs"): - if name == "logs" and not d.get("include_logs", True): + if name == "logs" and not d.get("include_logs", False): continue spec = walk( json.loads((HERE / "templates" / f"{name}.json").read_text(encoding="utf-8")), replace @@ -488,7 +488,7 @@ def replace(text: str) -> str: s.update(x=x, y=y) x += s["width"] height = max(height, s["height"]) - if not d.get("include_logs", True): + if not d.get("include_logs", False): spec["links"] = [x for x in spec["links"] if x.get("url") != "/d/" + d["logs_uid"]] if name == "logs": add_unique_sessions(spec, c) @@ -704,7 +704,11 @@ def redirect_request( return None try: - with urllib.request.build_opener(NoRedirect()).open(req, timeout=60) as response: + handlers = [NoRedirect()] + if urllib.parse.urlsplit(req.full_url).hostname in ("localhost", "127.0.0.1", "::1"): + # Loopback credentials must stay local even when proxy variables are set. + handlers.append(urllib.request.ProxyHandler({})) + with urllib.request.build_opener(*handlers).open(req, timeout=60) as response: return json.load(response) except urllib.error.HTTPError as error: if missing_ok and error.code == 404: diff --git a/examples/serve/grafana/templates/logs.json b/examples/serve/grafana/templates/logs.json index 28e9c049d7db..3501492c3e82 100644 --- a/examples/serve/grafana/templates/logs.json +++ b/examples/serve/grafana/templates/logs.json @@ -138,7 +138,7 @@ "name": "@@LOKI_UID@@" }, "spec": { - "expr": "{namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"@@ENGINE_PODS@@\",pod=~\".*\",container=~\"${container:regex}\"} |~ \"$search\"", + "expr": "{namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"(${worker:regex})|(@@FRONTEND_PODS@@)\",container=~\"${container:regex}\"} |~ \"$search\"", "legendFormat": "" } }, @@ -363,7 +363,7 @@ }, "spec": { "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*adp_dummy_ctx_tokens = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}} context" + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}} context" } }, "refId": "A", @@ -382,7 +382,7 @@ }, "spec": { "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*adp_dummy_gen_tokens = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}} generation" + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}} generation" } }, "refId": "B", @@ -452,7 +452,7 @@ }, "spec": { "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_fetched = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}}" + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}}" } }, "refId": "A", @@ -523,7 +523,7 @@ "spec": { "editorMode": "code", "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_active = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}}", "queryType": "range" } }, @@ -596,7 +596,7 @@ "spec": { "editorMode": "code", "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_unscheduled = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}}", "queryType": "range" } }, @@ -669,7 +669,7 @@ "spec": { "editorMode": "code", "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${prefill_worker:regex}\",pod=~\"@@PREFILL_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_dummy = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}}", "queryType": "range" } }, @@ -741,7 +741,7 @@ }, "spec": { "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*adp_dummy_ctx_tokens = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}} context" + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}} context" } }, "refId": "A", @@ -760,7 +760,7 @@ }, "spec": { "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*adp_dummy_gen_tokens = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}} generation" + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}} generation" } }, "refId": "B", @@ -830,7 +830,7 @@ }, "spec": { "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_fetched = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}}" + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}}" } }, "refId": "A", @@ -901,7 +901,7 @@ "spec": { "editorMode": "code", "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_active = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}}", "queryType": "range" } }, @@ -974,7 +974,7 @@ "spec": { "editorMode": "code", "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_unscheduled = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}}", "queryType": "range" } }, @@ -1047,7 +1047,7 @@ "spec": { "editorMode": "code", "expr": "last_over_time({namespace=\"$namespace\",job=\"@@LOG_JOB@@\",pod=~\"${role:raw}\",pod=~\"${decode_worker:regex}\",pod=~\"@@DECODE_PODS@@\",container=\"@@ENGINE_CONTAINER@@\"} |= \"iter = \" | regexp \"global_rank = (?P[0-9]+),.*num_dummy = (?P[0-9]+([.][0-9]+)?)\" | rank=~\"${mpi_rank:raw}\" | unwrap value | __error__=\"\" [$__interval]) by(@@LOG_ROLE_LABEL@@,pod,rank)", - "legendFormat": "{{pod}} / {{role}} / rank {{rank}}", + "legendFormat": "{{pod}} / {{@@LOG_ROLE_LABEL@@}} / rank {{rank}}", "queryType": "range" } }, diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 1972a4b6af9f..a293ed12b545 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -1640,6 +1640,7 @@ def rebind_resources(self, resources: PyExecutorResources) -> None: raise RuntimeError( "Cannot rebind an executor after an execution failure") self._resources_released = False + self._perf_metrics.reset() try: self._initialize_execution(resources) except BaseException: diff --git a/tensorrt_llm/metrics/perf.py b/tensorrt_llm/metrics/perf.py index 02fb230ca08e..b94dd7220e41 100644 --- a/tensorrt_llm/metrics/perf.py +++ b/tensorrt_llm/metrics/perf.py @@ -172,6 +172,10 @@ class PerfMetricsCollector: """ def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + """Begin a resource phase without invalidating exposure references.""" self.latest_loop: PerfMetricsRecord | None = None self.latest_rank: Mapping[MetricKey, MetricValue] = _empty_fields() self._previous_fetched = 0 diff --git a/tests/unittest/_torch/pyexecutor/test_executor_resource_rebind.py b/tests/unittest/_torch/pyexecutor/test_executor_resource_rebind.py index 68c004a38498..8d0423472757 100644 --- a/tests/unittest/_torch/pyexecutor/test_executor_resource_rebind.py +++ b/tests/unittest/_torch/pyexecutor/test_executor_resource_rebind.py @@ -20,6 +20,7 @@ from tensorrt_llm._torch.pyexecutor.resource_manager import ResourceManager, ResourceManagerType from tensorrt_llm._torch.pyexecutor.sampler import AsyncWorkerMixin from tensorrt_llm._torch.pyexecutor.seq_slot_manager import SeqSlotManager +from tensorrt_llm.metrics.perf import PerfMetricsCollector pytestmark = pytest.mark.cpu_only @@ -47,6 +48,7 @@ def release_engine_resources(*, release_model_owned_graphs): transceiver = SimpleNamespace(shutdown=lambda: events.append("unregister")) ex = object.__new__(PyExecutor) ex._closed = False + ex._perf_metrics = PerfMetricsCollector() ex.enable_attention_dp = False ex._kv_connector_failed = False ex._resources_released = False diff --git a/tests/unittest/_torch/pyexecutor/test_profile_endpoints.py b/tests/unittest/_torch/pyexecutor/test_profile_endpoints.py index cf7aa8b3e16a..0f3290d44d60 100644 --- a/tests/unittest/_torch/pyexecutor/test_profile_endpoints.py +++ b/tests/unittest/_torch/pyexecutor/test_profile_endpoints.py @@ -21,7 +21,12 @@ """ import threading +from types import SimpleNamespace +from unittest.mock import Mock +import pytest + +from tensorrt_llm._torch.pyexecutor import profiling from tensorrt_llm._torch.pyexecutor.profiling import PyExecutorProfileManager from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor from tensorrt_llm.metrics.perf import PerfMetricsCollector @@ -395,3 +400,84 @@ def test_apply_profile_stop_config_reports_cancel_vs_schedule(tmp_path): assert executor._apply_profile_stop_config() is False assert executor._runtime_profile_pending_stop_iter == 8 assert 8 in executor.profile_stop_iters + + +@pytest.mark.cpu_only +def test_rebind_resets_phase_metrics_without_stale_profile_reference() -> None: + executor = _bare_executor() + executor._closed = False + executor._resources_released = True + executor._event_loop_error = None + executor._fatal_error = None + collector = executor._perf_metrics + collector.collect_rank({"kvCacheStats": {"maxNumBlocks": 10}}, capture_iter=8) + values = {"iter": 8, "rank": 0, "globalRank": 0, "numFetchedRequestsTotal": 20} + collector.collect_loop(values) + executor._initialize_execution = Mock() + executor.rebind_resources(Mock()) + assert executor._profile_manager.metrics is collector + assert collector.latest_loop is None + assert not collector.latest_rank + new = collector.collect_loop({**values, "iter": 0, "numFetchedRequestsTotal": 2}) + assert new.per_iter["numFetchedRequests"] == 2 + assert new.per_iter["loopSequence"] == 0 + assert not new.per_rank + executor._initialize_execution.assert_called_once() + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("rank,stats_enabled", [(0, False), (1, False), (0, True), (1, True)]) +def test_profile_step_collects_with_source_and_logging_gates( + monkeypatch: pytest.MonkeyPatch, rank: int, stats_enabled: bool +) -> None: + executor = _bare_executor() + executor.dist = SimpleNamespace(rank=rank, pp_size=1) + executor.model_engine = SimpleNamespace(model=None, iter_states="decode") + executor.print_log = True + executor.enable_iter_perf_stats = stats_enabled + executor.is_warmup = False + executor.num_scheduled_requests = 0 + executor.num_fetch_requests_cur_rank = 2 + executor.num_fetch_requests = 2 + executor.active_requests = [] + executor.num_dummy_requests = 0 + executor.num_unscheduled_requests = 0 + executor._iter_adp_dummy_ctx_tokens = 0 + executor._iter_adp_dummy_gen_tokens = 0 + reader = Mock(return_value=SimpleNamespace(max_num_blocks=10, free_num_blocks=4)) + executor.kv_cache_manager = SimpleNamespace(get_kv_cache_stats=reader) + monkeypatch.setenv(profiling.PROFILE_LOG_RANKS_ENV_VAR_NAME, "0") + monkeypatch.delenv(profiling.PROFILE_START_STOP_ENV_VAR_NAME, raising=False) + monkeypatch.setattr(profiling.torch.cuda, "Event", Mock()) + monkeypatch.setattr(profiling, "get_calibrator", Mock(return_value=Mock())) + monkeypatch.setattr( + profiling, + "runtime_hooks_of", + Mock(return_value=SimpleNamespace(iteration_timing_one_event=False)), + ) + messages = [] + executor._profile_manager.metrics_logger._write = messages.append + with executor._profile_manager.profile_step() as step: + step() + executor.iter_counter = 1 + step() + assert executor._perf_metrics.latest_loop is not None + if stats_enabled: + import json + + from tensorrt_llm.executor.base_worker import BaseWorker + + executor.enable_attention_dp = False + executor.disable_overlap_scheduler = True + executor._latest_kv_iter_stats = None + executor.stats = [] + executor.stats_lock = threading.Lock() + executor.max_stats_len = 10 + native_stats = Mock() + native_stats.to_json_str.return_value = '{"iter": 0}' + executor._append_iter_stats(native_stats) + wire = json.loads(BaseWorker._stats_serializer(executor.stats[0])) + assert wire["executorLoop"]["per_iter"]["iter"] == 1 + assert wire["iter"] == 0 + assert reader.call_count == int(rank == 0 and not stats_enabled) + assert any("iter = " in message for message in messages) == (rank == 0) diff --git a/tests/unittest/executor/test_stats_serializer.py b/tests/unittest/executor/test_stats_serializer.py index bf952fb5a41a..52330828bfcc 100644 --- a/tests/unittest/executor/test_stats_serializer.py +++ b/tests/unittest/executor/test_stats_serializer.py @@ -369,7 +369,7 @@ def test_loop_observation_keeps_its_identity_separate_from_the_batch(self): }, } result = BaseWorker._stats_serializer( - (iter_stats, None, None, None, 12.5, None, "overlap", 4.25, None, loop) + (iter_stats, None, None, None, 12.5, None, "overlap", 4.25, loop) ) data = json.loads(result) assert data["executorLoop"] == loop diff --git a/tests/unittest/grpc/openengine/test_metrics.py b/tests/unittest/grpc/openengine/test_metrics.py index 3e97cc47d4b5..196691a30592 100644 --- a/tests/unittest/grpc/openengine/test_metrics.py +++ b/tests/unittest/grpc/openengine/test_metrics.py @@ -660,3 +660,24 @@ def _http_enablement() -> None: if mode != "attached": assert not owned_path.exists() assert os.environ.get("PROMETHEUS_MULTIPROC_DIR") == previous + + +@pytest.mark.cpu_only +def test_collect_once_exposes_iteration_and_rank(): + import threading + from unittest.mock import Mock + + from tensorrt_llm.grpc.openengine.metrics import OpenEngineMetrics + + owner = SimpleNamespace( + _llm=SimpleNamespace(get_stats=Mock(return_value=[{"iter": 7, "attentionDpRank": 2}])), + _stopping=threading.Event(), + collector=Mock(), + _record_error=Mock(), + ) + OpenEngineMetrics._collect_once(owner) + owner.collector.emit.assert_called_once() + record = owner.collector.emit.call_args.args[0] + assert record.per_iter["iter"] == 7 + assert record.per_iter["attentionDpRank"] == 2 + owner._record_error.assert_not_called() diff --git a/tests/unittest/metrics/test_grafana_dashboard.py b/tests/unittest/metrics/test_grafana_dashboard.py index 6b65a762f404..6aaa72b6f267 100644 --- a/tests/unittest/metrics/test_grafana_dashboard.py +++ b/tests/unittest/metrics/test_grafana_dashboard.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Portability, safe publication, and authentication boundary tests.""" +import base64 import hashlib import importlib.util import io @@ -11,6 +12,7 @@ import tempfile import unittest import urllib.error +import urllib.response from copy import deepcopy from pathlib import Path from unittest.mock import patch @@ -516,3 +518,132 @@ def test_failed_invocation_drops_stale_success_receipt(self) -> None: if __name__ == "__main__": unittest.main() + + +class CodeRabbitRegressionTests(unittest.TestCase): + def test_omitted_logs_flag_renders_metrics_without_logs_navigation(self) -> None: + c = config() + del c["dashboard"]["include_logs"] + dashboards, cm, _ = app.render(c) + self.assertEqual(set(dashboards), {"metrics"}) + self.assertNotIn("logs.json", cm["data"]) + self.assertFalse( + any( + link.get("url") == "/d/" + c["dashboard"]["logs_uid"] + for link in dashboards["metrics"]["links"] + ) + ) + + def test_loopback_credentials_bypass_environment_proxy(self) -> None: + c = config() + c["grafana"]["auth"] = {"token_env": "DASHBOARD_TEST_TOKEN"} + for host in ("127.0.0.1", "localhost", "[::1]"): + with self.subTest(host=host): + c["grafana"]["url"] = "http://" + host + ":3000" + requests = [] + + def respond(handler: object, request: object) -> object: + requests.append(request) + response = urllib.response.addinfourl( + io.BytesIO(b"{}"), {}, request.full_url, 200 + ) + response.msg = "OK" + return response + + with ( + patch.dict(os.environ, {"DASHBOARD_TEST_TOKEN": "test-token"}), + patch( + "urllib.request.getproxies", + return_value={"http": "http://proxy.invalid:3128"}, + ), + patch("urllib.request.HTTPHandler.http_open", respond), + ): + self.assertEqual(app.Client(c).api("/health"), {}) + self.assertEqual(len(requests), 1) + self.assertFalse(requests[0].has_proxy()) + self.assertNotEqual(requests[0].host, "proxy.invalid:3128") + self.assertEqual(requests[0].get_header("Authorization"), "Bearer test-token") + + def test_custom_role_legend_and_instance_worker_scope(self) -> None: + c = config() + c["workload"]["log_role_label"] = "worker_role" + dashboards, _, _ = app.render(c) + logs = dashboards["logs"] + instance = logs["elements"]["panel-15"]["spec"] + query = instance["data"]["spec"]["queries"][0]["spec"]["query"]["spec"]["expr"] + self.assertIn("${worker:regex}", query) + self.assertIn(c["workload"]["frontend_pod_regex"], query) + self.assertEqual(query.count("pod=~"), 1) + self.assertIn("${container:regex}", query) + text = json.dumps(logs) + self.assertGreaterEqual(text.count("{{worker_role}}"), 12) + self.assertNotIn("{{role}}", text) + + def test_finalize_only_accepts_matching_checkpoint_without_applying_it(self) -> None: + c = config() + c["grafana"]["publication"] = "configmap-v2" + dashboards, cm, scrape = app.render(c) + client = FakeClient(c) + client.kube(c["grafana"]["namespace"], "apply", payload=cm) + client.calls.clear() + app.deploy(c, dashboards, cm, scrape, client, finalize_only=True) + self.assertFalse( + any( + call[0] == "kube" and call[2][0] == "apply" and "--dry-run=server" not in call[2] + for call in client.calls + ) + ) + + def test_finalize_only_rejects_changed_checkpoint(self) -> None: + c = config() + c["grafana"]["publication"] = "configmap-v2" + dashboards, cm, scrape = app.render(c) + client = FakeClient(c) + client.cm = deepcopy(cm) + client.cm["metadata"]["resourceVersion"] = "1" + client.cm["data"]["metrics.json"] = "{}" + with self.assertRaisesRegex( + app.SetupError, "Resume requires the exact rendered ConfigMap checkpoint" + ): + app.deploy(c, dashboards, cm, scrape, client, finalize_only=True) + + def test_file_provisioner_timeout_is_bounded(self) -> None: + c = config() + c["grafana"]["publication"] = "configmap-v2" + dashboards, cm, scrape = app.render(c) + client = FakeClient(c) + client.cm = deepcopy(cm) + client.cm["metadata"]["resourceVersion"] = "1" + with ( + patch.object(app.time, "monotonic", side_effect=[0, 100]), + patch.object( + app.time, "sleep", side_effect=AssertionError("Should fail before sleeping") + ), + self.assertRaisesRegex(app.SetupError, "File provisioner did not acknowledge"), + ): + app.deploy(c, dashboards, cm, scrape, client, finalize_only=True) + + def test_secret_auth_decodes_keys_and_rejects_missing_values(self) -> None: + c = config() + c["grafana"]["auth"] = { + "secret": "grafana-auth", + "username_key": "user", + "password_key": "password", + } + with patch.object( + app.Client, + "kube", + return_value={ + "data": { + "user": base64.b64encode(b"admin").decode(), + "password": base64.b64encode(b"test-password").decode(), + } + }, + ): + expected = "Basic " + base64.b64encode(b"admin:test-password").decode() + self.assertEqual(app.Client(c).authorization, expected) + with ( + patch.object(app.Client, "kube", return_value={"data": {}}), + self.assertRaisesRegex(app.SetupError, "missing valid authentication keys"), + ): + app.Client(c) diff --git a/tests/unittest/metrics/test_perf.py b/tests/unittest/metrics/test_perf.py index 8890dc80b336..a3c7e51c7b8f 100644 --- a/tests/unittest/metrics/test_perf.py +++ b/tests/unittest/metrics/test_perf.py @@ -153,7 +153,16 @@ def test_request_candidates_keep_enum_keys_and_diagnostic_requests_are_not_compl prom.emit(diagnostic) assert prom.counter_request_success.collect()[0].samples == [] prom.emit(request) - assert prom.histogram_queue_time_request.labels(**prom.labels)._sum.get() == 0 + histogram = prom.histogram_queue_time_request.labels(**prom.labels) + assert histogram._sum.get() == 0 + assert ( + next( + sample.value + for sample in histogram.collect()[0].samples + if sample.name.endswith("_count") + ) + == 1 + ) assert ( prom.counter_request_success.labels(**prom.labels, finished_reason="stop")._value.get() == 1 )