diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json index 596e180410bb..a96a253b2f4e 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json @@ -26,6 +26,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which does not carry a golden across GPU architectures (nvbugs/6655359); this gate runs on B200 and measures inside threshold against this sm103 cut, so the media is not re-cut.", "sha256": "1fd9b0ab24de130f593056a32a7b8555fafbbf073b76c29a4633504870b1dad0" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json index 92ad6e4ba1f3..de6d8e2072e1 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json @@ -25,6 +25,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which does not carry a golden across GPU architectures (nvbugs/6655359); this gate runs on B200 and measures inside threshold against this sm103 cut, so the media is not re-cut.", "sha256": "3f7c9b958807356ced2de1734e301dc837fa0b095f8fed1e29da764993926046" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json index 3b053b41ad68..0ca249f097e3 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json @@ -25,6 +25,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which does not carry a golden across GPU architectures (nvbugs/6655359); this gate runs on B200 and measures inside threshold against this sm103 cut, so the media is not re-cut.", "sha256": "0ec80b5c906ae576deedf8fb48c55edd0c78203138608f12d0c439da11ab6f10" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json index 55d4cf08a551..66c8583e6b59 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json @@ -34,6 +34,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which does not carry a golden across GPU architectures (nvbugs/6655359); this gate runs on B200 and measures inside threshold against this sm103 cut, so the media is not re-cut.", "sha256": "32c080983eb8d94d1da5d21378ea87912dadf018f00f15123a33f000640341ad" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json index c09d9cc280a0..14862aa5f121 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json @@ -24,6 +24,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which does not carry a golden across GPU architectures (nvbugs/6655359); this gate runs on B200 and measures inside threshold against this sm103 cut, so the media is not re-cut.", "sha256": "035f3e764e6a36159071178a2d7be6ec3cabc60899736099ff89a59c037e15e1" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json index ee6014644db1..77f626af3084 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json @@ -21,9 +21,9 @@ "measured_lpips_at_creation": 0.0, "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", "diffusers_version": "0.39.0", - "torch_version": "2.12.0+cu130", + "torch_version": "2.12.0a0+5aff3928d8.nv26.05", "tensorrt_llm_version": "1.3.0rc25", - "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", - "sha256": "980849ba2f1ff1101c0dce2ac8897172212a614f23c3fb0cc6acbd970dd42976" + "tensorrt_llm_commit": "a6f6eedff20c32a6dbe53f6012d1aa008c93de62", + "environment": "NGC PyTorch 26.05 container; NVIDIA B200 (sm100), the arch this gate runs on (tests/integration/test_lists/test-db/l0_b200.yml). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which does not carry a golden across GPU architectures, so this media replaces a B300 (sm103) cut that drifted against this gate (nvbugs/6655359).", + "sha256": "efcf19b5ca1f0161debf1d81c0c9a2485eb0747fd426606dac56909d60f5f420" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json index 636b8c3b484f..79713f0d4f10 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json @@ -24,9 +24,9 @@ "measured_lpips_at_creation": 0.0, "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", "diffusers_version": "0.39.0", - "torch_version": "2.12.0+cu130", + "torch_version": "2.12.0a0+5aff3928d8.nv26.05", "tensorrt_llm_version": "1.3.0rc25", - "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", - "sha256": "728c9bed1c25bf7de2b727949cc8c985f0f5bf03fd78e19847f4bcc7edeb450b" + "tensorrt_llm_commit": "a6f6eedff20c32a6dbe53f6012d1aa008c93de62", + "environment": "NGC PyTorch 26.05 container; NVIDIA B200 (sm100), the arch this gate runs on (tests/integration/test_lists/test-db/l0_b200.yml). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which does not carry a golden across GPU architectures, so this media replaces a B300 (sm103) cut that drifted against this gate (nvbugs/6655359).", + "sha256": "b6cbb5dfc887a7bb1d8adede7c4a2e7cec57e4152447b49e711ed988cd427663" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip index c298c535a41b..fc1689545a1d 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:33da5841fe5594a739240d048a0c6139c58e7e8e04d76cce1fb0cce968ead133 -size 30674088 +oid sha256:5a373ff1e5ddfd6a3550157c6998c8f74c91c2004fe92db873cfa8f486271268 +size 30665081 diff --git a/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py b/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py index 14bc3c16a24d..0ce43cdec802 100644 --- a/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py +++ b/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py @@ -411,7 +411,7 @@ def _cleanup_cuda(): @contextlib.contextmanager def _lpips_pinned_fp32_matmul_precision() -> Iterator[None]: - """Pin fp32-matmul arithmetic so LPIPS goldens are portable across hosts. + """Pin fp32-matmul arithmetic so LPIPS goldens survive a torch-stack change. NGC PyTorch containers default matmul TF32 on (``float32_matmul_precision == "high"``); PyPI torch defaults it off (``"highest"``). A model with fp32 @@ -420,9 +420,18 @@ def _lpips_pinned_fp32_matmul_precision() -> Iterator[None]: ``transformer_cosmos3.py``) therefore produces a different trajectory under each default, and a golden cut under one fails under the other -- measured LPIPS-to-golden moved 0.132 -> 0.054 from this single flag. Pin "highest" - (IEEE fp32, measured bit-stable across torch 2.11/2.12 and B200/B300), and - pin cuDNN TF32 to its universal default so the second knob cannot drift. - bf16 compute -- all of the heavy kernels -- is unaffected by either knob. + (IEEE fp32, measured bit-stable across torch 2.11/2.12), and pin cuDNN TF32 + to its universal default so the second knob cannot drift. bf16 compute -- + all of the heavy kernels -- is unaffected by either knob. + + The pin does NOT make a golden portable across GPU architectures, not even + across steppings of one family: it fixes the arithmetic each kernel uses, + not the reduction order a kernel picks for a given SM count. A B300 (sm103) + cut of the Cosmos3-Nano goldens drifted against the B200 gate under this + same pin by an amount that grew with temporal extent (0.15 at 189 frames + versus a 0.05 gate, while the 1-frame sibling from that cut stayed green) -- + reduction-order drift, which no knob pins. Cut each golden on the GPU its + gate runs on (see ``test-db/l0_*.yml``); nvbugs/6655359. Applied per generation path rather than from ``_lpips_deterministic_algorithms``: that helper also wraps generation for diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index f152d21d3d94..98a9ed4fe35d 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -114,8 +114,6 @@ examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_accep examples/test_ad_speculative_decoding.py::test_nemotron_mtp_model_with_weights SKIP (https://nvbugs/6630699) examples/test_ray.py::test_ray_disaggregated_serving_python[tp2] SKIP (https://nvbugs/6601574) examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6572800) -examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden SKIP (https://nvbugs/6655359) -examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_v2v_lpips_against_golden SKIP (https://nvbugs/6655359) examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux1-nvfp4] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux2-nvfp4] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_glm.py::test_glm_image_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6644450)