From 4a5f786bf6ec89edadeda105d31b424c2339f9b9 Mon Sep 17 00:00:00 2001 From: Raghuvar Durga <177662415+drk-ram@users.noreply.github.com> Date: Sat, 17 Oct 2026 12:24:31 -0500 Subject: [PATCH 1/2] conformance(report): fix Level 0 modules, non-dict trace guard, and digest - Include "TR-APR" in report.LEVEL_MODULES[0] to match runner._LEVEL_MODULES[0]. - Guard non-dict record["trace"] values in report._assemble so malformed CMCP records report transparency=None instead of raising AttributeError. - Delegate report.record_digest to inclusion.canonical_claim_bytes so report digests enforce the registry-anchor-v1 section 1 profile, with a fallback in report._assemble when building reports over negative-test boundary records. Closes #557 Signed-off-by: Raghuvar Durga <177662415+drk-ram@users.noreply.github.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- conformance/src/trace_tests/report.py | 19 ++++-- conformance/tests/test_report.py | 60 +++++++++++++++++- tests/test_conformance_report_parity.py | 84 +++++++++++++++++++++++++ 3 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 tests/test_conformance_report_parity.py diff --git a/conformance/src/trace_tests/report.py b/conformance/src/trace_tests/report.py index 14e18d97..cbded9cd 100644 --- a/conformance/src/trace_tests/report.py +++ b/conformance/src/trace_tests/report.py @@ -26,10 +26,12 @@ from typing import Any from trace_tests import accounting +from trace_tests.inclusion import InclusionError, canonical_claim_bytes from trace_tests.modules.unverified import finding_counts_as_level_failure from trace_tests.result import Finding, Status, finding_from_row, finding_row __all__ = [ + "LEVEL_MODULES", "LEVEL_NAMES", "ReportData", "badge_svg", @@ -49,7 +51,7 @@ #: Modules introduced at each level, for the per-level breakdown. LEVEL_MODULES = { - 0: ("TR-ENV", "TR-SIG", "TR-POL"), + 0: ("TR-ENV", "TR-SIG", "TR-POL", "TR-APR"), 1: ("TR-RTE", "TR-SCA"), 2: ("TR-TXN", "TR-ANC"), } @@ -64,8 +66,7 @@ def record_digest(record: dict[str, Any]) -> str: specification, section 0; getting it backwards here would be harmless but confusing, so the choice is stated rather than left implicit. """ - canonical = json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True) - return "sha256:" + hashlib.sha256(canonical.encode("ascii")).hexdigest() + return "sha256:" + hashlib.sha256(canonical_claim_bytes(record)).hexdigest() @dataclass(frozen=True) @@ -142,11 +143,19 @@ def _assemble( library_version: str | None, generated_at: str, ) -> ReportData: - trace = record.get("trace", record) + raw_trace = record.get("trace", record) + trace = raw_trace if isinstance(raw_trace, dict) else {} + try: + digest = record_digest(record) + except InclusionError: + fallback = json.dumps( + record, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=True + ).encode("ascii") + digest = "sha256:" + hashlib.sha256(fallback).hexdigest() return ReportData( record_path=record_path, record_format=record_format, - digest=record_digest(record), + digest=digest, suite_version=suite_version, library_version=library_version, generated_at=generated_at, diff --git a/conformance/tests/test_report.py b/conformance/tests/test_report.py index 374f6dfa..4bab0d30 100644 --- a/conformance/tests/test_report.py +++ b/conformance/tests/test_report.py @@ -7,14 +7,16 @@ from __future__ import annotations +import hashlib import json import re import pytest from trace_tests import report as report_mod +from trace_tests.inclusion import InclusionError, canonical_claim_bytes from trace_tests.result import Finding, Status -from trace_tests.runner import run +from trace_tests.runner import _LEVEL_MODULES, run from .conftest import _build_signed_cmcp_record, load_vector @@ -215,3 +217,59 @@ def test_unanchored_record_says_so_rather_than_showing_blank() -> None: record.pop("transparency", None) out = report_mod.to_html(_build(record)) assert "unanchored" in out + + +# --- level module parity, malformed trace guard, and canonical digest ------ + + +def test_level_modules_include_tr_apr_at_level_zero_and_match_runner() -> None: + assert report_mod.LEVEL_MODULES[0] == ("TR-ENV", "TR-SIG", "TR-POL", "TR-APR") + assert report_mod.LEVEL_MODULES[0] == _LEVEL_MODULES[0] + cumulative = { + lv: tuple(m for k in range(lv + 1) for m in report_mod.LEVEL_MODULES[k]) + for lv in sorted(report_mod.LEVEL_MODULES) + } + assert cumulative == dict(_LEVEL_MODULES) + + +@pytest.mark.parametrize( + "bad_trace", + ["not-a-dict", 123, None, [1, 2], True], + ids=["str", "int", "none", "list", "bool"], +) +def test_build_tolerates_non_dict_trace_field(bad_trace: object) -> None: + try: + data = report_mod.build( + record={"trace": bad_trace}, + record_path="r.json", + record_format="cmcp-runtime", + results_by_level={ + 0: {"TR-ENV": [Finding("TR-ENV-001", Status.FAIL, "trace is not an object")]} + }, + suite_version="0.5.0", + library_version=None, + generated_at="now", + ) + except AttributeError: + pytest.fail(f"report.build crashed on non-dict trace value {bad_trace!r}") + assert data.transparency is None + assert "unanchored" in report_mod.to_html(data) + + +def test_record_digest_matches_canonical_claim_bytes() -> None: + record = load_vector("valid_level0.json") + expected = "sha256:" + hashlib.sha256(canonical_claim_bytes(record)).hexdigest() + assert report_mod.record_digest(record) == expected + + +@pytest.mark.parametrize( + "bad_value", + [float("nan"), float("inf"), float("-inf"), 1.5, 10**20, -(10**20)], + ids=["nan", "inf", "neg-inf", "float", "above-safe-int", "below-safe-int"], +) +def test_record_digest_rejects_values_outside_anchor_leaf_profile(bad_value: object) -> None: + record = load_vector("valid_level0.json") + record["data_class"] = bad_value + with pytest.raises(InclusionError): + report_mod.record_digest(record) + diff --git a/tests/test_conformance_report_parity.py b/tests/test_conformance_report_parity.py new file mode 100644 index 00000000..09b2b4c9 --- /dev/null +++ b/tests/test_conformance_report_parity.py @@ -0,0 +1,84 @@ +"""Regression tests for conformance report Level 0 modules, trace guard, and digest (#557).""" + +from __future__ import annotations + +import hashlib +import importlib +import json +import sys +from pathlib import Path +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +CONFORMANCE_SRC = ROOT / "conformance" / "src" +if str(CONFORMANCE_SRC) not in sys.path: + sys.path.insert(0, str(CONFORMANCE_SRC)) + +inclusion_mod = importlib.import_module("trace_tests.inclusion") +report_mod = importlib.import_module("trace_tests.report") +result_mod = importlib.import_module("trace_tests.result") +runner_mod = importlib.import_module("trace_tests.runner") + +InclusionError = inclusion_mod.InclusionError +canonical_claim_bytes = inclusion_mod.canonical_claim_bytes +Finding = result_mod.Finding +Status = result_mod.Status +_LEVEL_MODULES = runner_mod._LEVEL_MODULES + +VECTORS = ROOT / "conformance" / "tests" / "vectors" + + +def _load_vector(name: str) -> dict[str, Any]: + loaded: dict[str, Any] = json.loads((VECTORS / name).read_text(encoding="utf-8")) + return loaded + + +def test_report_level_modules_include_tr_apr_at_level_zero() -> None: + assert report_mod.LEVEL_MODULES[0] == ("TR-ENV", "TR-SIG", "TR-POL", "TR-APR") + assert report_mod.LEVEL_MODULES[0] == _LEVEL_MODULES[0] + cumulative = { + lv: tuple(m for k in range(lv + 1) for m in report_mod.LEVEL_MODULES[k]) + for lv in sorted(report_mod.LEVEL_MODULES) + } + assert cumulative == dict(_LEVEL_MODULES) + + +@pytest.mark.parametrize( + "bad_trace", + ["not-a-dict", 123, None, [1, 2], True], + ids=["str", "int", "none", "list", "bool"], +) +def test_report_build_guards_non_dict_trace(bad_trace: object) -> None: + try: + data = report_mod.build( + record={"trace": bad_trace}, + record_path="r.json", + record_format="cmcp-runtime", + results_by_level={ + 0: {"TR-ENV": [Finding("TR-ENV-001", Status.FAIL, "trace is not an object")]} + }, + suite_version="0.5.0", + library_version=None, + generated_at="now", + ) + except AttributeError: + pytest.fail(f"report.build crashed on non-dict trace value {bad_trace!r}") + assert data.transparency is None + assert "unanchored" in report_mod.to_html(data) + + +@pytest.mark.parametrize( + "bad_value", + [float("nan"), float("inf"), float("-inf"), 1.5, 10**20, -(10**20)], + ids=["nan", "inf", "neg-inf", "float", "above-safe-int", "below-safe-int"], +) +def test_report_record_digest_enforces_anchor_leaf_profile(bad_value: object) -> None: + record = _load_vector("valid_level0.json") + expected = "sha256:" + hashlib.sha256(canonical_claim_bytes(record)).hexdigest() + assert report_mod.record_digest(record) == expected + + record["data_class"] = bad_value + with pytest.raises(InclusionError): + report_mod.record_digest(record) From 7d16c5b57095895797939464d346bb3036d46634 Mon Sep 17 00:00:00 2001 From: Raghuvar Durga <177662415+drk-ram@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:20:15 -0500 Subject: [PATCH 2/2] fix(report): retain diagnostic digest per maintainer ruling Companion to drk-ram PR #564, preserving original ownership. Signed-off-by: Raghuvar Durga <177662415+drk-ram@users.noreply.github.com> Signed-off-by: Srinivasa Dinakar Jainti Co-authored-by: Srinivasa Consultants <78673153+dinakarjs@users.noreply.github.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- conformance/src/trace_tests/report.py | 14 ++++----- conformance/tests/test_report.py | 40 ++++++++++++++++++++----- tests/test_conformance_report_parity.py | 36 ++++++++++++++++++---- 3 files changed, 67 insertions(+), 23 deletions(-) diff --git a/conformance/src/trace_tests/report.py b/conformance/src/trace_tests/report.py index cbded9cd..c2c64673 100644 --- a/conformance/src/trace_tests/report.py +++ b/conformance/src/trace_tests/report.py @@ -26,7 +26,6 @@ from typing import Any from trace_tests import accounting -from trace_tests.inclusion import InclusionError, canonical_claim_bytes from trace_tests.modules.unverified import finding_counts_as_level_failure from trace_tests.result import Finding, Status, finding_from_row, finding_row @@ -66,7 +65,10 @@ def record_digest(record: dict[str, Any]) -> str: specification, section 0; getting it backwards here would be harmless but confusing, so the choice is stated rather than left implicit. """ - return "sha256:" + hashlib.sha256(canonical_claim_bytes(record)).hexdigest() + encoded = json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode( + "ascii" + ) + return "sha256:" + hashlib.sha256(encoded).hexdigest() @dataclass(frozen=True) @@ -145,13 +147,7 @@ def _assemble( ) -> ReportData: raw_trace = record.get("trace", record) trace = raw_trace if isinstance(raw_trace, dict) else {} - try: - digest = record_digest(record) - except InclusionError: - fallback = json.dumps( - record, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=True - ).encode("ascii") - digest = "sha256:" + hashlib.sha256(fallback).hexdigest() + digest = record_digest(record) return ReportData( record_path=record_path, record_format=record_format, diff --git a/conformance/tests/test_report.py b/conformance/tests/test_report.py index 4bab0d30..50e2464b 100644 --- a/conformance/tests/test_report.py +++ b/conformance/tests/test_report.py @@ -14,14 +14,13 @@ import pytest from trace_tests import report as report_mod -from trace_tests.inclusion import InclusionError, canonical_claim_bytes from trace_tests.result import Finding, Status from trace_tests.runner import _LEVEL_MODULES, run from .conftest import _build_signed_cmcp_record, load_vector #: Wide enough that a fixture's age never decides a test about reporting. -_MAX_AGE = 10 ** 9 +_MAX_AGE = 10**9 def _passing_record(): @@ -256,9 +255,16 @@ def test_build_tolerates_non_dict_trace_field(bad_trace: object) -> None: assert "unanchored" in report_mod.to_html(data) -def test_record_digest_matches_canonical_claim_bytes() -> None: +def test_record_digest_matches_diagnostic_serialization() -> None: record = load_vector("valid_level0.json") - expected = "sha256:" + hashlib.sha256(canonical_claim_bytes(record)).hexdigest() + expected = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode( + "ascii" + ) + ).hexdigest() + ) assert report_mod.record_digest(record) == expected @@ -267,9 +273,27 @@ def test_record_digest_matches_canonical_claim_bytes() -> None: [float("nan"), float("inf"), float("-inf"), 1.5, 10**20, -(10**20)], ids=["nan", "inf", "neg-inf", "float", "above-safe-int", "below-safe-int"], ) -def test_record_digest_rejects_values_outside_anchor_leaf_profile(bad_value: object) -> None: +def test_record_digest_remains_diagnostic_for_out_of_profile_values(bad_value: object) -> None: record = load_vector("valid_level0.json") record["data_class"] = bad_value - with pytest.raises(InclusionError): - report_mod.record_digest(record) - + expected = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode( + "ascii" + ) + ).hexdigest() + ) + assert report_mod.record_digest(record) == expected + data = report_mod.build( + record=record, + record_path="r.json", + record_format="trace", + results_by_level={0: {"TR-ENV": [Finding("TR-ENV-001", Status.FAIL, "boundary record")]}}, + suite_version="0.5.0", + library_version=None, + generated_at="now", + ) + assert data.digest == expected + assert json.loads(report_mod.to_json(data))["record"]["digest"] == expected + assert expected in report_mod.to_html(data) diff --git a/tests/test_conformance_report_parity.py b/tests/test_conformance_report_parity.py index 09b2b4c9..cb8c4e7a 100644 --- a/tests/test_conformance_report_parity.py +++ b/tests/test_conformance_report_parity.py @@ -21,8 +21,6 @@ result_mod = importlib.import_module("trace_tests.result") runner_mod = importlib.import_module("trace_tests.runner") -InclusionError = inclusion_mod.InclusionError -canonical_claim_bytes = inclusion_mod.canonical_claim_bytes Finding = result_mod.Finding Status = result_mod.Status _LEVEL_MODULES = runner_mod._LEVEL_MODULES @@ -74,11 +72,37 @@ def test_report_build_guards_non_dict_trace(bad_trace: object) -> None: [float("nan"), float("inf"), float("-inf"), 1.5, 10**20, -(10**20)], ids=["nan", "inf", "neg-inf", "float", "above-safe-int", "below-safe-int"], ) -def test_report_record_digest_enforces_anchor_leaf_profile(bad_value: object) -> None: +def test_report_record_digest_remains_diagnostic(bad_value: object) -> None: record = _load_vector("valid_level0.json") - expected = "sha256:" + hashlib.sha256(canonical_claim_bytes(record)).hexdigest() + expected = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode( + "ascii" + ) + ).hexdigest() + ) assert report_mod.record_digest(record) == expected record["data_class"] = bad_value - with pytest.raises(InclusionError): - report_mod.record_digest(record) + expected = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode( + "ascii" + ) + ).hexdigest() + ) + assert report_mod.record_digest(record) == expected + data = report_mod.build( + record=record, + record_path="r.json", + record_format="trace", + results_by_level={0: {"TR-ENV": [Finding("TR-ENV-001", Status.FAIL, "boundary record")]}}, + suite_version="0.5.0", + library_version=None, + generated_at="now", + ) + assert data.digest == expected + assert json.loads(report_mod.to_json(data))["record"]["digest"] == expected + assert expected in report_mod.to_html(data)