diff --git a/conformance/src/trace_tests/report.py b/conformance/src/trace_tests/report.py index 14e18d97..c2c64673 100644 --- a/conformance/src/trace_tests/report.py +++ b/conformance/src/trace_tests/report.py @@ -30,6 +30,7 @@ from trace_tests.result import Finding, Status, finding_from_row, finding_row __all__ = [ + "LEVEL_MODULES", "LEVEL_NAMES", "ReportData", "badge_svg", @@ -49,7 +50,7 @@ #: Modules introduced at each level, for the per-level breakdown. LEVEL_MODULES = { - 0: ("TR-ENV", "TR-SIG", "TR-POL"), + 0: ("TR-ENV", "TR-SIG", "TR-POL", "TR-APR"), 1: ("TR-RTE", "TR-SCA"), 2: ("TR-TXN", "TR-ANC"), } @@ -64,8 +65,10 @@ def record_digest(record: dict[str, Any]) -> str: specification, section 0; getting it backwards here would be harmless but confusing, so the choice is stated rather than left implicit. """ - canonical = json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True) - return "sha256:" + hashlib.sha256(canonical.encode("ascii")).hexdigest() + encoded = json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode( + "ascii" + ) + return "sha256:" + hashlib.sha256(encoded).hexdigest() @dataclass(frozen=True) @@ -142,11 +145,13 @@ def _assemble( library_version: str | None, generated_at: str, ) -> ReportData: - trace = record.get("trace", record) + raw_trace = record.get("trace", record) + trace = raw_trace if isinstance(raw_trace, dict) else {} + digest = record_digest(record) return ReportData( record_path=record_path, record_format=record_format, - digest=record_digest(record), + digest=digest, suite_version=suite_version, library_version=library_version, generated_at=generated_at, diff --git a/conformance/tests/test_report.py b/conformance/tests/test_report.py index 374f6dfa..50e2464b 100644 --- a/conformance/tests/test_report.py +++ b/conformance/tests/test_report.py @@ -7,6 +7,7 @@ from __future__ import annotations +import hashlib import json import re @@ -14,12 +15,12 @@ from trace_tests import report as report_mod from trace_tests.result import Finding, Status -from trace_tests.runner import run +from trace_tests.runner import _LEVEL_MODULES, run from .conftest import _build_signed_cmcp_record, load_vector #: Wide enough that a fixture's age never decides a test about reporting. -_MAX_AGE = 10 ** 9 +_MAX_AGE = 10**9 def _passing_record(): @@ -215,3 +216,84 @@ def test_unanchored_record_says_so_rather_than_showing_blank() -> None: record.pop("transparency", None) out = report_mod.to_html(_build(record)) assert "unanchored" in out + + +# --- level module parity, malformed trace guard, and canonical digest ------ + + +def test_level_modules_include_tr_apr_at_level_zero_and_match_runner() -> None: + assert report_mod.LEVEL_MODULES[0] == ("TR-ENV", "TR-SIG", "TR-POL", "TR-APR") + assert report_mod.LEVEL_MODULES[0] == _LEVEL_MODULES[0] + cumulative = { + lv: tuple(m for k in range(lv + 1) for m in report_mod.LEVEL_MODULES[k]) + for lv in sorted(report_mod.LEVEL_MODULES) + } + assert cumulative == dict(_LEVEL_MODULES) + + +@pytest.mark.parametrize( + "bad_trace", + ["not-a-dict", 123, None, [1, 2], True], + ids=["str", "int", "none", "list", "bool"], +) +def test_build_tolerates_non_dict_trace_field(bad_trace: object) -> None: + try: + data = report_mod.build( + record={"trace": bad_trace}, + record_path="r.json", + record_format="cmcp-runtime", + results_by_level={ + 0: {"TR-ENV": [Finding("TR-ENV-001", Status.FAIL, "trace is not an object")]} + }, + suite_version="0.5.0", + library_version=None, + generated_at="now", + ) + except AttributeError: + pytest.fail(f"report.build crashed on non-dict trace value {bad_trace!r}") + assert data.transparency is None + assert "unanchored" in report_mod.to_html(data) + + +def test_record_digest_matches_diagnostic_serialization() -> None: + record = load_vector("valid_level0.json") + expected = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode( + "ascii" + ) + ).hexdigest() + ) + assert report_mod.record_digest(record) == expected + + +@pytest.mark.parametrize( + "bad_value", + [float("nan"), float("inf"), float("-inf"), 1.5, 10**20, -(10**20)], + ids=["nan", "inf", "neg-inf", "float", "above-safe-int", "below-safe-int"], +) +def test_record_digest_remains_diagnostic_for_out_of_profile_values(bad_value: object) -> None: + record = load_vector("valid_level0.json") + record["data_class"] = bad_value + expected = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode( + "ascii" + ) + ).hexdigest() + ) + assert report_mod.record_digest(record) == expected + data = report_mod.build( + record=record, + record_path="r.json", + record_format="trace", + results_by_level={0: {"TR-ENV": [Finding("TR-ENV-001", Status.FAIL, "boundary record")]}}, + suite_version="0.5.0", + library_version=None, + generated_at="now", + ) + assert data.digest == expected + assert json.loads(report_mod.to_json(data))["record"]["digest"] == expected + assert expected in report_mod.to_html(data) diff --git a/tests/test_conformance_report_parity.py b/tests/test_conformance_report_parity.py new file mode 100644 index 00000000..cb8c4e7a --- /dev/null +++ b/tests/test_conformance_report_parity.py @@ -0,0 +1,108 @@ +"""Regression tests for conformance report Level 0 modules, trace guard, and digest (#557).""" + +from __future__ import annotations + +import hashlib +import importlib +import json +import sys +from pathlib import Path +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +CONFORMANCE_SRC = ROOT / "conformance" / "src" +if str(CONFORMANCE_SRC) not in sys.path: + sys.path.insert(0, str(CONFORMANCE_SRC)) + +inclusion_mod = importlib.import_module("trace_tests.inclusion") +report_mod = importlib.import_module("trace_tests.report") +result_mod = importlib.import_module("trace_tests.result") +runner_mod = importlib.import_module("trace_tests.runner") + +Finding = result_mod.Finding +Status = result_mod.Status +_LEVEL_MODULES = runner_mod._LEVEL_MODULES + +VECTORS = ROOT / "conformance" / "tests" / "vectors" + + +def _load_vector(name: str) -> dict[str, Any]: + loaded: dict[str, Any] = json.loads((VECTORS / name).read_text(encoding="utf-8")) + return loaded + + +def test_report_level_modules_include_tr_apr_at_level_zero() -> None: + assert report_mod.LEVEL_MODULES[0] == ("TR-ENV", "TR-SIG", "TR-POL", "TR-APR") + assert report_mod.LEVEL_MODULES[0] == _LEVEL_MODULES[0] + cumulative = { + lv: tuple(m for k in range(lv + 1) for m in report_mod.LEVEL_MODULES[k]) + for lv in sorted(report_mod.LEVEL_MODULES) + } + assert cumulative == dict(_LEVEL_MODULES) + + +@pytest.mark.parametrize( + "bad_trace", + ["not-a-dict", 123, None, [1, 2], True], + ids=["str", "int", "none", "list", "bool"], +) +def test_report_build_guards_non_dict_trace(bad_trace: object) -> None: + try: + data = report_mod.build( + record={"trace": bad_trace}, + record_path="r.json", + record_format="cmcp-runtime", + results_by_level={ + 0: {"TR-ENV": [Finding("TR-ENV-001", Status.FAIL, "trace is not an object")]} + }, + suite_version="0.5.0", + library_version=None, + generated_at="now", + ) + except AttributeError: + pytest.fail(f"report.build crashed on non-dict trace value {bad_trace!r}") + assert data.transparency is None + assert "unanchored" in report_mod.to_html(data) + + +@pytest.mark.parametrize( + "bad_value", + [float("nan"), float("inf"), float("-inf"), 1.5, 10**20, -(10**20)], + ids=["nan", "inf", "neg-inf", "float", "above-safe-int", "below-safe-int"], +) +def test_report_record_digest_remains_diagnostic(bad_value: object) -> None: + record = _load_vector("valid_level0.json") + expected = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode( + "ascii" + ) + ).hexdigest() + ) + assert report_mod.record_digest(record) == expected + + record["data_class"] = bad_value + expected = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode( + "ascii" + ) + ).hexdigest() + ) + assert report_mod.record_digest(record) == expected + data = report_mod.build( + record=record, + record_path="r.json", + record_format="trace", + results_by_level={0: {"TR-ENV": [Finding("TR-ENV-001", Status.FAIL, "boundary record")]}}, + suite_version="0.5.0", + library_version=None, + generated_at="now", + ) + assert data.digest == expected + assert json.loads(report_mod.to_json(data))["record"]["digest"] == expected + assert expected in report_mod.to_html(data)