From 7582d93ae8af4f284d7a0666433ee993b171f3ac Mon Sep 17 00:00:00 2001 From: Conduction Release Bot Date: Sat, 8 Aug 2026 15:30:59 +0200 Subject: [PATCH] test(gate-28): acceptance evidence for #178, and the assertions it was missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #178 (gate-28 fails every PHP-free PR in a PHP repo) was already FIXED by #182 on 2026-08-06, released in v1.5.2 and v1.6.0. The issue was simply never closed. Measured, not assumed — all three states behave correctly on main today. This commit adds no runtime change; it adds the acceptance evidence and closes three real gaps in what the suites actually asserted. WHAT WAS NOT BEING ASSERTED 1. The suite never ran --require-full-coverage, so it proved "gate-28 prints NOT APPLICABLE" and INFERRED "the run is not failed". Those are different claims and have come apart before (#164 defaulted the flag TRUE in the workflow while every local suite ran without it). 2. Nothing exercised the _fail 28 branch end to end. A licence MISMATCH is the gate's actual purpose; only the weaker "no file carried a tag" control existed. The helper->log->wc -l->_fail wiring was untested. 3. check_license_triangle.py's declared_file_count() had NO test in either direction, despite being the single number the runner's PASS-vs-structural decision rests on, and nothing pinned the stdout/stderr split it travels over. A TRAP FOUND WHILE WRITING THIS The coverage-failure branch lives inside `if [ "${_FAILED}" -eq 0 ]`. Any other failing gate suppresses the sentence entirely, so "the coverage sentence is absent" passes automatically on a fixture that trips an unrelated gate — including on a fully reverted runner. The first draft wrote its Vue file to src/App.vue, which gate-38 fails; both runs exited 1, not 98, and both new assertions were measuring gate-38. Fixtures are now otherwise green (src/components/Widget.vue), the pass case reads the POSITIVE "ALL N APPLICABLE GATES GREEN" sentence, and the control uses a composer.json with no license field rather than untagged PHP (which also fails gate-1, making the coverage branch unreachable). MUTATION-CHECKED — every new assertion shown capable of failing: M1 empty diff scope -> structural (pre-#182) -> 2 new shell assertions RED, status 98 M2 structural -> na (gate stops failing) -> 2 new shell assertions RED M3 _fail 28 disabled -> 2 new shell assertions RED (drift became PASS) M4 declared_file_count -> len(files) -> 3 new python tests RED M5 count printed to stdout -> 4 new python tests RED; clean repo turns FAIL M6 empty scope invents a finding + a count -> 3 new python tests RED M7 declared_file_count -> 0 -> 5 new python tests RED; PASS became structural Counts: shell 5 -> 11 assertions, python 19 -> 29 tests. Both appended to, never overwritten. Full helper suite: 27 passed, 0 failed, 2 quarantined. No runtime file is touched: the diff is the two test files only. Closes #178 --- .../lib/test_check_license_triangle.py | 144 +++++++++++++ .../lib/test_gate_license_triangle_scope.sh | 196 ++++++++++++++++++ 2 files changed, 340 insertions(+) diff --git a/hydra-gates/scripts/lib/test_check_license_triangle.py b/hydra-gates/scripts/lib/test_check_license_triangle.py index 883afbdae..9ad0b7f8a 100644 --- a/hydra-gates/scripts/lib/test_check_license_triangle.py +++ b/hydra-gates/scripts/lib/test_check_license_triangle.py @@ -223,5 +223,149 @@ def test_a_wrong_licence_is_reported_in_every_declaration_shape(self): self.assertEqual(_rules(_scan(body)), ["license-triangle-drift"]) +# ========================================================================== +# APPENDED for ConductionNL/.github#178 — the helper's HALF of the +# applicability decision. +# +# #178: gate-28 failed every PHP-free PR in a PHP repo, because scope came +# from the DIFF while "subject matter exists" was judged from the REPO. #182 +# fixed the runner. Nothing here tested the helper side of that contract, and +# the contract is narrow enough to break silently: +# +# * The runner decides PASS vs `structural` on ONE number — the +# `declared_files=N` line this module writes. `declared_file_count()` had +# no test at all, in either direction. +# * It reaches the runner over a STREAM SPLIT: findings on stdout, the count +# on stderr, separated by `2>&1 >>log |`. A helper that printed the count +# to stdout instead would leave every assertion above green, put the count +# line into the findings log, and make `wc -l` report a finding that does +# not exist — a gate that FAILS a clean repo. The `main()` tests below pin +# which stream each half arrives on. +# +# The empty-list cases are the ones #178 turns on: with zero files in scope +# the helper must report zero compared files and — the half that is easy to +# get wrong — ZERO FINDINGS. A helper that manufactured a finding from an +# empty scope would move the false red from the coverage block into the +# failure count, which is worse, not better. +# ========================================================================== +import contextlib # noqa: E402 +import io # noqa: E402 + + +class DeclaredFileCount(unittest.TestCase): + """`declared_file_count()` is the runner's entire PASS-vs-structural + evidence. Both directions, because a function that always returns 0 and a + function that always returns len(files) each satisfy half of this.""" + + @staticmethod + def _write(d: str, name: str, body: str) -> str: + p = Path(d) / name + p.write_text(body, encoding="utf-8") + return str(p) + + def test_empty_scope_counts_zero(self): + # #178's case: nothing was in scope, so nothing was compared. + self.assertEqual(clt.declared_file_count([]), 0) + + def test_empty_scope_produces_no_findings(self): + # ...and must not invent one. An empty scope is not a defect. + self.assertEqual(clt.scan_files([], EUPL), []) + + def test_a_tagged_file_counts_one(self): + with tempfile.TemporaryDirectory() as d: + self.assertEqual( + clt.declared_file_count([self._write(d, "A.php", CLEAN)]), 1) + + def test_an_untagged_file_counts_zero(self): + # The `structural` signal: files WERE in scope and declared nothing. + # This is the state that must keep failing, and the one an + # over-broad #178 fix would have swallowed. + with tempfile.TemporaryDirectory() as d: + body = " tuple[int, str, str]: + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + rc = clt.main(argv) + return rc, out.getvalue(), err.getvalue() + + def test_empty_scope_prints_no_stdout_and_a_zero_count_on_stderr(self): + rc, out, err = self._main(["check_license_triangle.py", EUPL]) + self.assertEqual(rc, 0) + # Every stdout line is counted as a finding by the runner. One stray + # line here fails a repo that has nothing wrong with it. + self.assertEqual(out, "") + self.assertIn("declared_files=0", err) + + def test_the_count_is_on_stderr_not_stdout(self): + with tempfile.TemporaryDirectory() as d: + p = Path(d) / "A.php" + p.write_text(CLEAN, encoding="utf-8") + rc, out, err = self._main( + ["check_license_triangle.py", EUPL, str(p)]) + self.assertEqual(rc, 0) + self.assertEqual(out, "") # clean file: no findings + self.assertIn("declared_files=1", err) + self.assertNotIn("declared_files=", out) # the stream split holds + + def test_a_finding_goes_to_stdout_while_the_count_stays_on_stderr(self): + with tempfile.TemporaryDirectory() as d: + p = Path(d) / "A.php" + p.write_text(DRIFTED, encoding="utf-8") + rc, out, err = self._main( + ["check_license_triangle.py", EUPL, str(p)]) + self.assertEqual(rc, 0) + self.assertEqual(len(out.strip().splitlines()), 1) + self.assertIn("license-triangle-drift", out) + self.assertIn("declared_files=1", err) + self.assertNotIn("license-triangle-drift", err) + + def test_the_return_code_is_a_status_not_a_finding_count(self): + # #209: gate-19 returned its finding COUNT as an exit status, so 266 + # findings reported as 10 and 256 would have reported PASS. This helper + # must never adopt that shape — the count travels as text on stderr. + with tempfile.TemporaryDirectory() as d: + files = [] + for i in range(3): + p = Path(d) / f"D{i}.php" + p.write_text(DRIFTED, encoding="utf-8") + files.append(str(p)) + rc, out, err = self._main( + ["check_license_triangle.py", EUPL, *files]) + self.assertEqual(len(out.strip().splitlines()), 3) # 3 findings... + self.assertEqual(rc, 0) # ...status still 0 + self.assertIn("declared_files=3", err) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/hydra-gates/scripts/lib/test_gate_license_triangle_scope.sh b/hydra-gates/scripts/lib/test_gate_license_triangle_scope.sh index 27adbe31c..94bd07ba8 100755 --- a/hydra-gates/scripts/lib/test_gate_license_triangle_scope.sh +++ b/hydra-gates/scripts/lib/test_gate_license_triangle_scope.sh @@ -221,6 +221,202 @@ if _run; then _expect NOTAPPLICABLE "no lib/ at all: NOT APPLICABLE, never PASS (#172)" fi +# =========================================================================== +# APPENDED for ConductionNL/.github#178 — the ACCEPTANCE evidence the suite +# above only approximated. +# +# #178 was filed against hermiq#159, a frontend + spec + docs PR, and #182 +# fixed it. The four cases above lock that fix in — verified by mutation: with +# the `na` branch removed, cases 1 and 2 both go red. So why more? +# +# Because of what case 1 does NOT run. #178's claim is not "the gate prints +# NOT APPLICABLE"; it is "**the run is not FAILED** by the coverage +# requirement". The suite above never passes `--require-full-coverage`, so it +# proves the first and infers the second. Those came apart once already: #164 +# made the flag default TRUE in the WORKFLOW while every local suite ran +# without it, which is precisely how a gate can be green in this file and red +# in CI. The pair below runs the flag and reads the runner's own verdict line. +# +# And case 2's control is the WEAKER of the two true positives. "No file +# carried a tag" is `structural`; a file that carries the WRONG licence is a +# `_fail` — a different branch, a different exit path, and the only one that +# is the gate's actual purpose. Nothing end-to-end exercised it: the drift +# rules are unit-tested in test_check_license_triangle.py, but the wiring from +# helper stdout -> _lt_log -> `wc -l` -> `_fail 28` was not. A helper that +# printed its findings to stderr would keep every unit test green and turn +# this gate into a no-op. +# =========================================================================== + +# _run_fullcov — same as _run, but with the flag CI actually sets. The runner +# exits 98 on incomplete coverage; that status is deliberately NOT the +# assertion. Statuses in this package have carried finding COUNTS rather than +# verdicts (#209: gate-19 returned 266 findings as exit status 10, and 256 +# would have read as PASS), so the assertion reads the runner's own sentence +# on stdout. The status is captured only to be reported alongside it. +_OUT_FC="" +_STATUS_FC="" +_run_fullcov() { + local _logdir + _logdir="$(mktemp -d "${TMPDIR:-/tmp}/hydra-gate-test.XXXXXXXX")" + _OUT_FC="$(HYDRA_GATE_LOG_DIR="${_logdir}" bash "${RUNNER}" \ + --app-dir "${_REPO}" --base "${_BASE}" --scope-to-diff \ + --require-full-coverage 2>&1)" + _STATUS_FC=$? + rm -rf "${_logdir}" + if ! printf '%s' "${_OUT_FC}" | grep -q '^\[hydra-gates\] COVERAGE:'; then + _bad "full-coverage run in ${_REPO} ABORTED before the summary — not a result" + printf '%s\n' "${_OUT_FC}" | tail -15 | sed 's/^/ /' + return 1 + fi + return 0 +} + +# _expect_gate28 +# A FAIL-aware sibling of _expect that reads a caller-supplied capture, so the +# full-coverage runs can be asserted without disturbing _OUT. +_expect_gate28() { + local _want="$1" _out="$2" _desc="$3" _line _pattern + _line="$(printf '%s' "${_out}" | grep -E '^\[gate-28\] ' | head -1)" + if [ -z "${_line}" ]; then + _bad "${_desc} — gate-28 emitted NO line at all (a silent gate is the #147 shape)" + return + fi + case "${_want}" in + PASS) _pattern=': PASS' ;; + NOTAPPLICABLE) _pattern=': NOT APPLICABLE' ;; + structural) _pattern=': SKIPPED \(structural\)' ;; + FAIL) _pattern=': FAIL' ;; + *) + _bad "${_desc} — test bug: unknown expectation '${_want}'" + return ;; + esac + if printf '%s' "${_line}" | grep -qE "${_pattern}"; then + _ok "${_desc}" + else + _bad "${_desc} — got: ${_line}" + fi +} + +# The runner's own sentence for "this run is being failed for coverage". Only +# printed when --require-full-coverage is set AND a gate did not run. +_COVFAIL_LINE='treating incomplete coverage as failure' + +_MISMATCHED_PHP='. Both full-coverage runs exited 1, not 98, and BOTH new +# assertions were measuring gate-38's failure rather than gate-28's verdict — +# the absence-assertion would have been dead. The file is written to +# `src/components/Widget.vue` for that reason, and the assertion reads the +# runner's positive "ALL … APPLICABLE GATES GREEN" sentence rather than the +# absence of a negative one: a sentence that must be PRESENT cannot be +# satisfied by a run that died earlier. +# +# For the same reason the control at case 6 CANNOT use untagged PHP: an +# untagged file also fails gate-1 (spdx-headers), so `_FAILED` is never 0 and +# the coverage branch is unreachable. It uses the other structural route +# instead — a composer.json with no `license` field, with a correctly tagged +# lib PHP file in the diff — which reaches `structural` with every other gate +# green. + +# --- 5. #178 AS REPORTED: a frontend + docs diff (.vue + .md), no PHP at all, +# in a repo that HAS lib/ and a composer licence. Case 1 used a +# workflow file; this is the shape the issue was actually filed on. +_mkrepo scope-frontend-only yes +mkdir -p "${_REPO}/src/components" +printf '\n' > "${_REPO}/src/components/Widget.vue" +git -C "${_REPO}" add -A >/dev/null 2>&1 +git -C "${_REPO}" commit -qm "add a frontend component" >/dev/null 2>&1 +_BASE="$(git -C "${_REPO}" rev-parse HEAD)" +printf '\n' > "${_REPO}/src/components/Widget.vue" +printf '# changed\n' > "${_REPO}/README.md" +git -C "${_REPO}" add -A >/dev/null 2>&1 +git -C "${_REPO}" commit -qm "frontend + docs only, no PHP" >/dev/null 2>&1 +if _run_fullcov; then + _expect_gate28 NOTAPPLICABLE "${_OUT_FC}" \ + "#178: .vue + .md diff in a tagged repo is NOT APPLICABLE, not structural" + # THE ASSERTION #178 IS ACTUALLY ABOUT: the run must not be failed for it. + # Read the POSITIVE verdict, per the note above. + if printf '%s' "${_OUT_FC}" | grep -qE '^\[hydra-gates\] ALL [0-9]+ APPLICABLE GATES GREEN'; then + _ok "#178: the PHP-free diff run is GREEN under --require-full-coverage (status ${_STATUS_FC})" + else + _bad "#178: PHP-free diff did not end green under --require-full-coverage (status ${_STATUS_FC}) — the false red is back, or the fixture trips an unrelated gate and this assertion is measuring nothing" + fi +fi + +# --- 6. THE CONTROL FOR 5, and the one that makes it mean something. Same +# flag, same runner, an otherwise-identical fixture: a run whose ONLY +# problem is a structural gate-28 must still be failed for coverage. +# Without this, assertion 5 would also pass on a runner that had simply +# stopped enforcing coverage at all. +# +# The licence field is dropped in a BASE-ADVANCING commit, not in the +# diff under test. Putting composer.json in the diff makes gate-4 +# (composer-audit) applicable, and it fails on a fixture with no vendor +# tree — `_FAILED` becomes 1 and the coverage sentence is unreachable +# again. The diff must contain the lib PHP file and nothing else. +_mkrepo scope-nolicense-fullcov yes +printf '{\n "name": "conduction/fixture"\n}\n' > "${_REPO}/composer.json" +git -C "${_REPO}" add -A >/dev/null 2>&1 +git -C "${_REPO}" commit -qm "drop the composer license field" >/dev/null 2>&1 +_BASE="$(git -C "${_REPO}" rev-parse HEAD)" +printf '%s\n// touched\n' "${_LICENSED_PHP}" > "${_REPO}/lib/Tagged.php" +git -C "${_REPO}" add -A >/dev/null 2>&1 +git -C "${_REPO}" commit -qm "touch the tagged lib PHP file" >/dev/null 2>&1 +if _run_fullcov; then + _expect_gate28 structural "${_OUT_FC}" \ + "control: composer.json with no license is still structural under --require-full-coverage" + if printf '%s' "${_OUT_FC}" | grep -qF "${_COVFAIL_LINE}"; then + _ok "control: --require-full-coverage DID fail the run on a real gap (status ${_STATUS_FC})" + else + _bad "control: --require-full-coverage did NOT fail a run whose only gap is gate-28 (status ${_STATUS_FC}) — the coverage requirement is inert here, so assertion 5 proves nothing" + fi +fi + +# --- 7. THE GATE'S ACTUAL PURPOSE, end to end. A file declaring a licence +# that composer.json does not — the `_fail 28` branch, which no test +# above reaches. This is the assertion that would catch the helper +# writing findings anywhere but stdout. +_mkrepo scope-mismatch yes +printf '%s' "${_MISMATCHED_PHP}" > "${_REPO}/lib/Drifted.php" +git -C "${_REPO}" add -A >/dev/null 2>&1 +git -C "${_REPO}" commit -qm "lib PHP declaring AGPL against an EUPL composer.json" >/dev/null 2>&1 +if _run; then + _expect_gate28 FAIL "${_OUT}" \ + "licence MISMATCH in scope (@license AGPL vs composer EUPL): gate-28 FAILS" + # ...and it must fail for the RIGHT reason. A gate that failed here for any + # other cause would satisfy the line above while comparing nothing. + if printf '%s' "${_OUT}" | grep -qE '^\[gate-28\].*@license != composer\.json'; then + _ok "mismatch failure names the licence comparison as its cause" + else + _bad "mismatch failed, but not with the licence-comparison reason — cause unverified" + fi +fi + echo echo "== summary ==" echo " passed: ${_pass_n}"