Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 144 additions & 0 deletions hydra-gates/scripts/lib/test_check_license_triangle.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,5 +223,149 @@ def test_a_wrong_licence_is_reported_in_every_declaration_shape(self):
self.assertEqual(_rules(_scan(body)), ["license-triangle-drift"])


# ==========================================================================
# APPENDED for ConductionNL/.github#178 — the helper's HALF of the
# applicability decision.
#
# #178: gate-28 failed every PHP-free PR in a PHP repo, because scope came
# from the DIFF while "subject matter exists" was judged from the REPO. #182
# fixed the runner. Nothing here tested the helper side of that contract, and
# the contract is narrow enough to break silently:
#
# * The runner decides PASS vs `structural` on ONE number — the
# `declared_files=N` line this module writes. `declared_file_count()` had
# no test at all, in either direction.
# * It reaches the runner over a STREAM SPLIT: findings on stdout, the count
# on stderr, separated by `2>&1 >>log |`. A helper that printed the count
# to stdout instead would leave every assertion above green, put the count
# line into the findings log, and make `wc -l` report a finding that does
# not exist — a gate that FAILS a clean repo. The `main()` tests below pin
# which stream each half arrives on.
#
# The empty-list cases are the ones #178 turns on: with zero files in scope
# the helper must report zero compared files and — the half that is easy to
# get wrong — ZERO FINDINGS. A helper that manufactured a finding from an
# empty scope would move the false red from the coverage block into the
# failure count, which is worse, not better.
# ==========================================================================
import contextlib # noqa: E402
import io # noqa: E402


class DeclaredFileCount(unittest.TestCase):
"""`declared_file_count()` is the runner's entire PASS-vs-structural
evidence. Both directions, because a function that always returns 0 and a
function that always returns len(files) each satisfy half of this."""

@staticmethod
def _write(d: str, name: str, body: str) -> str:
p = Path(d) / name
p.write_text(body, encoding="utf-8")
return str(p)

def test_empty_scope_counts_zero(self):
# #178's case: nothing was in scope, so nothing was compared.
self.assertEqual(clt.declared_file_count([]), 0)

def test_empty_scope_produces_no_findings(self):
# ...and must not invent one. An empty scope is not a defect.
self.assertEqual(clt.scan_files([], EUPL), [])

def test_a_tagged_file_counts_one(self):
with tempfile.TemporaryDirectory() as d:
self.assertEqual(
clt.declared_file_count([self._write(d, "A.php", CLEAN)]), 1)

def test_an_untagged_file_counts_zero(self):
# The `structural` signal: files WERE in scope and declared nothing.
# This is the state that must keep failing, and the one an
# over-broad #178 fix would have swallowed.
with tempfile.TemporaryDirectory() as d:
body = "<?php\nnamespace OCA\\Fixture;\nclass Untagged {}\n"
self.assertEqual(
clt.declared_file_count([self._write(d, "A.php", body)]), 0)

def test_mixed_scope_counts_only_the_declaring_files(self):
with tempfile.TemporaryDirectory() as d:
files = [
self._write(d, "Tagged.php", CLEAN),
self._write(d, "Untagged.php", "<?php\nclass U {}\n"),
self._write(d, "Drifted.php", DRIFTED),
]
# Tagged and Drifted both DECLARE; only Drifted is wrong. The count
# is "how many were compared", not "how many passed" — conflating
# those would make a wholly-drifted diff report structural and skip
# instead of fail.
self.assertEqual(clt.declared_file_count(files), 2)
self.assertEqual(_rules(clt.scan_files(files, EUPL)),
["license-triangle-drift"])

def test_an_unreadable_path_counts_zero_and_does_not_raise(self):
# A path the runner listed but the helper cannot open must not be
# counted as compared — that is the falsely-GREEN shape #172 opened on.
missing = str(Path(tempfile.gettempdir()) / "gate28-does-not-exist.php")
self.assertEqual(clt.declared_file_count([missing]), 0)
self.assertEqual(clt.scan_files([missing], EUPL), [])


class MainStreamContract(unittest.TestCase):
"""The runner parses `declared_files=` off STDERR and treats every STDOUT
line as a finding (`wc -l`). Pin both."""

@staticmethod
def _main(argv: list[str]) -> tuple[int, str, str]:
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
rc = clt.main(argv)
return rc, out.getvalue(), err.getvalue()

def test_empty_scope_prints_no_stdout_and_a_zero_count_on_stderr(self):
rc, out, err = self._main(["check_license_triangle.py", EUPL])
self.assertEqual(rc, 0)
# Every stdout line is counted as a finding by the runner. One stray
# line here fails a repo that has nothing wrong with it.
self.assertEqual(out, "")
self.assertIn("declared_files=0", err)

def test_the_count_is_on_stderr_not_stdout(self):
with tempfile.TemporaryDirectory() as d:
p = Path(d) / "A.php"
p.write_text(CLEAN, encoding="utf-8")
rc, out, err = self._main(
["check_license_triangle.py", EUPL, str(p)])
self.assertEqual(rc, 0)
self.assertEqual(out, "") # clean file: no findings
self.assertIn("declared_files=1", err)
self.assertNotIn("declared_files=", out) # the stream split holds

def test_a_finding_goes_to_stdout_while_the_count_stays_on_stderr(self):
with tempfile.TemporaryDirectory() as d:
p = Path(d) / "A.php"
p.write_text(DRIFTED, encoding="utf-8")
rc, out, err = self._main(
["check_license_triangle.py", EUPL, str(p)])
self.assertEqual(rc, 0)
self.assertEqual(len(out.strip().splitlines()), 1)
self.assertIn("license-triangle-drift", out)
self.assertIn("declared_files=1", err)
self.assertNotIn("license-triangle-drift", err)

def test_the_return_code_is_a_status_not_a_finding_count(self):
# #209: gate-19 returned its finding COUNT as an exit status, so 266
# findings reported as 10 and 256 would have reported PASS. This helper
# must never adopt that shape — the count travels as text on stderr.
with tempfile.TemporaryDirectory() as d:
files = []
for i in range(3):
p = Path(d) / f"D{i}.php"
p.write_text(DRIFTED, encoding="utf-8")
files.append(str(p))
rc, out, err = self._main(
["check_license_triangle.py", EUPL, *files])
self.assertEqual(len(out.strip().splitlines()), 3) # 3 findings...
self.assertEqual(rc, 0) # ...status still 0
self.assertIn("declared_files=3", err)


if __name__ == "__main__":
unittest.main(verbosity=2)
196 changes: 196 additions & 0 deletions hydra-gates/scripts/lib/test_gate_license_triangle_scope.sh
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,202 @@ if _run; then
_expect NOTAPPLICABLE "no lib/ at all: NOT APPLICABLE, never PASS (#172)"
fi

# ===========================================================================
# APPENDED for ConductionNL/.github#178 — the ACCEPTANCE evidence the suite
# above only approximated.
#
# #178 was filed against hermiq#159, a frontend + spec + docs PR, and #182
# fixed it. The four cases above lock that fix in — verified by mutation: with
# the `na` branch removed, cases 1 and 2 both go red. So why more?
#
# Because of what case 1 does NOT run. #178's claim is not "the gate prints
# NOT APPLICABLE"; it is "**the run is not FAILED** by the coverage
# requirement". The suite above never passes `--require-full-coverage`, so it
# proves the first and infers the second. Those came apart once already: #164
# made the flag default TRUE in the WORKFLOW while every local suite ran
# without it, which is precisely how a gate can be green in this file and red
# in CI. The pair below runs the flag and reads the runner's own verdict line.
#
# And case 2's control is the WEAKER of the two true positives. "No file
# carried a tag" is `structural`; a file that carries the WRONG licence is a
# `_fail` — a different branch, a different exit path, and the only one that
# is the gate's actual purpose. Nothing end-to-end exercised it: the drift
# rules are unit-tested in test_check_license_triangle.py, but the wiring from
# helper stdout -> _lt_log -> `wc -l` -> `_fail 28` was not. A helper that
# printed its findings to stderr would keep every unit test green and turn
# this gate into a no-op.
# ===========================================================================

# _run_fullcov — same as _run, but with the flag CI actually sets. The runner
# exits 98 on incomplete coverage; that status is deliberately NOT the
# assertion. Statuses in this package have carried finding COUNTS rather than
# verdicts (#209: gate-19 returned 266 findings as exit status 10, and 256
# would have read as PASS), so the assertion reads the runner's own sentence
# on stdout. The status is captured only to be reported alongside it.
_OUT_FC=""
_STATUS_FC=""
_run_fullcov() {
local _logdir
_logdir="$(mktemp -d "${TMPDIR:-/tmp}/hydra-gate-test.XXXXXXXX")"
_OUT_FC="$(HYDRA_GATE_LOG_DIR="${_logdir}" bash "${RUNNER}" \
--app-dir "${_REPO}" --base "${_BASE}" --scope-to-diff \
--require-full-coverage 2>&1)"
_STATUS_FC=$?
rm -rf "${_logdir}"
if ! printf '%s' "${_OUT_FC}" | grep -q '^\[hydra-gates\] COVERAGE:'; then
_bad "full-coverage run in ${_REPO} ABORTED before the summary — not a result"
printf '%s\n' "${_OUT_FC}" | tail -15 | sed 's/^/ /'
return 1
fi
return 0
}

# _expect_gate28 <PASS|NOTAPPLICABLE|structural|FAIL> <output> <description>
# A FAIL-aware sibling of _expect that reads a caller-supplied capture, so the
# full-coverage runs can be asserted without disturbing _OUT.
_expect_gate28() {
local _want="$1" _out="$2" _desc="$3" _line _pattern
_line="$(printf '%s' "${_out}" | grep -E '^\[gate-28\] ' | head -1)"
if [ -z "${_line}" ]; then
_bad "${_desc} — gate-28 emitted NO line at all (a silent gate is the #147 shape)"
return
fi
case "${_want}" in
PASS) _pattern=': PASS' ;;
NOTAPPLICABLE) _pattern=': NOT APPLICABLE' ;;
structural) _pattern=': SKIPPED \(structural\)' ;;
FAIL) _pattern=': FAIL' ;;
*)
_bad "${_desc} — test bug: unknown expectation '${_want}'"
return ;;
esac
if printf '%s' "${_line}" | grep -qE "${_pattern}"; then
_ok "${_desc}"
else
_bad "${_desc} — got: ${_line}"
fi
}

# The runner's own sentence for "this run is being failed for coverage". Only
# printed when --require-full-coverage is set AND a gate did not run.
_COVFAIL_LINE='treating incomplete coverage as failure'

_MISMATCHED_PHP='<?php
/**
* @copyright Copyright (c) 2026 Conduction
* @license AGPL-3.0-or-later
*/

namespace OCA\Fixture;

class Drifted
{
public function value(): int
{
return 3;
}
}
'

# THE FIXTURES BELOW MUST BE OTHERWISE GREEN, and that is not a detail.
#
# The runner's coverage-failure branch lives inside `if [ "${_FAILED}" -eq 0 ]`.
# If ANY other gate has already failed, the run exits with the failure count and
# the coverage sentence is NEVER PRINTED. So an assertion of the form "the
# coverage sentence is absent" passes automatically in any fixture that trips an
# unrelated gate — including on a runner where #178 had been fully reverted.
#
# This was not hypothetical. The first version of case 5 wrote its Vue file to
# `src/App.vue`, which gate-38 (skip-link) fails as a root component without
# <NcContent>. Both full-coverage runs exited 1, not 98, and BOTH new
# assertions were measuring gate-38's failure rather than gate-28's verdict —
# the absence-assertion would have been dead. The file is written to
# `src/components/Widget.vue` for that reason, and the assertion reads the
# runner's positive "ALL … APPLICABLE GATES GREEN" sentence rather than the
# absence of a negative one: a sentence that must be PRESENT cannot be
# satisfied by a run that died earlier.
#
# For the same reason the control at case 6 CANNOT use untagged PHP: an
# untagged file also fails gate-1 (spdx-headers), so `_FAILED` is never 0 and
# the coverage branch is unreachable. It uses the other structural route
# instead — a composer.json with no `license` field, with a correctly tagged
# lib PHP file in the diff — which reaches `structural` with every other gate
# green.

# --- 5. #178 AS REPORTED: a frontend + docs diff (.vue + .md), no PHP at all,
# in a repo that HAS lib/ and a composer licence. Case 1 used a
# workflow file; this is the shape the issue was actually filed on.
_mkrepo scope-frontend-only yes
mkdir -p "${_REPO}/src/components"
printf '<template><div>base</div></template>\n' > "${_REPO}/src/components/Widget.vue"
git -C "${_REPO}" add -A >/dev/null 2>&1
git -C "${_REPO}" commit -qm "add a frontend component" >/dev/null 2>&1
_BASE="$(git -C "${_REPO}" rev-parse HEAD)"
printf '<template><div>changed</div></template>\n' > "${_REPO}/src/components/Widget.vue"
printf '# changed\n' > "${_REPO}/README.md"
git -C "${_REPO}" add -A >/dev/null 2>&1
git -C "${_REPO}" commit -qm "frontend + docs only, no PHP" >/dev/null 2>&1
if _run_fullcov; then
_expect_gate28 NOTAPPLICABLE "${_OUT_FC}" \
"#178: .vue + .md diff in a tagged repo is NOT APPLICABLE, not structural"
# THE ASSERTION #178 IS ACTUALLY ABOUT: the run must not be failed for it.
# Read the POSITIVE verdict, per the note above.
if printf '%s' "${_OUT_FC}" | grep -qE '^\[hydra-gates\] ALL [0-9]+ APPLICABLE GATES GREEN'; then
_ok "#178: the PHP-free diff run is GREEN under --require-full-coverage (status ${_STATUS_FC})"
else
_bad "#178: PHP-free diff did not end green under --require-full-coverage (status ${_STATUS_FC}) — the false red is back, or the fixture trips an unrelated gate and this assertion is measuring nothing"
fi
fi

# --- 6. THE CONTROL FOR 5, and the one that makes it mean something. Same
# flag, same runner, an otherwise-identical fixture: a run whose ONLY
# problem is a structural gate-28 must still be failed for coverage.
# Without this, assertion 5 would also pass on a runner that had simply
# stopped enforcing coverage at all.
#
# The licence field is dropped in a BASE-ADVANCING commit, not in the
# diff under test. Putting composer.json in the diff makes gate-4
# (composer-audit) applicable, and it fails on a fixture with no vendor
# tree — `_FAILED` becomes 1 and the coverage sentence is unreachable
# again. The diff must contain the lib PHP file and nothing else.
_mkrepo scope-nolicense-fullcov yes
printf '{\n "name": "conduction/fixture"\n}\n' > "${_REPO}/composer.json"
git -C "${_REPO}" add -A >/dev/null 2>&1
git -C "${_REPO}" commit -qm "drop the composer license field" >/dev/null 2>&1
_BASE="$(git -C "${_REPO}" rev-parse HEAD)"
printf '%s\n// touched\n' "${_LICENSED_PHP}" > "${_REPO}/lib/Tagged.php"
git -C "${_REPO}" add -A >/dev/null 2>&1
git -C "${_REPO}" commit -qm "touch the tagged lib PHP file" >/dev/null 2>&1
if _run_fullcov; then
_expect_gate28 structural "${_OUT_FC}" \
"control: composer.json with no license is still structural under --require-full-coverage"
if printf '%s' "${_OUT_FC}" | grep -qF "${_COVFAIL_LINE}"; then
_ok "control: --require-full-coverage DID fail the run on a real gap (status ${_STATUS_FC})"
else
_bad "control: --require-full-coverage did NOT fail a run whose only gap is gate-28 (status ${_STATUS_FC}) — the coverage requirement is inert here, so assertion 5 proves nothing"
fi
fi

# --- 7. THE GATE'S ACTUAL PURPOSE, end to end. A file declaring a licence
# that composer.json does not — the `_fail 28` branch, which no test
# above reaches. This is the assertion that would catch the helper
# writing findings anywhere but stdout.
_mkrepo scope-mismatch yes
printf '%s' "${_MISMATCHED_PHP}" > "${_REPO}/lib/Drifted.php"
git -C "${_REPO}" add -A >/dev/null 2>&1
git -C "${_REPO}" commit -qm "lib PHP declaring AGPL against an EUPL composer.json" >/dev/null 2>&1
if _run; then
_expect_gate28 FAIL "${_OUT}" \
"licence MISMATCH in scope (@license AGPL vs composer EUPL): gate-28 FAILS"
# ...and it must fail for the RIGHT reason. A gate that failed here for any
# other cause would satisfy the line above while comparing nothing.
if printf '%s' "${_OUT}" | grep -qE '^\[gate-28\].*@license != composer\.json'; then
_ok "mismatch failure names the licence comparison as its cause"
else
_bad "mismatch failed, but not with the licence-comparison reason — cause unverified"
fi
fi

echo
echo "== summary =="
echo " passed: ${_pass_n}"
Expand Down
Loading