Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 31 additions & 5 deletions hydra-gates/scripts/lib/check_spec_anchors.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: EUPL-1.2
"""Gate-46 spec-anchor-existence — does the `@spec` tag point at something real?
"""Gate-46 spec-anchor-existence — does the tag point at something real?

Reads `@spec` and `@e2e`, which are written with the same target grammar and
mean the same thing in opposite directions: this code implements that
requirement, this test proves it.

Gate-16 checks that a `@spec` tag is PRESENT. This gate checks that its
target RESOLVES: the file exists, and — when the tag carries a `#fragment`
Expand Down Expand Up @@ -96,10 +100,31 @@
# the shape below is 0.0000s. Same family as the `py/redos` CodeQL raised on
# gate-28's tail rule in this change; found by sweeping the other regexes it
# adds rather than by waiting for the alert.
#
# BOTH TRACEABILITY TAGS, NOT ONLY `@spec` (.github#726).
#
# `@e2e openspec/...` is written with the identical target grammar and means
# the identical thing — this test proves that requirement — and until this
# change no gate resolved one. Measured across the 21 fleet apps: 1,964 `@e2e`
# anchors, 194 of them dangling, against 55,720 `@spec` anchors with ONE
# unresolved. The gate was never wrong; it read four fifths of the corpus.
#
# The measurement is reproducible and was NOT a reimplementation: every `@e2e`
# target was rewritten to `@spec` in a probe file outside the repo and run
# through THIS helper with cwd set to the app, so the numbers are this
# resolver's own semantics. That probe also established the two facts this
# widening depends on — every `@e2e` line carrying an `openspec/` path already
# matches the shape below, and no such tag lives outside `tests/`, which is
# already enumerated.
#
# The tag NAME is captured so the finding can say which one it came from, and
# so the runner can block on `@spec` while reporting `@e2e`. That split is the
# whole reason the group exists; do not collapse it back to a non-capturing
# group without moving the label somewhere else.
TAG = re.compile(
r'^[^\S\n]*(?:(?:\*+/?|//+|\#(?!\[)|/\*+|<!--|[-+]|\d+\.)[^\S\n]*)?'
r'(?:\[[ xX~\-]\][^\S\n]*)?'
r'@spec\s+(openspec/[^\s`\'"]+)',
r'@(spec|e2e)\s+(openspec/[^\s`\'"]+)',
re.MULTILINE,
)
DATE = re.compile(r'\b\d{4}-\d{2}-\d{2}\b')
Expand Down Expand Up @@ -542,18 +567,19 @@ def scan_files(files: list[str], root: str | None = None) -> list[str]:
except OSError:
continue
for m in TAG.finditer(src):
target = m.group(1)
tag, target = m.group(1), m.group(2)
if '#' in target:
path, frag = target.split('#', 1)
else:
path, frag = target, None
candidates = resolve(path, root, archive_index, capability_index)
if not candidates:
findings.append(f"{fp}: @spec target file not found → {target}")
findings.append(f"{fp}: @{tag} target file not found → {target}")
continue
if frag and path.endswith('.md'):
if not any(has_anchor(c, frag) for c in candidates):
findings.append(f"{fp}: @spec anchor not found in {path} → #{frag}")
findings.append(
f"{fp}: @{tag} anchor not found in {path} → #{frag}")
return findings


Expand Down
72 changes: 72 additions & 0 deletions hydra-gates/scripts/lib/test_check_spec_anchors.py
Original file line number Diff line number Diff line change
Expand Up @@ -786,5 +786,77 @@ def test_tp_a_prefixed_number_past_the_end_still_fails(self):
self.assertFalse(_anchor(self.PREFIXED, "task-9.9"))


class TheGateReadsE2eAnchorsToo(unittest.TestCase):
"""#726 — `@e2e openspec/...` is the same claim, pointed the other way.

1,964 of them across the 21 fleet apps, 194 dangling, resolved by nothing
until this change, against 55,720 `@spec` anchors with ONE unresolved.

Every arm here ships with the control that stops the widening from
becoming noise, because that is how this gate earned its 54% false
positive rate the first time: a resolving `@e2e` anchor must stay silent,
and `@e2e exclude` must never be read as a target.
"""

TREE = {
"openspec/specs/real/spec.md":
"# Spec\n\n## Requirement: Real Thing\n\ntext\n",
}

def test_e1_a_dangling_e2e_target_is_a_finding(self):
out = _scan(self.TREE, "tests/e2e/thing.spec.ts",
"/** @e2e openspec/specs/absent/spec.md */\n"
"test('x', async () => {})\n")
self.assertEqual(len(out), 1, out)
self.assertIn("absent", out[0])

def test_e2_a_dangling_e2e_anchor_is_a_finding(self):
out = _scan(self.TREE, "tests/e2e/thing.spec.ts",
"/** @e2e openspec/specs/real/spec.md#requirement-imaginary */\n"
"test('x', async () => {})\n")
self.assertEqual(len(out), 1, out)

def test_e3_the_finding_is_labelled_by_the_tag_it_came_from(self):
# THE LOAD-BEARING ONE. The runner blocks on `@spec` and reports
# `@e2e`, and it computes that split by reading these labels out of
# the log. Collapse the label and the two counts become one.
e2e = _scan(self.TREE, "tests/e2e/thing.spec.ts",
"/** @e2e openspec/specs/absent/spec.md */\n")
spec = _scan(self.TREE, "lib/Thing.php",
"<?php\n/**\n * @spec openspec/specs/absent/spec.md\n */\n")
self.assertIn("@e2e ", e2e[0])
self.assertNotIn("@spec ", e2e[0])
self.assertIn("@spec ", spec[0])
self.assertNotIn("@e2e ", spec[0])

def test_e4_control_a_resolving_e2e_anchor_is_silent(self):
out = _scan(self.TREE, "tests/e2e/thing.spec.ts",
"/** @e2e openspec/specs/real/spec.md#requirement-real-thing */\n")
self.assertEqual(out, [])

def test_e5_control_an_e2e_exclusion_is_not_a_target(self):
# 5,529 reason-bearing exclusions fleet-wide. The `openspec/` prefix
# in the pattern is the only thing keeping them out, so pin it.
out = _scan(self.TREE, "tests/e2e/thing.spec.ts",
"/** @e2e exclude asserted below the HTTP surface by "
"GhostServiceTest */\n")
self.assertEqual(out, [])

def test_e6_control_the_tag_is_still_position_anchored(self):
# #415/#423 applies to both tags or to neither: a sentence about a
# tag is not a tag.
out = _scan(
self.TREE, "tests/e2e/thing.spec.ts",
"// See @e2e openspec/specs/gone/spec.md#missing for why this went.\n")
self.assertEqual(out, [])

def test_e7_control_an_unrelated_at_tag_is_not_read(self):
# `@spec` and `@e2e` and nothing else. `@contract` and `@visual` are
# other gates' vocabulary and resolve against other things.
out = _scan(self.TREE, "tests/e2e/thing.spec.ts",
"/** @contract openspec/specs/absent/spec.md */\n")
self.assertEqual(out, [])


if __name__ == "__main__":
unittest.main(verbosity=2)
134 changes: 131 additions & 3 deletions hydra-gates/scripts/lib/test_gate_46_tests_scope.sh
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: EUPL-1.2
#
# test_gate_46_tests_scope.sh — gate-46 must resolve `@spec` targets wherever
# the tag is WRITTEN, and `tests/` is one of those places.
# test_gate_46_tests_scope.sh — gate-46 must resolve its targets wherever the
# tag is WRITTEN, and `tests/` is one of those places.
#
# WHAT THIS GUARDS (.github#322)
# ------------------------------
Expand Down Expand Up @@ -38,6 +38,19 @@
# a path. Arm 5 is that file's shape verbatim, arm 6 is the anti-widening
# control, and arm 7 pins `scripts/` so the weaker half of the request cannot
# be dropped by accident later.
#
# ARMS 8 TO 11 cover the OTHER tag (.github#726). `@e2e openspec/...` uses the
# identical target grammar and means the identical thing in the opposite
# direction, and until #726 no gate resolved one: 1,964 of them fleet-wide,
# 194 dangling, against 55,720 `@spec` anchors with one unresolved. They are
# report-only, and arms 9 and 10 are what stop that from drifting in either
# direction — a widening that quietly starts blocking is as much a defect as
# one that quietly stops reporting.
#
# ⚠️ Those four arms were 5 to 8 before #734 landed first and took 5 to 7.
# Renumbering them is the whole reason this note carries numbers at all: an
# arm header and a prose reference that disagree is how a suite ends up
# describing a test it is not running.

set -u

Expand All @@ -59,7 +72,7 @@ trap 'rm -rf "${_tmp}"' EXIT
# missing file. `openspec/changes/dso-omgevingsloket/` no longer exists;
# `openspec/changes/archive/2026-06-13-dso-omgevingsloket/` does.
_mkapp() { # _mkapp <dir>
mkdir -p "$1/lib" "$1/src" "$1/tests/Unit" "$1/appinfo" "$1/scripts" \
mkdir -p "$1/lib" "$1/src" "$1/tests/Unit" "$1/tests/e2e" "$1/appinfo" "$1/scripts" \
"$1/openspec/changes/archive/2026-06-13-dso-omgevingsloket"
printf '{"name":"fx","menu":[]}\n' > "$1/src/manifest.json"
cat > "$1/openspec/changes/archive/2026-06-13-dso-omgevingsloket/tasks.md" <<'MD'
Expand Down Expand Up @@ -244,6 +257,121 @@ else
_bad "the finding does not name scripts/seed.php — the directory was not opened"
fi

# ---------------------------------------------------------------------------
# ARM 8 — a dangling `@e2e` anchor is a finding, and it WARNS (#726).
#
# dossiq#2057's shape verbatim: a spec file renumbered its scenarios and the
# e2e spec kept citing the old ids. Every check passed, and the tests went on
# reporting green while naming scenarios that no longer existed.
# ---------------------------------------------------------------------------
_app="${_tmp}/a8"
_mkapp "${_app}"
cat > "${_app}/tests/e2e/kanban-board.spec.ts" <<'TS'
/** @e2e openspec/changes/dso-omgevingsloket/tasks.md#T14 */
test('the board moves a card', async () => {})
TS
_out="$(_run46 "${_app}")"
_assert "a dangling @e2e anchor → WARNING" "WARNING" "${_out}"
if grep -q 'kanban-board.spec.ts' "$(cat "${_LAST_LOG_PTR}")" 2>/dev/null; then
_ok "the @e2e finding NAMES the spec file"
else
_bad "the @e2e finding does not name the spec file"
fi
if grep -q '@e2e anchor not found' "$(cat "${_LAST_LOG_PTR}")" 2>/dev/null; then
_ok "the finding is LABELLED @e2e, so the runner can tell the two counts apart"
else
_bad "the finding is not labelled @e2e — the split verdict cannot be computed from this log"
fi

# ---------------------------------------------------------------------------
# ARM 9 — ANTI-WIDENING, twice over.
#
# A resolving `@e2e` anchor must PASS, or reading the second tag has simply
# moved the blindness into noise. And a reason-bearing `@e2e exclude` is not a
# target at all: the tag regex requires an `openspec/` path, and 5,529
# exclusions fleet-wide would become findings if it did not.
# ---------------------------------------------------------------------------
_app="${_tmp}/a9"
_mkapp "${_app}"
cat > "${_app}/tests/e2e/deadline.spec.ts" <<'TS'
/** @e2e openspec/changes/dso-omgevingsloket/tasks.md#T02 */
test('the deadline warns', async () => {})
TS
cat > "${_app}/tests/e2e/excluded.spec.ts" <<'TS'
/** @e2e exclude reconciliation runs below the HTTP surface, asserted by GhostServiceTest */
test('nothing to see', async () => {})
TS
_assert "a REAL @e2e anchor, and an @e2e exclude beside it → PASS" \
"PASS" "$(_run46 "${_app}")"

# ---------------------------------------------------------------------------
# ARM 10 — THE ADVISORY HALF MUST NOT MASK THE BLOCKING HALF.
#
# One repo, both tags, both dangling. `@spec` still FAILS. This is the arm
# that fails if somebody "simplifies" the split by warning on everything, and
# it is the reason the counts are computed per tag rather than in total.
# ---------------------------------------------------------------------------
_app="${_tmp}/a10"
_mkapp "${_app}"
cat > "${_app}/tests/e2e/both.spec.ts" <<'TS'
/** @e2e openspec/changes/dso-omgevingsloket/tasks.md#T14 */
test('a', async () => {})
TS
cat > "${_app}/lib/Both.php" <<'PHP'
<?php
class Both {
/** @spec openspec/changes/dso-omgevingsloket/tasks.md#T99 */
public function run(): void {}
}
PHP
_out="$(_run46 "${_app}")"
_assert "a dangling @spec alongside a dangling @e2e → FAIL, not WARNING" "FAIL" "${_out}"
case "${_out}" in
*"1 @spec"*) _ok "the FAIL states BOTH counts, so the advisory half is visible in the blocking verdict" ;;
*) _bad "the FAIL does not state the per-tag counts — got: ${_out}" ;;
esac

# ---------------------------------------------------------------------------
# ARM 11 — the per-repo switch actually blocks.
#
# A report-only gate whose opt-in does nothing is worse than no opt-in: it
# reads as a promise that the debt can be locked down once it is cleared.
# ---------------------------------------------------------------------------
_app="${_tmp}/a11"
_mkapp "${_app}"
cat > "${_app}/tests/e2e/switch.spec.ts" <<'TS'
/** @e2e openspec/changes/dso-omgevingsloket/tasks.md#T14 */
test('a', async () => {})
TS
_assert "HYDRA_GATE_SPEC_ANCHOR_E2E_BLOCKING=1 turns the warning into a failure" \
"FAIL" "$(HYDRA_GATE_SPEC_ANCHOR_E2E_BLOCKING=1 _run46 "${_app}")"

# ---------------------------------------------------------------------------
# ARM 12 — THE INTERACTION THE REBASE CREATED, which neither change covers on
# its own: the SECOND tag inside the NEWLY OPENED directory.
#
# #727 anticipated it in as many words — "if `@e2e` starts being read, the
# scope question gets slightly bigger, because `appinfo/` carries none but
# `scripts/` might in future". Arms 5 to 7 only ever write `@spec` there, arms
# 8 to 11 only ever write `@e2e` under `tests/`, so the product of the two
# changes was the one square nothing landed on. It works, and this is what
# says so out loud rather than by inference from two green suites.
# ---------------------------------------------------------------------------
_app="${_tmp}/a12"
_mkapp "${_app}"
cat > "${_app}/scripts/smoke.js" <<'JS'
/** @e2e openspec/changes/dso-omgevingsloket/tasks.md#T14 */
run()
JS
_out="$(_run46 "${_app}")"
_assert "a dangling @e2e in scripts/ → WARNING (both changes, one square)" \
"WARNING" "${_out}"
if grep -q '@e2e anchor not found' "$(cat "${_LAST_LOG_PTR}")" 2>/dev/null; then
_ok "it is labelled @e2e, so the newly-opened directory feeds the split verdict too"
else
_bad "the finding in scripts/ is not labelled @e2e"
fi

echo ""
if [ "${_failures}" -eq 0 ]; then
echo "test_gate_46_tests_scope.sh: ALL GREEN"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -805,6 +805,16 @@ fi
# gate -> the reason it is permitted to be non-blocking.
#
# 19 e2e-coverage .github#477, temporary, owned.
# 46 spec-anchor-existence
# .github#726, and PARTIAL: only the `@e2e` half is
# advisory. An unresolved `@spec` target still FAILS
# in the same run, so this is not a demotion of the
# gate — it is the ship condition for a widening.
# 194 dangling `@e2e` anchors fleet-wide, 154 of them
# in four repos (dossiq 46, zaakafhandelapp 43,
# pipelinq 36, decidesk 29), which are the four with
# the most e2e coverage. Per-repo opt-in:
# HYDRA_GATE_SPEC_ANCHOR_E2E_BLOCKING=1.
# 112 newman-reach .github#712. Shipped blocking and failed 10 of the
# 21 fleet repos within the hour, on inherited debt no
# PR introduced (openregister alone: 53 findings).
Expand All @@ -829,9 +839,9 @@ fi
# remainder needs exclusions, not repoints. Per-repo
# opt-in: HYDRA_GATE_STALE_FLEET_APP_ID_BLOCKING=1.
#
# All four are worked down per app and flipped back individually; the findings
# All five are worked down per app and flipped back individually; the findings
# print on every run either way, so none is hidden.
_ADVISORY_ALLOWED_GATES="19 112 113 114 115"
_ADVISORY_ALLOWED_GATES="19 46 112 113 114 115"

_runner_src="${GF_PKG_ROOT}/scripts/run-hydra-gates.sh"
_warn_gates="$(grep -oE '^[[:space:]]*_warn[[:space:]]+[0-9]+' "${_runner_src}" \
Expand Down
Loading
Loading