Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 90 additions & 1 deletion hydra-gates/scripts/lib/check_license_triangle.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,92 @@
DECL = re.compile(
r'^[\s]*(?:/\*+|\*+/?|//+|#+|<!--)?[\s]*'
r'(?:@license|SPDX-License-Identifier:)[\s]+'
r'([^\s*\'"`;,)\]]+)',
r'([^\s*\'"`;,)\]]+)'
r'(?P<rest>[^\n]*)',
re.MULTILINE,
)

# A SENTENCE THAT BEGINS WITH THE TAG IS NOT A SECOND DECLARATION (#415/#423).
#
# The anchor above was already right about position — a quote before the tag
# means test data, not a claim — but it said nothing about what FOLLOWS the
# identifier. So a docblock line explaining a licence the file does NOT use:
#
# * @license EUPL-1.2
# * @license MIT was never used here — see the note above.
#
# read as a file declaring two licences, and produced BOTH
# `license-internal-conflict` and `license-triangle-drift`. The note about
# the licence that was ruled out scored as the licence.
#
# THE REMAINDER MAY NOT BE EMPTY, because the fleet's own header is not.
# Measured over every tracked `lib/**/*.php` in openregister, opencatalogi,
# procest, docudesk, larpingapp and softwarecatalog — 2,394 files, every
# distinct declaration form:
#
# 2724 @license EUPL-1.2 https://joinup.ec.europa.eu/…/eupl-text-eupl-12
# 2095 @license EUPL-1.2 (and `SPDX-License-Identifier: EUPL-1.2`)
# 11 @license https://www.gnu.org/licenses/agpl-3.0.html GNU AGPL v3 or later
# 8 @license AGPL-3.0-or-later https://www.gnu.org/licenses/agpl-3.0.en.html
#
# A rule of "nothing may follow the identifier" would have deleted the FIRST
# and most common form — 2,724 real declarations, in the fleet's own house
# style — and the gate would have gone quiet on exactly the files it exists
# to compare. So the remainder is allowed to be a URL, a comment terminator
# or trailing punctuation, and nothing else: those are what a header carries
# and prose is what it does not.
#
# The URL-FIRST form (11 files) is deliberately left alone: its identifier is
# a URL, `_looks_like_a_path` already reports it as a parser malfunction, and
# that diagnostic is not this change's to silence. The rule below therefore
# only applies to a value that is not itself path-shaped.
# ⚠️ THIS WAS A REGEX AND THE REGEX WAS A ReDoS (CodeQL `py/redos`, HIGH).
# The first cut read
#
# ^(?:\s|https?://\S+|\*/|--!?>|[.,;:)\]]|\(\s*\)|\s*\Z)*$
#
# `\S+` inside a STARRED alternation is ambiguous with itself: `https://ab`
# can be consumed by one iteration or by several, so a subject that never
# reaches `$` has exponentially many splits to try.
#
# MEASURED, because the alert's own witness string did NOT reproduce here.
# CodeQL named `'http://' + '!http://' * n`; in CPython that is 0.000s even
# at n=32,000, because `\s*\Z` can match empty and the engine's empty-repeat
# guard cuts the search short. The alert is still right — the ambiguity is
# real — it just names the wrong subject. The one that fires is a repeated
# URL with one trailing byte that cannot match:
#
# 'https://a' * n + ' \x01' OLD NEW
# n=18 len=164 0.0949s 0.0000s
# n=22 len=200 1.5284s 0.0000s
# n=24 len=218 6.6797s 0.0000s
# n=26 len=236 26.1822s 0.0000s
# n=28 len=254 >60s 0.0000s
#
# Every +2 multiplies the old time by ~4. A 254-character line — shorter
# than plenty of real headers — hangs the gate, and gate-28 runs over every
# tracked lib/**/*.php in the repo.
#
# Rewritten as a linear scan rather than tightened, so the ambiguity cannot
# exist rather than being harder to reach: split the remainder on whitespace
# and ask of each token whether a header could carry it. No quantifier, no
# backtracking. Flat on the adversarial input above, and still flat at
# 256 KB (0.0103s, linear in length).
def _decl_tail_ok(rest: str) -> bool:
"""Is what follows the identifier header FURNITURE rather than prose?

A header carries the identifier and at most a URL, a comment terminator
and punctuation. A sentence carries words. See the measurement above the
caller for why "nothing may follow" was not an option.
"""
for token in rest.split():
token = _TRAILING_COMMENT.sub('', token).strip('.,;:()[]')
if not token:
continue
if token.startswith(('http://', 'https://')):
continue
return False
return True
# `/` is deliberately ALLOWED in the captured value. Excluding it (to stop
# `@license EUPL-1.2*/` capturing the comment terminator) also truncated a
# path-shaped value to its first segment, so `lib/Service/Thing.php` arrived
Expand Down Expand Up @@ -100,6 +183,12 @@ def declarations(src: str) -> list[str]:
v = _TRAILING_COMMENT.sub('', m.group(1).strip()).strip().rstrip('.,;')
if not v or v in seen:
continue
# See _decl_tail_ok: a header carries an identifier and at most a URL;
# prose after the identifier means the line is a SENTENCE about a
# licence, not a declaration of one. Path-shaped values keep their
# existing "parser malfunction" report regardless of what follows.
if not _looks_like_a_path(v) and not _decl_tail_ok(m.group('rest')):
continue
seen.append(v)
return seen

Expand Down
51 changes: 45 additions & 6 deletions hydra-gates/scripts/lib/check_listener_placement.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@
import sys
from pathlib import Path

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from source_scope import php_mask # noqa: E402

GATE_NUM = 61

# The three POST events. A listener on one of these cannot influence the write,
Expand Down Expand Up @@ -293,10 +296,29 @@ def parse_registrations(path: Path) -> list[dict]:
Each entry: {'event', 'listener', 'start_line', 'end_line'}.
"""
try:
text = path.read_text(encoding='utf-8', errors='replace')
raw = path.read_text(encoding='utf-8', errors='replace')
except OSError:
return []

# A COMMENT MUST NOT MANUFACTURE A FINDING (#415/#423).
#
# This function read the RAW file, so a registration that exists ONLY as
# a commented-out line was parsed as a live one:
#
# // Removed 2026-08: SyncListener did its HTTP work inline, so the
# // listener and its registration both went:
# // $context->registerEventListener(ObjectCreatedEvent::class,
# // SyncListener::class);
#
# produced `FAIL … ObjectCreatedEvent -> SyncListener — no class file
# found under lib/`. Nothing is wired; the listener was DELETED, and the
# gate reported the deletion as a broken registration. The author's
# cheapest fix is to delete the explanation of the deletion.
#
# `_strip_comments` is offset-preserving, so `text.count('\\n', 0, …)`
# below still yields the line number the finding reports.
text = _strip_comments(raw)

# Post events named anywhere in the file — used to attribute a registration
# whose event argument is a variable (see the foreach case below).
post_tokens = {e for e in POST_EVENTS
Expand Down Expand Up @@ -470,11 +492,28 @@ def classify_annotation(block: list[str]) -> tuple[str, str | None]:


def _strip_comments(text: str) -> str:
"""Blank out comments so a signal named only in prose is not a finding."""
text = re.sub(r'/\*.*?\*/', '', text, flags=re.DOTALL)
text = re.sub(r'(?m)//.*$', '', text)
text = re.sub(r'(?m)^\s*#(?!\[).*$', '', text)
return text
"""Blank out comments so a signal named only in prose is not a finding.

Delegates to ``source_scope.php_mask`` (#415/#423) instead of the three
regexes that used to live here. Three concrete differences, each of which
was a defect:

* it is OFFSET-PRESERVING — comments are blanked, not deleted — so the
result can be searched interchangeably with the raw text and a line
number computed on it still names the right line. The old version
shortened the string, which is why it could not be used by
:func:`parse_registrations`, which needs line numbers;
* it is STRING-AWARE, so a `//` inside `'https://example.org/hook'`
does not truncate the line and delete the very call the gate is
looking for;
* it knows a TRAILING `#` opens a comment (the old `^\\s*#` only saw
one at the start of a line) and that `#[` opens a PHP 8 attribute.

String CONTENTS are kept (``blank_strings`` at its default): the URL in
an inline `->post('https://…')` and the class name in a `class_exists()`
are evidence about code, and blanking them would delete it.
"""
return php_mask(text)


def scan_body(text: str) -> list[tuple[str, str]]:
Expand Down
65 changes: 59 additions & 6 deletions hydra-gates/scripts/lib/check_phantom_cross_app_rpc.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,58 @@
import re
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from source_scope import js_comment_mask, php_mask # noqa: E402

# A COMMENT MUST NOT MANUFACTURE A FINDING (#415/#423).
#
# The comment test used to be `stripped.startswith(("//", "*", "/*", "#"))`
# applied per LINE, which only recognises a comment-OPENING line. One clean
# fixture produced THREE findings:
#
# * an interior line of an unindented `/* … */` block, describing the
# phantom call that was REMOVED;
# * a TRAILING `// never do $reg->getLeaf($id) here`;
# * a string literal warning against the same call.
#
# The first two are closed here by masking the file with the shared,
# offset-preserving strippers before any rule runs. Line numbering survives
# because the masks blank text without removing lines, and every finding
# reports a line number.
#
# STRINGS ARE KEPT ON PURPOSE (`blank_strings` left at its default). Rule B
# is `->call('<fleetAppId>', …)` — the evidence IS a string literal, and the
# rule's own docstring records that requiring a quoted app id is what makes
# it zero-false-positive. Blanking literals to close the third finding would
# delete Rule B entirely: a false positive traded for a false negative in
# the gate that exists because a phantom call only ever dies at runtime.
# The literal axis is tracked as #424, not smuggled in here.
#
# ⚠️ `.vue` IS DELIBERATELY NOT MASKED. The mask for an SFC is
# `source_scope.script_mask`, which routes through the `<!-- … -->` handling
# that #424 is currently repairing (a `<!--` inside a string literal blanks
# live code, i.e. it can silence the gate). Adopting it today would inherit
# that hole into a gate whose findings are runtime fatals. The `.vue`
# false positive therefore STANDS, and is blocked on #424 — stated rather
# than papered over, because a partially-masked checker looks exactly like a
# fully-masked one from the outside.
_MASKERS = {
".php": php_mask,
".js": js_comment_mask,
".ts": js_comment_mask,
".mjs": js_comment_mask,
".cjs": js_comment_mask,
".jsx": js_comment_mask,
".tsx": js_comment_mask,
}


def _code_text(path: str, text: str) -> str:
"""*text* with comments blanked, offsets and line numbers preserved."""
masker = _MASKERS.get(os.path.splitext(path)[1].lower())
return masker(text) if masker else text


# Known fleet app ids. A ->call('<id>', ...) with one of these as the
# first string-literal argument is a phantom cross-app RPC (Rule B).
FLEET_APP_IDS = {
Expand Down Expand Up @@ -249,14 +301,15 @@ def scan_file(path: str) -> int:
except OSError:
return 0

lines = text.split("\n")
lines = _code_text(path, text).split("\n")
violations = 0

for idx, line in enumerate(lines, start=1):
# Skip obvious comment-only lines (PHP // / * / # and JS //) so a
# documented example or a forbidden-pattern note never trips the
# gate. We keep it simple: a line whose first non-space char starts
# a comment is ignored for the literal-call rules.
# Retained for the file types `_code_text` does not mask (`.vue`,
# blocked on #424): a line whose first non-space char opens a comment
# is ignored for the literal-call rules. On a masked file every
# comment line is already blank, so this costs nothing and the two
# never disagree.
stripped = line.lstrip()
is_comment = (
stripped.startswith("//")
Expand Down Expand Up @@ -376,7 +429,7 @@ def check_against_or_contract(path: str, contract: dict) -> None:

try:
with open(path, "r", encoding="utf-8", errors="replace") as fh:
lines = fh.read().split("\n")
lines = _code_text(path, fh.read()).split("\n")
except OSError:
return

Expand Down
60 changes: 56 additions & 4 deletions hydra-gates/scripts/lib/check_security_cochange.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@
import subprocess
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from source_scope import js_comment_mask, php_mask # noqa: E402

# Security code identified by LOCATION. Any change to such a file qualifies —
# there is no "incidental" edit to lib/Service/Auth/TokenVerifier.php.
_PATH_PATTERNS = (
Expand Down Expand Up @@ -139,7 +142,51 @@
_CANDIDATE_RE = re.compile(r"^(lib/.*\.php|src/.*\.(vue|js|ts))$")

# Comment shapes, per line, after leading whitespace.
_COMMENT_LINE_RE = re.compile(r"^\s*(?:\*|//|/\*|\#(?!\[))")
#
# `<!--` was missing from this vocabulary entirely while `src/**/*.vue` has
# always been a candidate path (#415/#423), so
#
# <!-- IUserSession is resolved server-side, see ADR-005 -->
#
# was read as a security-relevant change and demanded a test co-change.
_COMMENT_LINE_RE = re.compile(r"^\s*(?:\*|//|/\*|<!--|\#(?!\[))")

# A TRAILING COMMENT IS A COMMENT TOO (#415/#423).
#
# The test above is `match`, i.e. it only ever recognised a line that STARTS
# with a comment. The module's own docstring promises the opposite —
# *"Prose that merely mentions `IUserSession` is not — it is a sentence"* —
# and the annotation arm implements it, but a sentence written after code on
# the same line was still read as code:
#
# return $this->render(); // resolved via IUserSession elsewhere
#
# One reworded end-of-line comment therefore turned gate-47 red and demanded
# a test for a change containing none. That is #191's shape one level up: the
# cheapest way to clear the finding is to reword the prose again, so the gate
# manufactures the appearance of a security review.
#
# Masked with the shared, offset-preserving strippers, dispatched on the
# file's own extension. STRING CONTENTS ARE KEPT (`js_comment_mask`, and
# `php_mask`'s default): `requesttoken` reaches this gate as the KEY of a
# header object and `'IUserSession'` as a DI id, so blanking literals would
# delete real evidence — the axis tracked as #424.
#
# ⚠️ A `<!--` appearing MID-LINE in an SFC template is not handled here: that
# needs `source_scope.markup_mask`, whose delimiter handling is under repair
# in #424. Line-START is what the fixture in #423 shows and what is closed.
_LINE_MASKERS = {
".php": php_mask,
".js": js_comment_mask,
".ts": js_comment_mask,
".vue": js_comment_mask,
}


def _line_code(line: str, path: str) -> str:
"""*line* with any trailing comment blanked, same length."""
masker = _LINE_MASKERS.get(os.path.splitext(path)[1].lower())
return masker(line) if masker else line


def is_test_path(path: str) -> bool:
Expand All @@ -150,7 +197,7 @@ def is_security_path(path: str) -> bool:
return any(p.match(path) for p in _PATH_PATTERNS)


def line_is_security_relevant(line: str) -> bool:
def line_is_security_relevant(line: str, path: str = "") -> bool:
"""Is this ONE changed line a security change?

A comment line qualifies only via ``_ANNOTATION_RE``, and only when the
Expand All @@ -162,12 +209,17 @@ def line_is_security_relevant(line: str) -> bool:
``match`` rather than ``search``: ``_ANNOTATION_RE`` is anchored with
``^`` on both branches, so the two are equivalent here, but ``match``
states the intent — position is the whole point of this regex.

*path* names the file the line came from, so the trailing-comment mask
can be chosen by extension (see ``_LINE_MASKERS``). It defaults to empty,
in which case the line is judged unmasked — the pre-#423 behaviour, and
the reason no caller is obliged to know about it.
"""
if _ANNOTATION_RE.match(line):
return True
if _COMMENT_LINE_RE.match(line):
return False
return bool(_CODE_TOKEN_RE.search(line))
return bool(_CODE_TOKEN_RE.search(_line_code(line, path)))


def changed_lines(base_ref: str, path: str, cwd: str,
Expand Down Expand Up @@ -276,7 +328,7 @@ def scan(base_ref: str, cwd: str = ".") -> tuple[list[str], bool]:
if not _CANDIDATE_RE.match(f):
continue
for line in changed_lines(base_ref, f, cwd, renames.get(f)):
if line_is_security_relevant(line):
if line_is_security_relevant(line, f):
security.append(f)
break
return security, has_test
Expand Down
Loading
Loading