From 542e93e9e761a7740efb67d5a2560e0a937c2e3a Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:46:05 -0500 Subject: [PATCH 1/3] The redactor reads a status line of spaces in time, and SonarCloud's shell style rule is set aside for two scripts STATUS_LINE's reason phrase and its trailing space could both hold a space, so a status line with a long run of them and then what no status holds took time in its square: 2.8 s at 32,000 characters through the redactor, on a capture read before anyone has looked at it. The phrase now ends in no space. The language is the same, and a test holds both. S8786 on check_context's TITLE is declined where it stands: issue 369's test holds its CPU linear on the shape the rule fears. Issue 361's 43 findings are all shell:S7688, `[[` in place of `[`, style alone. The Factory Builder ruled they are set aside in sonar-project.properties for tools/context_eval.sh and tools/trial_as_app.sh, and neither script is edited: the second holds a credential's path, and CI cannot run it for real. Co-Authored-By: Claude Opus 5.5 --- sonar-project.properties | 8 +++++++- tests/test_tools.py | 13 +++++++++++++ tools/check_context.py | 5 ++++- tools/mutants.py | 6 ++++++ tools/redact_trial.py | 3 ++- 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/sonar-project.properties b/sonar-project.properties index d401aaf7c..7045e7fcc 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -10,6 +10,12 @@ sonar.coverage.exclusions=**/* # One rule set aside for one file: python:S5332 (clear-text HTTP) on tools/trial_p50_stub.py, P-50's stub of the model's # API. It serves the pinned CLI on 127.0.0.1 alone, in the canary's workflow and this repo's tests, and HTTPS would need a # local CA the CLI does not trust. tools/check_repo.py holds it to loopback; decision 0044's Boundaries row says why. -sonar.issue.ignore.multicriteria=p50stubhttp +sonar.issue.ignore.multicriteria=p50stubhttp,contexttest,trialapptest sonar.issue.ignore.multicriteria.p50stubhttp.ruleKey=python:S5332 sonar.issue.ignore.multicriteria.p50stubhttp.resourceKey=tools/trial_p50_stub.py +# One rule set aside for two scripts: shell:S7688 (`[[` in place of `[`). It is style alone, and no finding of it is a +# defect. tools/trial_as_app.sh holds a credential's path and CI cannot run it for real, so it is not edited for style. +sonar.issue.ignore.multicriteria.contexttest.ruleKey=shell:S7688 +sonar.issue.ignore.multicriteria.contexttest.resourceKey=tools/context_eval.sh +sonar.issue.ignore.multicriteria.trialapptest.ruleKey=shell:S7688 +sonar.issue.ignore.multicriteria.trialapptest.resourceKey=tools/trial_as_app.sh diff --git a/tests/test_tools.py b/tests/test_tools.py index f14512e82..952c7b76e 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -6900,6 +6900,19 @@ def test_each_refusal_stops_before_anything_is_asked(self): class Redaction(unittest.TestCase): """`tools/redact_trial.py` runs on the raw capture, outside the repo, before anyone reads it.""" + def test_a_status_line_of_spaces_is_read_in_time_and_reads_as_before(self): + # The phrase's spaces and the trailing ones overlapped, so a status line with a run of them and then what no + # status holds took time in its square: a crafted capture could hold the redactor up. + line = redact_trial.STATUS_LINE.fullmatch + read_in_linear_cpu(self, lambda spaces: line("HTTP/1.1 200 a" + spaces + "!"), " ", 16000) + read_in_linear_cpu(self, lambda words: line("HTTP/1.1 200 " + words + "!"), "a ", 16000) + for text, reads in (("HTTP/1.1 200", True), ("HTTP/1.1 200 ", True), ("HTTP/2 404 Not Found", True), + ("HTTP/1.1 200 OK \r", True), ("HTTP/1.1 200 a", True), ("HTTP/1.1 200 a ", True), + ("HTTP/1.1 200 a -", True), ("HTTP/1.1 200 it's", True), ("HTTP/1.1 200 a '", True), + ("HTTP/1.1 200 OK!", False), ("HTTP/1.1 200 -a", False), ("HTTP/1.1 200 a", False), + ("HTTP/1.1 200 a\nb", False), ("HTTP/1.1 20", False), ("http/1.1 200", False)): + self.assertEqual(line(text) is not None, reads, text) + def redact(self, raw: str) -> tuple: out, said = io.StringIO(), io.StringIO() self.assertEqual(redact_trial.main([], stdin=io.BytesIO(raw.encode()), stdout=out, stderr=said), 0) diff --git a/tools/check_context.py b/tools/check_context.py index 9127bcba6..d7e8241fd 100644 --- a/tools/check_context.py +++ b/tools/check_context.py @@ -50,7 +50,10 @@ # A title: an opening quote, eight characters or more, and a closing one. Only the first opening quote after a # closing one or a line's start is tried, since a later one inside the same run is shorter and fails if it does; # trying every one took time in the square of the length of a line of opening quotes left unclosed (issue 369). -TITLE = re.compile(r"(?:\"|(?:^|(?<=[”\n]))[^\"“”\n]*“)[^\"”\n]{8,}[\"”]") +# S8786 reads it as super-linear. It is not: +# tests/test_tools.py::test_a_line_of_unclosed_quotes_is_read_in_time_and_a_title_is_found_as_before +# holds its CPU linear on the shape it feared. +TITLE = re.compile(r"(?:\"|(?:^|(?<=[”\n]))[^\"“”\n]*“)[^\"”\n]{8,}[\"”]") # NOSONAR def _text(path: pathlib.Path) -> str: diff --git a/tools/mutants.py b/tools/mutants.py index c82536b7a..fde2637cc 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -7138,6 +7138,12 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("a failed Engine is the Role's fault", 'harness/apply.py', '"item" if reason in THE_ROLES or result["status"] != "failed" else "environment"', '"item"', ('tests.test_apply.Apply.test_a_result_not_the_runs_or_not_completed_writes_nothing',)), + ("a status line's spaces may be read in time in their square", 'tools/redact_trial.py', + "[A-Za-z](?:[A-Za-z '-]*[A-Za-z'-])?)?", "[A-Za-z][A-Za-z '-]*)?", + ('tests.test_tools.Redaction.test_a_status_line_of_spaces_is_read_in_time_and_reads_as_before',)), + ("a status line's phrase may end only in a letter", 'tools/redact_trial.py', + "[A-Za-z '-]*[A-Za-z'-])?)?", "[A-Za-z '-]*[A-Za-z])?)?", + ('tests.test_tools.Redaction.test_a_status_line_of_spaces_is_read_in_time_and_reads_as_before',)), ("an Engine past its time is the item's fault", 'harness/apply.py', 'THE_ROLES = ("result_invalid", "task_too_large")', 'THE_ROLES = ("result_invalid", "task_too_large", "engine_timeout")', ('tests.test_apply.Apply.test_a_result_not_the_runs_or_not_completed_writes_nothing',)), diff --git a/tools/redact_trial.py b/tools/redact_trial.py index c87291c59..af8272354 100644 --- a/tools/redact_trial.py +++ b/tools/redact_trial.py @@ -33,7 +33,8 @@ # A status line in any case or indent, wherever a body is read: a head the reading did not find as one. ANY_STATUS = re.compile(r"\s*HTTP/[0-9.]+\s+[0-9]{3}\b", re.I) # A status line and nothing else: the version, the code and a reason phrase of words alone, as HTTP writes one. -STATUS_LINE = re.compile(r"HTTP/[0-9.]+ [0-9]{3}(?: [A-Za-z][A-Za-z '-]*)?\s*") +# The phrase ends in no space, so that it and the trailing space share none: a run of spaces is read in time. +STATUS_LINE = re.compile(r"HTTP/[0-9.]+ [0-9]{3}(?: [A-Za-z](?:[A-Za-z '-]*[A-Za-z'-])?)?\s*") # A quoted name and a colon, as a field of JSON is written inside text: one named for a credential is never kept. QUOTED_FIELD = re.compile(r'\\?"([^"\\]{1,64})\\?"\s*:') # its quotes escaped once, too # The same in single quotes, each kind read on its own, so the other quote inside a name is part of it. From 046635d36dccade4b57aa3fc9e01d98aa6dc1071 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 31 Oct 2026 13:09:20 -0500 Subject: [PATCH 2/3] The redactor reads base64 nested inside keys in time, and the status line's test holds a hyphen The reads. `_credentials` read each key twice, and `shaped` reads what it decodes through `_credentials`, so the work doubled at each level of base64 inside a key while the text grew by a third: 5,600 characters took over four seconds. That is worse than the regex this pull request set out to fix, and it is on the same path. A key is now read once, with a test and a mutant. STATUS_LINE's test had no hyphen inside a phrase, so a mutant that dropped it survived. It has one now, with that mutant. Co-Authored-By: Claude Opus 5.5 --- tests/test_tools.py | 19 +++++++++++++++++++ tools/check_trials.py | 7 +++++-- tools/mutants.py | 8 +++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/tests/test_tools.py b/tests/test_tools.py index 952c7b76e..3f2aeb0f6 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -6900,6 +6900,24 @@ def test_each_refusal_stops_before_anything_is_asked(self): class Redaction(unittest.TestCase): """`tools/redact_trial.py` runs on the raw capture, outside the repo, before anyone reads it.""" + def test_base64_nested_inside_keys_is_read_in_time_and_a_credential_inside_is_still_seen(self): + # A key was read twice at each level, and each reading reads what it decodes, so the work doubled per level + # while the text grew by a third. Sixteen levels, 5,600 characters, took four seconds of CPU and more. + def nested(inner: str, levels: int) -> str: + for _ in range(levels): + inner = base64.b64encode(json.dumps({inner: 1}).encode()).decode() + return inner + text = nested("abcdefghijklmnopqrstuvwxyz", 16) + with support.Spent() as spent: + self.assertEqual(check_trials._credentials({"body": text}, "$"), []) + self.assertLess(spent.cpu, 1.0) + # One level is read through, at any place: its key is a credential's shape. + hidden = nested("ghp_" + "a" * 36, 1) + self.assertTrue(check_trials.shaped(hidden)) + said = check_trials._credentials({hidden: 1}, "$") + self.assertEqual(said, [("$.%s" % check_trials.KEY_SHAPED, + "is a key shaped like a credential; leave it out, and list where in removed")]) + def test_a_status_line_of_spaces_is_read_in_time_and_reads_as_before(self): # The phrase's spaces and the trailing ones overlapped, so a status line with a run of them and then what no # status holds took time in its square: a crafted capture could hold the redactor up. @@ -6909,6 +6927,7 @@ def test_a_status_line_of_spaces_is_read_in_time_and_reads_as_before(self): for text, reads in (("HTTP/1.1 200", True), ("HTTP/1.1 200 ", True), ("HTTP/2 404 Not Found", True), ("HTTP/1.1 200 OK \r", True), ("HTTP/1.1 200 a", True), ("HTTP/1.1 200 a ", True), ("HTTP/1.1 200 a -", True), ("HTTP/1.1 200 it's", True), ("HTTP/1.1 200 a '", True), + ("HTTP/1.1 203 Non-Authoritative Information", True), ("HTTP/1.1 207 Multi-Status", True), ("HTTP/1.1 200 OK!", False), ("HTTP/1.1 200 -a", False), ("HTTP/1.1 200 a", False), ("HTTP/1.1 200 a\nb", False), ("HTTP/1.1 20", False), ("http/1.1 200", False)): self.assertEqual(line(text) is not None, reads, text) diff --git a/tools/check_trials.py b/tools/check_trials.py index d0e8c631a..2351f2132 100644 --- a/tools/check_trials.py +++ b/tools/check_trials.py @@ -207,11 +207,14 @@ def _credentials(value, where: str) -> list: out = [] if isinstance(value, dict): for key, item in value.items(): - shown = KEY_SHAPED if shaped(key) else key # a place is never named by a credential + # Read once: shaped reads what it decodes through this function, so a second reading doubled the work at each + # level of base64 inside a key, and a capture of a few kilobytes held up the redactor for a minute. + key_is_shaped = shaped(key) + shown = KEY_SHAPED if key_is_shaped else key # a place is never named by a credential here = "%s.%s" % (where, shown) if where else shown if refused_key(key): out.append((here, "is a field for a credential; leave it out, and list where in removed")) - elif shaped(key): + elif key_is_shaped: out.append((here, "is a key shaped like a credential; leave it out, and list where in removed")) out += _credentials(item, here) elif isinstance(value, list): diff --git a/tools/mutants.py b/tools/mutants.py index fde2637cc..363a7992c 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -5837,7 +5837,10 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: "REGISTRATION.match(text.strip()):", "REGISTRATION.match(text):", ("tests.test_tools.Trials.test_a_credential_in_a_fixture_is_caught",)), ("the model's text is a kept field", P13, 'KEPT_RESULT="type subtype', 'KEPT_RESULT="result type subtype', (P13_TESTS + "test_no_model_text_reaches_out_whether_or_not_the_output_validates",)), - ("a key shaped like a credential is kept", "tools/check_trials.py", " elif shaped(key):\n", " elif False:\n", + ("a key is read twice at each level of base64", "tools/check_trials.py", " elif key_is_shaped:\n", + " elif shaped(key):\n", + ("tests.test_tools.Redaction.test_base64_nested_inside_keys_is_read_in_time_and_a_credential_inside_is_still_seen",)), + ("a key shaped like a credential is kept", "tools/check_trials.py", " elif key_is_shaped:\n", " elif False:\n", ("tests.test_tools.Trials.test_a_credential_in_a_fixture_is_caught",)), ("the redaction keeps a key shaped like a credential", "tools/redact_trial.py", " if refused_key(key) or _held(key) or (isinstance(item, str) and _held(item)):\n", @@ -7141,6 +7144,9 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("a status line's spaces may be read in time in their square", 'tools/redact_trial.py', "[A-Za-z](?:[A-Za-z '-]*[A-Za-z'-])?)?", "[A-Za-z][A-Za-z '-]*)?", ('tests.test_tools.Redaction.test_a_status_line_of_spaces_is_read_in_time_and_reads_as_before',)), + ("a status line's phrase may hold no hyphen", 'tools/redact_trial.py', + "[A-Za-z '-]*[A-Za-z'-])?)?", "[A-Za-z ']*[A-Za-z'-])?)?", + ('tests.test_tools.Redaction.test_a_status_line_of_spaces_is_read_in_time_and_reads_as_before',)), ("a status line's phrase may end only in a letter", 'tools/redact_trial.py', "[A-Za-z '-]*[A-Za-z'-])?)?", "[A-Za-z '-]*[A-Za-z])?)?", ('tests.test_tools.Redaction.test_a_status_line_of_spaces_is_read_in_time_and_reads_as_before',)), From ca4029b052c136df4cecdd2845a1e8a133fae6cd Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 31 Oct 2026 14:54:39 -0500 Subject: [PATCH 3/3] A test of the reader's script makes its own scratch directory, so a parallel run cannot remove it Every test of ReaderTrialScript cleans up what INSIDE names, and a mutant shard's baseline deals a class's methods to parts run at once. So another method's clean-up could remove `__pycache__/scratch-of-the-reader` while this test used it: mktemp failed, the script exited 1 and not 2, and the shard failed before any mutant ran, on no fault of the pull request. Co-Authored-By: Claude Opus 5.5 --- tests/test_tools.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_tools.py b/tests/test_tools.py index d89261f63..868059bd9 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -8682,7 +8682,7 @@ def setUp(self): # lends to the next mutant, so each is removed after the test whether or not it was refused. # The scratch directory is one a test does make, so it stands where the walk leaves out: a test running beside this one # copies the tree, and a directory that goes while it copies fails that test. - INSIDE = ("never-made-by-the-reader", "never-made-by-the-reader-through-a-link", "__pycache__/scratch-of-the-reader") + INSIDE = ("never-made-by-the-reader", "never-made-by-the-reader-through-a-link") def tearDown(self): for name in self.INSIDE: @@ -8768,8 +8768,10 @@ def test_a_mint_answered_whole_by_a_transfer_that_failed_is_revoked_and_told(sel def test_a_scratch_directory_inside_the_repo_is_refused_before_anything_is_minted(self): # The raw answers are kept outside the repo, as OUT is: a TMPDIR inside it, or reached through a link into it, # is refused and the directory mktemp made is removed (Codex on PR 298). - inside = ROOT / self.INSIDE[2] - inside.mkdir(parents=True) + # Of its own: this class's clean-up removes what INSIDE names, and a run deals its methods to parts run at once. + (ROOT / "__pycache__").mkdir(exist_ok=True) + inside = pathlib.Path(tempfile.mkdtemp(dir=str(ROOT / "__pycache__"), prefix="scratch-of-the-reader-")) + self.addCleanup(shutil.rmtree, inside, ignore_errors=True) link = pathlib.Path(tempfile.mkdtemp(dir=SCRATCH.name)).resolve() / "into-the-repo" link.symlink_to(inside) for tmpdir in (inside, link):