From b02ece0002b287790408c7ac305a74bf9872e84f Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:39:26 -0600 Subject: [PATCH 1/6] A Run's outcome says whether its Engine step started, and the run record carries it (#757's first part, record 0200) The run job hands on its Engine step's outcome; apply writes engine_started into outcome/1, false only where GitHub skipped the step, and leaves it out where the job said nothing. The records port carries it into the run record and keeps it with a kept outcome. Nothing reads it yet. The record number is provisional. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/run-jobs.yml | 1 + adapters/artifacts.py | 7 +- adapters/records.py | 40 ++++++++++- controller/__main__.py | 3 +- ...me-says-whether-its-engine-step-started.md | 30 +++++++++ harness/apply_step.py | 23 +++++++ schemas/outcome.schema.json | 4 ++ schemas/run-record.schema.json | 4 ++ tests/test_adapters.py | 67 ++++++++++++++++++- tests/test_controller.py | 10 ++- tests/test_steps.py | 32 +++++++++ tests/test_tools.py | 7 ++ tools/check_workflows.py | 6 +- tools/mutants.py | 58 ++++++++++++++-- 14 files changed, 273 insertions(+), 19 deletions(-) create mode 100644 docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md diff --git a/.github/workflows/run-jobs.yml b/.github/workflows/run-jobs.yml index 287800d0..d2e824a4 100644 --- a/.github/workflows/run-jobs.yml +++ b/.github/workflows/run-jobs.yml @@ -26,6 +26,7 @@ jobs: outputs: commit: ${{ steps.commit.outputs.commit }} digest: ${{ steps.engine.outputs.digest }} + engine: ${{ steps.engine.outcome }} artifact: ${{ steps.result.outputs.artifact-id }} reader: ${{ steps.reader.outputs.artifact-id }} steps: diff --git a/adapters/artifacts.py b/adapters/artifacts.py index 0172ea3d..79859855 100644 --- a/adapters/artifacts.py +++ b/adapters/artifacts.py @@ -114,9 +114,10 @@ def read(transport, repo: str, github_id: int, seen=None, name: str = NAME, memb return held, held is not None -def outcome(transport, repo: str, github_id: int, run: str, seen=None) -> tuple: +def outcome(transport, repo: str, github_id: int, run: str, seen=None, started=None) -> tuple: """`(outcome, whole)` of the Run GitHub numbers `github_id` and the Factory names `run`: the fault its outcome - names, None for a Run that left none, and whether it was read whole. `seen` is handed the list's answer.""" + names, None for a Run that left none, and whether it was read whole. `seen` is handed the list's answer, and + `started` whether the Run's Engine step started, where its outcome says (issue 757).""" held, whole = read(transport, repo, github_id, seen) if held is None: return None, whole @@ -127,6 +128,8 @@ def outcome(transport, repo: str, github_id: int, run: str, seen=None) -> tuple: record = None if not isinstance(record, dict) or validate(record, SCHEMA) or record["run"] != run or record["outcome"] is None: return None, False + if started is not None and "engine_started" in record: + started.append(record["engine_started"]) return record["outcome"], True diff --git a/adapters/records.py b/adapters/records.py index dd6ef191..91a8b70a 100644 --- a/adapters/records.py +++ b/adapters/records.py @@ -299,8 +299,34 @@ def read(record) -> dict: return {k: v for k, v in last.items() if v is not None or (k in before and before[k] is None)} +def engines(previous: list) -> dict: + """Whether each Run's Engine step started, as the pass records before this one read it with its outcome, by its + Run id and when its attempt started (issue 757): the last two at most, and the earlier's word only for a Run the + later does not list. Only a true or a false is kept: an outcome is written once, so what it said stands for its + attempt. A key a record lists twice is kept from neither listing, and nothing from what lists no runs.""" + def said_in(record) -> tuple: + rows = record.get("runs") if isinstance(record, dict) else None + if not isinstance(rows, list): + return {}, [] + said, keys = {}, [] + for row in rows: + key = (row.get("run"), row.get("started_at")) if isinstance(row, dict) else (None, None) + if not (isinstance(key[0], str) and isinstance(key[1], str)): + continue + keys.append(key) + if type(row.get("engine_started")) is bool: + said[key] = row["engine_started"] + return {key: started for key, started in said.items() if keys.count(key) == 1}, keys + held = {} + for record in (previous[-2:] if isinstance(previous, list) else []): + said, keys = said_in(record) + held = {key: started for key, started in held.items() if key not in keys} + held.update(said) + return held + + def runs(transport, repo: str, branch: str, workflows: list, dispatcher: tuple, per_page: int = PAGE, - sleep=time.sleep, kept=None, began=None) -> dict: + sleep=time.sleep, kept=None, began=None, engines=None) -> dict: """The run records of the Factory's Role workflows, the ids of counted Runs not yet named, whether the list was read whole, and each read's date and age for the pass record. `dispatcher` is the pass job's own token's account, by its id and type, `(id, "Bot")`: only a run it dispatched is a Run. @@ -314,11 +340,14 @@ def runs(transport, repo: str, branch: str, workflows: list, dispatcher: tuple, GitHub run its name names (issue 338). Nothing calls `known` until the pass's entry, the effects port's next part (#193), hands it the records it placed. + `engine_started`, where a failed Run's outcome says it, is whether its Engine step started; `engines` is what + `engines` read of it from the pass records before, kept with an outcome `kept` holds (issue 757). + `began` is what `begun` read from the pass records before, and None asks no job: no record then says when it began. Given, each Run in progress or completed has its jobs read for `began_at` (`_began`), but one `began` holds, for a row that names when its attempt started, once in this list. A Run past the `JOBS` asked, or whose jobs cannot be read, has no `began_at` and leaves the list not whole.""" - kept = kept or {} + kept, engines = kept or {}, engines or {} roles = {".github/workflows/%s" % name for name in workflows} def read() -> tuple: @@ -357,10 +386,15 @@ def read() -> tuple: for record, github_id, keyed in failed: if keyed and (record["run"], record["started_at"]) in kept: record["outcome"] = kept[(record["run"], record["started_at"])] + if (record["run"], record["started_at"]) in engines: + record["engine_started"] = engines[(record["run"], record["started_at"])] continue + told = [] record["outcome"], read_whole = artifacts.outcome( transport, repo, github_id, record["run"], - lambda answer: reads.append(dict(read_at(answer), read="outcome"))) + lambda answer: reads.append(dict(read_at(answer), read="outcome")), told) + if told: + record["engine_started"] = told[0] whole = whole and read_whole asked = 0 for record, github_id, attempt, keyed in ran if began is not None else []: diff --git a/controller/__main__.py b/controller/__main__.py index dc545e5f..57b6a6f9 100644 --- a/controller/__main__.py +++ b/controller/__main__.py @@ -156,7 +156,8 @@ def read(transport, config: dict, run_id: str, previous, now: str, clock, starte earlier = pass_.earlier(reads_.previous_records(previous), now) or [] try: runs = records.runs(transport, home, branch, run_files(controller), dispatcher(controller), - kept=records.known(earlier), began=records.begun(earlier)) + kept=records.known(earlier), began=records.begun(earlier), + engines=records.engines(earlier)) except Unanswered: runs = None try: diff --git a/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md b/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md new file mode 100644 index 00000000..6fd360b9 --- /dev/null +++ b/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md @@ -0,0 +1,30 @@ +# 0200. A Run's outcome says whether its Engine step started + +Status: accepted. The Factory Builder's ruling of 2026-11-12 on "A Run that ended environment does not use up the item's run ceiling" (issue 757): before a Run that ended `environment` may be counted apart from the Run ceiling, the run record must say whether the Run reached the model. This record adds that fact alone. What counts toward which bound is the next record's. + +## Decision + +| What | Decided | Because | +|---|---|---| +| The signal | The `run` job hands on its Engine step's outcome as the output `engine`, `${{ steps.engine.outcome }}` | GitHub skips a step once a step before it has failed, so a Run whose CLI install or read failed shows the Engine step `skipped`. That was every Run of the runner DNS outage of 2026-11-11. The step's outcome is GitHub's own word, set by no Factory code, and one of four | +| What `apply` writes | `outcome/1`'s `engine_started`: false where the output is `skipped`; true where it is `success`, `failure` or `cancelled`; left out where it is anything else or missing | A Run whose Engine step ran may have spent on the model, whatever ended it. One whose output cannot be read is read as one that may have: the field is left out, never false | +| Where it is written | Every `outcome/1` `apply` writes, whatever its outcome. The fallback the last step writes, `apply_failed`, and the reader's `release_moved` carry none | Only `apply` is given the run job's outputs (`NEEDS`) | +| The run record | `engine_started`, as the failed Run's outcome said it, and absent where the outcome said nothing. The records port reads it with the outcome, and keeps it with a kept outcome from the last two pass records, as it keeps when a Run began (decision 0193) | An outcome is written once for its attempt, so what it said stands. Kept outcomes are not read again, so the field must be kept with them, or a later pass would lose it | +| The workflow check | A job may hand on a step's `outcome` as an output, beside a step's own output | It is one of four words, so it cannot become a command, as `tools/check_workflows.py`'s expression rule requires | +| What reads it | Nothing yet: the Kernel's counting is the next record's | One change, one rule | + +## Why + +`environment` covers a CLI install that failed, as in the outage, and a Run stopped at its step limit after the model read the item. Both reach `apply` as `run_failed`. A count that cannot tell them apart either gives up the cost ceiling to a slow item, or holds every item after an outage. Whether the Engine step started is the one thing that splits them, and only the `run` job knows it. + +It fails closed: a Run says it did not start the Engine only where GitHub said the step was skipped. A job killed before it handed on its outputs, an output missing, or a word GitHub has not given before all leave the field out. An older outcome, written before this change, has none. + +## Boundaries + +| Boundary | Check | +|---|---| +| `apply` says the Engine did not start only where the run job hands on `skipped`, that it started where it hands on any word GitHub gives a step that ran, and nothing otherwise; the live workflow hands the Engine step's outcome on, and gives `apply` every job's results and outputs | `tests/test_steps.py::test_apply_says_whether_the_engine_step_started`, `harness/apply_step.py`, `.github/workflows/run-jobs.yml`, `tools/mutants.py`, mutant: "the run job hands on nothing of its Engine step", mutant: "a skipped Engine step says it started", mutant: "an Engine step that ran says nothing", mutant: "any word but skipped says the Engine started", mutant: "an Engine step cancelled says nothing", mutant: "apply leaves out whether the Engine started", mutant: "apply says the Engine did not start where the job said nothing" | +| A job may hand on a step's outcome, and nothing else of a step but its own output | `tests/test_tools.py::test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running`, `tools/check_workflows.py`, `tools/mutants.py`, mutant: "a job's output may be any of its step's", mutant: "a job may not hand on a step's outcome" | +| An outcome's `engine_started` is handed on where it says, and an outcome that says it otherwise is no `outcome/1` | `tests/test_adapters.py::test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1`, `adapters/artifacts.py`, `tools/mutants.py`, mutant: "an outcome hands on nothing of the Engine" | +| The run record carries it as the outcome said it, and keeps it with a kept outcome: the later record's word, the earlier's for a Run the later does not list, a true or a false alone, from neither for a Run a record lists twice | `tests/test_adapters.py::test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt`, `adapters/records.py`, `tools/mutants.py`, mutant: "a Run's outcome read says nothing of its Engine", mutant: "a kept outcome keeps nothing of the Engine", mutant: "anything said of the Engine is kept", mutant: "what a record says of the Engine twice is kept", mutant: "every record's word of the Engine is read, not the last two", mutant: "the earlier record's word of the Engine stands over the later's", mutant: "the earlier record's word of the Engine stands for a Run the later lists", mutant: "a key that is no Run or no start is kept for the Engine" | +| The pass's entry hands the records port what the records before said of the Engine | `tests/test_controller.py::test_the_entry_gathers_what_reconciliation_reads`, `controller/__main__.py`, `tools/mutants.py`, mutant: "the entry keeps nothing of the Engine" | diff --git a/harness/apply_step.py b/harness/apply_step.py index 190f593f..50fc22c6 100644 --- a/harness/apply_step.py +++ b/harness/apply_step.py @@ -68,6 +68,26 @@ def needs_met(needs) -> tuple: return None +def engine_started(needs): + """Whether the `run` job's Engine step started, by the outcome the job hands on for it (issue 757), or None. + + GitHub skips the step once a step before it failed, as the CLI's install or the read did: then the Run spent no + model, and this is False. True for an outcome it gives a step that ran. Anything else, an output missing or not + one of these, says nothing, and the Run is read as one that may have reached the model.""" + try: + said = json.loads(needs) if isinstance(needs, str) else None + except ValueError: + said = None + run = said.get("run") if isinstance(said, dict) else None + outputs = run.get("outputs") if isinstance(run, dict) else None + engine = outputs.get("engine") if isinstance(outputs, dict) else None + if engine == "skipped": + return False + if engine in ("success", "failure", "cancelled"): + return True + return None + + def written(argv: list, environ: dict, base, clock, connect) -> tuple: """`(applied/1, outcome/1)` of the Run.""" name, item, pass_id, found, _ = argv @@ -170,6 +190,9 @@ def step(argv: list, environ: dict, base, clock=time.time, connect=Live) -> str: raise Stop("usage: python3 -m harness.apply_step NAME ITEM PASS RESULT OUT") folder = steps.new_folder(base, argv[4]) done, ended = written(argv, environ, base, clock, connect) + started = engine_started(environ.get("NEEDS")) + if started is not None: + ended = dict(ended, engine_started=started) if shared.errors(done, "applied") or shared.errors(ended, "outcome"): raise Stop("records_unreadable") steps.write(folder, "applied.json", done) diff --git a/schemas/outcome.schema.json b/schemas/outcome.schema.json index e959e9d9..943c3a25 100644 --- a/schemas/outcome.schema.json +++ b/schemas/outcome.schema.json @@ -37,6 +37,10 @@ "null" ], "pattern": "^[a-z_]+$" + }, + "engine_started": { + "description": "Whether the run job's Engine step started, by the outcome the job hands on for it: false where GitHub skipped it because a step before it failed, and the Run spent no model; true where it ran. Absent where the job said neither, and read then as a Run that may have reached the model (issue 757)", + "type": "boolean" } } } diff --git a/schemas/run-record.schema.json b/schemas/run-record.schema.json index b5ac8247..5c5e656b 100644 --- a/schemas/run-record.schema.json +++ b/schemas/run-record.schema.json @@ -81,6 +81,10 @@ "skipped" ] }, + "engine_started": { + "description": "Present where a failed Run's outcome/1 says it: whether the run job's Engine step started. False where GitHub skipped it because a step before it failed, so the Run spent no model; absent where the outcome said neither, and read then as a Run that may have reached the model (issue 757)", + "type": "boolean" + }, "rerun_by_dispatcher": { "description": "Present on a re-run alone, or where GitHub gives no attempt that can be read: whether the pass job's own token, the dispatcher's account, re-ran it, as GitHub's triggering_actor says. A re-run runs nothing (decision 0058); one by that token is a write no Effect records, and alarms (decision 0063); null where its attempt or re-runner cannot be read, and reconciliation is short", "type": [ diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 1decc2fa..16eb6b27 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -1058,7 +1058,7 @@ def read(self, *found, **kw): slept = [] got = records.runs(github.Replay(list(found)), canary(), "main", [WORKFLOW], kw.get("dispatcher", DISPATCHER), per_page=kw.get("per_page", 10), sleep=slept.append, kept=kw.get("kept"), - began=kw.get("began")) + began=kw.get("began"), engines=kw.get("engines")) return got, slept def test_a_run_listed_under_its_workflows_name_is_unidentified_after_one_more_read(self): @@ -1308,6 +1308,58 @@ def record(got): self.assertEqual((records.known(None), records.known(record(first)), records.known([{"runs": None}]), records.known([{"runs": ["x"]}])), ({}, {}, {}, {})) + def test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt(self): # issue 757 + # A failed Run's outcome says whether its Engine step started; the run record carries it as the outcome said it, + # and leaves it out where the outcome said nothing. + done = {"name": NAME, "status": "completed", "conclusion": "failure"} + rows = listing("P-17", 3, rows=done) + github_id = rows["response"]["body"]["workflow_runs"][0]["id"] + key = ("r_01JA6Z", rows["response"]["body"]["workflow_runs"][0]["run_started_at"]) + for said in (False, True): + zipped = outcome_zip(dict(outcome_of("r_01JA6Z", "environment", "run_failed"), engine_started=said)) + got, _ = self.read(rows, *stored(canary(), github_id, zipped)) + self.assertEqual((got["runs"][0]["outcome"], got["runs"][0]["engine_started"], got["complete"]), + ("environment", said, True), said) + self.assertEqual(records.engines([{"runs": got["runs"]}]), {key: said}) + got, _ = self.read(rows, *stored(canary(), github_id, outcome_zip(outcome_of("r_01JA6Z", "environment")))) + self.assertNotIn("engine_started", got["runs"][0]) + self.assertEqual(records.engines([{"runs": got["runs"]}]), {}) + # An outcome kept from the records before is not read again, and what they said of the Engine is kept with it. + kept = {key: "environment"} + for engines, want in (({key: False}, False), ({key: True}, True)): + got, _ = self.read(rows, kept=kept, engines=engines) + self.assertEqual((got["runs"][0]["outcome"], got["runs"][0]["engine_started"]), ("environment", want)) + got, _ = self.read(rows, kept=kept, engines={}) + self.assertNotIn("engine_started", got["runs"][0]) + got, _ = self.read(rows, kept=kept) + self.assertNotIn("engine_started", got["runs"][0]) + # A Run that succeeded, or is not done, has no outcome read, and nothing of the Engine. + for change in ({"name": NAME, "status": "completed", "conclusion": "success"}, + {"name": NAME, "status": "in_progress", "conclusion": None}): + got, _ = self.read(listing("P-17", 3, rows=change), engines={key: False}) + self.assertNotIn("engine_started", got["runs"][0], change) + # What the records before said: the later's word, the earlier's for a Run the later does not list; a true or a + # false alone; and from neither for a Run a record lists twice. + run = {"run": key[0], "started_at": key[1], "status": "completed", "conclusion": "failure", + "outcome": "environment"} + other = dict(run, run="r_01JA70") + later = {"runs": [dict(run, engine_started=True)]} + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, later]), {key: True}) + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": [other]}]), + {key: False}) + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": [run]}]), {}) + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": []}, later]), + {key: True}) + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": []}]), {key: False}) + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": []}, {"runs": []}]), {}) + for said in (None, "false", 0, 1): + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=said)]}]), {}, said) + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)] * 2}]), {}) + for change in ({"run": None}, {"started_at": None}): + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False, **change)]}]), {}, change) + self.assertEqual((records.engines(None), records.engines({"runs": [run]}), records.engines([{"runs": None}]), + records.engines([{"runs": ["x"]}])), ({}, {}, {}, {})) + def began(self, status, *jobs, **kw): """`(the one record, whether the list was whole)` of a Run GitHub lists with `status`, its jobs read as given.""" row = listing("P-17", 3, rows=dict({"name": NAME, "status": status, "run_attempt": 1, @@ -1629,6 +1681,19 @@ def read(self, zipped, **listed): def test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1(self): for fault in ("item", "capacity", "environment", "superseded"): self.assertEqual(self.read(outcome_zip(outcome_of(self.RUN, fault))), (fault, True), fault) + # Whether its Engine step started is handed to `started`, where it says; an outcome that says it otherwise is no + # outcome/1 (issue 757). + for said in (True, False): + told = [] + zipped = outcome_zip(dict(outcome_of(self.RUN, "environment"), engine_started=said)) + self.assertEqual(artifacts.outcome(github.Replay(stored("o/r", 7, zipped)), "o/r", 7, self.RUN, started=told), + ("environment", True)) + self.assertEqual(told, [said]) + told = [] + self.assertEqual(artifacts.outcome(github.Replay(stored("o/r", 7, outcome_zip(outcome_of(self.RUN, "item")))), + "o/r", 7, self.RUN, started=told), ("item", True)) + self.assertEqual(told, []) + self.assertEqual(self.read(outcome_zip(dict(outcome_of(self.RUN, "item"), engine_started="no"))), (None, False)) # A listing that gives no digest is read by its size alone. zipped = outcome_zip(outcome_of(self.RUN, "item")) self.assertEqual(self.read(zipped, digest=None), ("item", True)) diff --git a/tests/test_controller.py b/tests/test_controller.py index ca10d9ca..29c58071 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -1157,10 +1157,12 @@ def written_read(transport, repo, branch, poller, own, since, known): # When a Run began, as the records before read it from its jobs, is handed to the records port (decision 0193): # the last record's word, and the one before's for a Run the last does not list. before["runs"][0]["began_at"] = "2026-10-11T12:05:04Z" + # So is whether each Run's Engine step started, as the records before read it with its outcome (issue 757). + before["runs"][0]["engine_started"] = False listed = entry.records.runs def runs(*args, **kw): - efforts["began"] = kw["began"] + efforts["began"], efforts["engines"] = kw["began"], kw["engines"] return listed(*args, **kw) before["reconciled"] = {"runs": [{"id": "r_01JA00", "at": "2026-10-12T11:00:00Z"}], "passes": [{"id": 40, "attempt": 1, "at": "2026-10-12T11:00:00Z"}], "keys": [], "items": [], @@ -1168,8 +1170,8 @@ def runs(*args, **kw): # The last of two records says what has left: the one before it names another that has not been served since. older = dict(before, now="2026-10-12T10:00:00Z", reconciled=dict(before["reconciled"], owed=[], runs=[]), pull_requests=[dict(served[0], facts=dict(served[0]["facts"], item=dict(served[0]["facts"]["item"], number=12)))], - runs=[dict(before["runs"][0], began_at="2026-10-11T12:05:01Z"), - dict(before["runs"][0], run="r_01JA5Y", began_at="2026-10-11T12:05:09Z")]) + runs=[dict(before["runs"][0], began_at="2026-10-11T12:05:01Z", engine_started=True), + dict(before["runs"][0], run="r_01JA5Y", began_at="2026-10-11T12:05:09Z", engine_started=True)]) table = self.table(numbers=[7, 8]) pull = copy.deepcopy(exchanges("P-17")[4]["response"]) pull["headers"] = {"date": "Mon, 12 Oct 2026 11:59:30 GMT", "age": "2"} @@ -1185,6 +1187,8 @@ def runs(*args, **kw): self.assertEqual(code, 0) self.assertEqual(efforts["began"], {("r_01JA6Z", "2026-10-11T12:05:00Z"): "2026-10-11T12:05:04Z", ("r_01JA5Y", "2026-10-11T12:05:00Z"): "2026-10-11T12:05:09Z"}) + self.assertEqual(efforts["engines"], {("r_01JA6Z", "2026-10-11T12:05:00Z"): False, + ("r_01JA5Y", "2026-10-11T12:05:00Z"): True}) since = "2026-10-11T06:00:00Z" item = lambda n: {"repo": self.HOME, "type": "pull_request", "number": n} identities, apps = spied[0][2], spied[0][3] diff --git a/tests/test_steps.py b/tests/test_steps.py index 07e117ec..d9959510 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -597,6 +597,38 @@ def test_apply_waits_for_its_jobs(self): self.assertEqual((code, ended["outcome"], ended["reason"], calls["minted"]), (0, "environment", want, []), needs) + def test_apply_says_whether_the_engine_step_started(self): # issue 757 + # The run job hands on its Engine step's outcome. GitHub skips that step once a step before it failed, as the + # CLI's install did in the outage: then the Run spent no model, and its outcome says `engine_started` false. + # The run job hands on its Engine step's outcome, and `apply` is given every job's results and outputs. + live = (ROOT / ".github" / "workflows" / "run-jobs.yml").read_text() + for line in (" engine: ${{ steps.engine.outcome }}\n", " - name: Run the Engine\n id: engine\n", + " NEEDS: ${{ toJSON(needs) }}\n"): + self.assertEqual(live.count(line), 1, line) + + def needs(run_result, engine, verify="skipped"): + run = {"result": run_result, "outputs": {} if engine is None else {"engine": engine}} + return json.dumps({"run": run, "verify": {"result": verify, "outputs": {}}}) + for engine, run_result, started in (("skipped", "failure", False), ("failure", "failure", True), + ("cancelled", "cancelled", True), ("success", "failure", True)): + code, _, ended, calls = self.go({"NEEDS": needs(run_result, engine)}) + self.assertEqual((code, ended["outcome"], ended["reason"], ended.get("engine_started"), calls["minted"]), + (0, "environment", "run_failed", started, []), engine) + # A Run whose run job succeeded reached the Engine, whatever ends it after. + code, _, ended, _ = self.go({"NEEDS": needs("success", "success", verify="failure")}) + self.assertEqual((ended["reason"], ended.get("engine_started")), ("verify_failed", True)) + code, printed, ended, _ = self.go({"NEEDS": needs("success", "success", verify="success")}) + self.assertEqual((code, ended["outcome"], ended.get("engine_started")), (0, None, True), printed) + # What does not say the Engine step was skipped, or ran, says nothing: the field is left out, and the Run is read + # as one that may have reached the model. + for engine in (None, "", "neutral", 0, True, ["skipped"]): + code, _, ended, _ = self.go({"NEEDS": needs("failure", engine)}) + self.assertEqual((ended["reason"], "engine_started" in ended), ("run_failed", False), engine) + odd = json.dumps({"run": {"result": "failure", "outputs": "skipped"}, "verify": {"result": "skipped"}}) + for said in (odd, json.dumps({"run": {"result": "failure"}, "verify": {"result": "skipped"}}), "not json", None): + code, _, ended, _ = self.go({"NEEDS": said}) + self.assertNotIn("engine_started", ended, said) + def test_apply_reads_with_reads(self): code, printed, ended, calls = self.go() self.assertEqual((code, ended["outcome"]), (0, None), printed) diff --git a/tests/test_tools.py b/tests/test_tools.py index 8ae2965f..839634a1 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -3542,6 +3542,13 @@ def test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running(self): self.assertNotIn("expression", {rule for rule, _ in check_workflows.lint(outputs)}) self.assertIn("expression", {rule for rule, _ in check_workflows.lint(outputs.replace( "${{ steps.run.outputs.digest }}", "${{ github.event.pull_request.title }}"))}) + # A step's outcome is one of four words GitHub gives, so a job may hand it on (issue 757); nothing else of a step. + self.assertNotIn("expression", {rule for rule, _ in check_workflows.lint(outputs.replace( + "${{ steps.run.outputs.digest }}", "${{ steps.run.outcome }}"))}) + for other in ("${{ steps.run.conclusion }}", "${{ steps.run.outcome }}x", "${{ steps.run.outcome.x }}", + "${{ steps.run.outcome || github.event.pull_request.title }}"): + self.assertIn("expression", {rule for rule, _ in check_workflows.lint(outputs.replace( + "${{ steps.run.outputs.digest }}", other))}, other) def run_tree(self, text: str, account, name: str = "run.yml") -> list: config = json.loads((ROOT / "config" / "controller.json").read_text()) diff --git a/tools/check_workflows.py b/tools/check_workflows.py index 73f7178e..f1bcc1a0 100644 --- a/tools/check_workflows.py +++ b/tools/check_workflows.py @@ -98,13 +98,14 @@ WAKES, RUN_WORKFLOW = "poller.yml", "sf-run" # The Run's lane (issue 196's ruling 6): one concurrency group for each Work item, on the one job that calls the Run's # jobs, so that it holds from run to apply and never cancels what runs. And the three places an expression may stand -# whole beyond env:, with: and secrets:: the run's name, the lane, and a job's output of one of its steps. +# whole beyond env:, with: and secrets:: the run's name, the lane, and a job's output of one of its steps: a step's own +# output, or its outcome, which GitHub gives as one of four words (issue 757). RUN_JOBS = "run-jobs.yml" APPROVE_JOB = "approve-job.yml" QUEUE_JOB = "queue-job.yml" LANE = "${{ inputs.item }}" RUN_NAME = "${{ inputs.name }}" -OUTPUT = re.compile(r"^\$\{\{ steps\.[a-z][\w-]*\.outputs\.[a-z][\w-]* \}\}$") +OUTPUT = re.compile(r"^\$\{\{ steps\.[a-z][\w-]*\.(?:outputs\.[a-z][\w-]*|outcome) \}\}$") # The Run's shim, as the ruling gives it in full: its dispatcher is the pass's own token, by the account controller/1 # names for it, on the run's first attempt alone, since a re-run keeps the dispatcher's id though a person started it; # and a Pause of the repo or the org starts no Run, any value but unset or exactly false a Pause (decision 0003; I-34). @@ -334,6 +335,7 @@ outputs: commit: ${{ steps.commit.outputs.commit }} digest: ${{ steps.engine.outputs.digest }} + engine: ${{ steps.engine.outcome }} artifact: ${{ steps.result.outputs.artifact-id }} reader: ${{ steps.reader.outputs.artifact-id }} steps: diff --git a/tools/mutants.py b/tools/mutants.py index adff618a..2c3d48b4 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -4346,8 +4346,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' key = (record["run"], record["started_at"])\n attempts[key] = attempts.get(key, 0) + 1\n ids.setdefault(record["run"], []).append(row["id"])\n if record["status"] == "completed" and record["conclusion"] != "success":\n failed.append((record, row["id"], bool(row.get("run_started_at"))))\n', ' key = (record["run"], record["started_at"])\n ids.setdefault(record["run"], []).append(row["id"])\n if record["status"] == "completed" and record["conclusion"] != "success":\n attempts[key] = attempts.get(key, 0) + 1\n failed.append((record, row["id"], bool(row.get("run_started_at"))))\n', ('tests.test_adapters.Records.test_an_outcome_is_read_once_and_kept_by_its_run_and_attempt',)), ('an outcome read is not stamped', 'adapters/records.py', - ' lambda answer: reads.append(dict(read_at(answer), read="outcome")))', - ' lambda answer: None)', ('tests.test_adapters',)), + ' lambda answer: reads.append(dict(read_at(answer), read="outcome")), told)', + ' lambda answer: None, told)', ('tests.test_adapters',)), # Decision 0193: when a Run began is read from its jobs, under P-64's rule. ('a job that waits says when its Run began', 'adapters/records.py', 'if j.get("status") == "in_progress"\n', @@ -4494,15 +4494,15 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("the earlier record's beginning stands over the later's", 'adapters/records.py', ' for record in (list(previous[-2:]) if isinstance(previous, list) else []):', ' for record in (list(reversed(previous[-2:])) if isinstance(previous, list) else []):', KEPT_BEGUN), - ('the entry asks the records port no job', 'controller/__main__.py', 'began=records.begun(earlier))', 'began=None)', WIRE_ENTRY), + ('the entry asks the records port no job', 'controller/__main__.py', 'began=records.begun(earlier),', 'began=None,', WIRE_ENTRY), ('a pass with nothing kept asks the records port no job', 'controller/__main__.py', - 'began=records.begun(earlier))', 'began=records.begun(earlier) or None)', WIRE_ENTRY), + 'began=records.begun(earlier),', 'began=records.begun(earlier) or None,', WIRE_ENTRY), ('the entry keeps beginnings from records as they were downloaded', 'controller/__main__.py', - 'began=records.begun(earlier))', 'began=records.begun(previous))', WIRE_ENTRY), + 'began=records.begun(earlier),', 'began=records.begun(previous),', WIRE_ENTRY), ("the entry keeps the earlier record's beginnings alone", 'controller/__main__.py', - 'began=records.begun(earlier))', 'began=records.begun(earlier[:1]))', WIRE_ENTRY), + 'began=records.begun(earlier),', 'began=records.begun(earlier[:1]),', WIRE_ENTRY), ("the entry keeps the last record's beginnings alone", 'controller/__main__.py', - 'began=records.begun(earlier))', 'began=records.begun(earlier[-1:]))', WIRE_ENTRY), + 'began=records.begun(earlier),', 'began=records.begun(earlier[-1:]),', WIRE_ENTRY), ('an outcome list that cannot be read is not stamped', 'adapters/artifacts.py', ' if seen is not None:\n seen(answer)\n listed, total = answer["body"].get("artifacts"), answer["body"].get("total_count")\n if not isinstance(listed, list) or not isinstance(total, int) or isinstance(total, bool) or total != len(listed):\n return False\n', ' listed, total = answer["body"].get("artifacts"), answer["body"].get("total_count")\n if not isinstance(listed, list) or not isinstance(total, int) or isinstance(total, bool) or total != len(listed):\n return False\n if seen is not None:\n seen(answer)\n', @@ -10976,6 +10976,50 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("the pass checks out the tag again", '.github/workflows/pass.yml', ' ref: ${{ job.workflow_sha }}\n', ' ref: factory/v0-next\n', ('tests.test_tools.Workflows.test_the_pass_uploads_its_record_before_it_dispatches',)), + # Issue 757: the run record says whether a Run's Engine step started, from the outcome apply writes. + ("the run job hands on nothing of its Engine step", '.github/workflows/run-jobs.yml', + ' engine: ${{ steps.engine.outcome }}\n', '', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("a skipped Engine step says it started", 'harness/apply_step.py', + ' if engine == "skipped":\n return False\n', ' if engine == "skipped":\n return True\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("an Engine step that ran says nothing", 'harness/apply_step.py', + ' if engine in ("success", "failure", "cancelled"):\n', ' if False:\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("any word but skipped says the Engine started", 'harness/apply_step.py', + ' if engine in ("success", "failure", "cancelled"):\n', ' if engine is not None:\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("an Engine step cancelled says nothing", 'harness/apply_step.py', + '("success", "failure", "cancelled")', '("success", "failure")', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("apply leaves out whether the Engine started", 'harness/apply_step.py', + ' ended = dict(ended, engine_started=started)\n', ' pass\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("apply says the Engine did not start where the job said nothing", 'harness/apply_step.py', + ' if started is not None:\n ended = dict(ended, engine_started=started)\n', + ' if True:\n ended = dict(ended, engine_started=bool(started))\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("a job's output may be any of its step's", 'tools/check_workflows.py', + '\\.(?:outputs\\.[a-z][\\w-]*|outcome) \\}\\}$")', '\\.(?:outputs\\.[a-z][\\w-]*|[a-z]+) \\}\\}$")', ('tests.test_tools.Workflows.test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running',)), + ("a job may not hand on a step's outcome", 'tools/check_workflows.py', + '\\.(?:outputs\\.[a-z][\\w-]*|outcome) \\}\\}$")', '\\.(?:outputs\\.[a-z][\\w-]*) \\}\\}$")', ('tests.test_tools.Workflows.test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running',)), + ("an outcome hands on nothing of the Engine", 'adapters/artifacts.py', + ' started.append(record["engine_started"])\n', ' pass\n', ('tests.test_adapters.Artifacts.test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1',)), + ("a Run's outcome read says nothing of its Engine", 'adapters/records.py', + ' if told:\n record["engine_started"] = told[0]\n', + ' if False:\n record["engine_started"] = told[0]\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("a kept outcome keeps nothing of the Engine", 'adapters/records.py', + ' if (record["run"], record["started_at"]) in engines:\n', + ' if False:\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("anything said of the Engine is kept", 'adapters/records.py', + ' if type(row.get("engine_started")) is bool:\n', ' if "engine_started" in row:\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("what a record says of the Engine twice is kept", 'adapters/records.py', + 'for key, started in said.items() if keys.count(key) == 1}', 'for key, started in said.items()}', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("every record's word of the Engine is read, not the last two", 'adapters/records.py', + ' for record in (previous[-2:] if isinstance(previous, list) else []):\n', + ' for record in (previous if isinstance(previous, list) else []):\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("the earlier record's word of the Engine stands over the later's", 'adapters/records.py', + ' for record in (previous[-2:] if isinstance(previous, list) else []):\n', + ' for record in (list(reversed(previous[-2:])) if isinstance(previous, list) else []):\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("the earlier record's word of the Engine stands for a Run the later lists", 'adapters/records.py', + ' held = {key: started for key, started in held.items() if key not in keys}\n', '', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("a key that is no Run or no start is kept for the Engine", 'adapters/records.py', + ' if not (isinstance(key[0], str) and isinstance(key[1], str)):\n continue\n', '', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("the entry keeps nothing of the Engine", 'controller/__main__.py', + 'engines=records.engines(earlier))', 'engines=None)', WIRE_ENTRY), ("the Run's first job checks out the tag again", '.github/workflows/run-jobs.yml', ' ref: ${{ job.workflow_sha }}\n', ' ref: factory/v0-next\n', ('tests.test_tools.Workflows.test_the_commit_a_later_job_checks_out_is_the_releases_own',)), From a46e5c7a68980ecf6044348fb1bca99253569657 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:48:34 -0600 Subject: [PATCH 2/6] The Engine's word is kept from the last pass record alone, and each part of it is held by a test (#757's first part, record 0200) The Claude review's findings: engines() read an earlier record whose word could never reach a run record; the entry's reading from records dated before the pass, the run record's boolean and the outcome read's guard were held by no test. Record 0200 names the rollback cost and what is not verified on GitHub. Co-Authored-By: Claude Opus 5.5 --- adapters/records.py | 16 ++++++---------- ...ome-says-whether-its-engine-step-started.md | 10 ++++++---- .../run-record.an-engine-start-in-words.json | 17 +++++++++++++++++ tests/test_adapters.py | 18 ++++++++---------- tests/test_controller.py | 6 +++--- tools/mutants.py | 16 ++++++++-------- 6 files changed, 48 insertions(+), 35 deletions(-) create mode 100644 examples/invalid/run-record.an-engine-start-in-words.json diff --git a/adapters/records.py b/adapters/records.py index 91a8b70a..4b2c5cde 100644 --- a/adapters/records.py +++ b/adapters/records.py @@ -300,10 +300,11 @@ def read(record) -> dict: def engines(previous: list) -> dict: - """Whether each Run's Engine step started, as the pass records before this one read it with its outcome, by its - Run id and when its attempt started (issue 757): the last two at most, and the earlier's word only for a Run the - later does not list. Only a true or a false is kept: an outcome is written once, so what it said stands for its - attempt. A key a record lists twice is kept from neither listing, and nothing from what lists no runs.""" + """Whether each Run's Engine step started, as the last pass record before this one read it with its outcome, by + its Run id and when its attempt started (issue 757). It is read only beside an outcome `known` keeps, and `known` + keeps a fault from the last record alone, so no earlier record's word is needed. Only a true or a false is kept: + an outcome is written once, so what it said stands for its attempt. A key the record lists twice is not kept, and + nothing is kept from what lists no runs.""" def said_in(record) -> tuple: rows = record.get("runs") if isinstance(record, dict) else None if not isinstance(rows, list): @@ -317,12 +318,7 @@ def said_in(record) -> tuple: if type(row.get("engine_started")) is bool: said[key] = row["engine_started"] return {key: started for key, started in said.items() if keys.count(key) == 1}, keys - held = {} - for record in (previous[-2:] if isinstance(previous, list) else []): - said, keys = said_in(record) - held = {key: started for key, started in held.items() if key not in keys} - held.update(said) - return held + return said_in(previous[-1])[0] if isinstance(previous, list) and previous else {} def runs(transport, repo: str, branch: str, workflows: list, dispatcher: tuple, per_page: int = PAGE, diff --git a/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md b/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md index 6fd360b9..832580b7 100644 --- a/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md +++ b/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md @@ -9,7 +9,7 @@ Status: accepted. The Factory Builder's ruling of 2026-11-12 on "A Run that ende | The signal | The `run` job hands on its Engine step's outcome as the output `engine`, `${{ steps.engine.outcome }}` | GitHub skips a step once a step before it has failed, so a Run whose CLI install or read failed shows the Engine step `skipped`. That was every Run of the runner DNS outage of 2026-11-11. The step's outcome is GitHub's own word, set by no Factory code, and one of four | | What `apply` writes | `outcome/1`'s `engine_started`: false where the output is `skipped`; true where it is `success`, `failure` or `cancelled`; left out where it is anything else or missing | A Run whose Engine step ran may have spent on the model, whatever ended it. One whose output cannot be read is read as one that may have: the field is left out, never false | | Where it is written | Every `outcome/1` `apply` writes, whatever its outcome. The fallback the last step writes, `apply_failed`, and the reader's `release_moved` carry none | Only `apply` is given the run job's outputs (`NEEDS`) | -| The run record | `engine_started`, as the failed Run's outcome said it, and absent where the outcome said nothing. The records port reads it with the outcome, and keeps it with a kept outcome from the last two pass records, as it keeps when a Run began (decision 0193) | An outcome is written once for its attempt, so what it said stands. Kept outcomes are not read again, so the field must be kept with them, or a later pass would lose it | +| The run record | `engine_started`, a true or a false alone, as the failed Run's outcome said it, and absent where the outcome said nothing. The records port reads it with the outcome, and keeps it with a kept outcome from the last pass record, the one `known` keeps a fault from | An outcome is written once for its attempt, so what it said stands. Kept outcomes are not read again, so the field must be kept with them, or a later pass would lose it | | The workflow check | A job may hand on a step's `outcome` as an output, beside a step's own output | It is one of four words, so it cannot become a command, as `tools/check_workflows.py`'s expression rule requires | | What reads it | Nothing yet: the Kernel's counting is the next record's | One change, one rule | @@ -19,12 +19,14 @@ Status: accepted. The Factory Builder's ruling of 2026-11-12 on "A Run that ende It fails closed: a Run says it did not start the Engine only where GitHub said the step was skipped. A job killed before it handed on its outputs, an output missing, or a word GitHub has not given before all leave the field out. An older outcome, written before this change, has none. +It costs one thing, accepted. `outcome/1` admits no field it does not name, so after a rollback of `next` to a Release cut before this change, that Release reads a newer Run's outcome as no `outcome/1`, and its list of Runs is not whole until those Runs leave the window. Pass records' run records already carry such a field, `began_at` (decision 0193). Not verified on GitHub: that the job output reads `skipped` for a step skipped after an earlier one failed. GitHub's documentation says so, and if it reads otherwise the field is left out. + ## Boundaries | Boundary | Check | |---|---| | `apply` says the Engine did not start only where the run job hands on `skipped`, that it started where it hands on any word GitHub gives a step that ran, and nothing otherwise; the live workflow hands the Engine step's outcome on, and gives `apply` every job's results and outputs | `tests/test_steps.py::test_apply_says_whether_the_engine_step_started`, `harness/apply_step.py`, `.github/workflows/run-jobs.yml`, `tools/mutants.py`, mutant: "the run job hands on nothing of its Engine step", mutant: "a skipped Engine step says it started", mutant: "an Engine step that ran says nothing", mutant: "any word but skipped says the Engine started", mutant: "an Engine step cancelled says nothing", mutant: "apply leaves out whether the Engine started", mutant: "apply says the Engine did not start where the job said nothing" | | A job may hand on a step's outcome, and nothing else of a step but its own output | `tests/test_tools.py::test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running`, `tools/check_workflows.py`, `tools/mutants.py`, mutant: "a job's output may be any of its step's", mutant: "a job may not hand on a step's outcome" | -| An outcome's `engine_started` is handed on where it says, and an outcome that says it otherwise is no `outcome/1` | `tests/test_adapters.py::test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1`, `adapters/artifacts.py`, `tools/mutants.py`, mutant: "an outcome hands on nothing of the Engine" | -| The run record carries it as the outcome said it, and keeps it with a kept outcome: the later record's word, the earlier's for a Run the later does not list, a true or a false alone, from neither for a Run a record lists twice | `tests/test_adapters.py::test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt`, `adapters/records.py`, `tools/mutants.py`, mutant: "a Run's outcome read says nothing of its Engine", mutant: "a kept outcome keeps nothing of the Engine", mutant: "anything said of the Engine is kept", mutant: "what a record says of the Engine twice is kept", mutant: "every record's word of the Engine is read, not the last two", mutant: "the earlier record's word of the Engine stands over the later's", mutant: "the earlier record's word of the Engine stands for a Run the later lists", mutant: "a key that is no Run or no start is kept for the Engine" | -| The pass's entry hands the records port what the records before said of the Engine | `tests/test_controller.py::test_the_entry_gathers_what_reconciliation_reads`, `controller/__main__.py`, `tools/mutants.py`, mutant: "the entry keeps nothing of the Engine" | +| An outcome's `engine_started` is handed on where the caller asks and the outcome says, and an outcome that says it otherwise is no `outcome/1` | `tests/test_adapters.py::test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1`, `adapters/artifacts.py`, `tools/mutants.py`, mutant: "an outcome hands on nothing of the Engine", mutant: "a caller that does not ask is handed the Engine" | +| The run record carries it as the outcome said it, and keeps it with a kept outcome: the last record's word alone, a true or a false alone, and nothing for a Run the record lists twice | `tests/test_adapters.py::test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt`, `adapters/records.py`, `tools/mutants.py`, mutant: "a Run's outcome read says nothing of its Engine", mutant: "a kept outcome keeps nothing of the Engine", mutant: "anything said of the Engine is kept", mutant: "what a record says of the Engine twice is kept", mutant: "the earlier record's word of the Engine is read", mutant: "a key that is no Run or no start is kept for the Engine" | +| The pass's entry hands the records port what the records before said of the Engine, from pass records alone, each from before the pass | `tests/test_controller.py::test_the_entry_gathers_what_reconciliation_reads`, `controller/__main__.py`, `tools/mutants.py`, mutant: "the entry keeps nothing of the Engine", mutant: "the entry keeps the Engine from records as they were downloaded" | diff --git a/examples/invalid/run-record.an-engine-start-in-words.json b/examples/invalid/run-record.an-engine-start-in-words.json new file mode 100644 index 00000000..b54b6789 --- /dev/null +++ b/examples/invalid/run-record.an-engine-start-in-words.json @@ -0,0 +1,17 @@ +{ + "run": "r_01JA6Z", + "item": { + "repo": "combsmith/software-factory", + "type": "pull_request", + "number": 42 + }, + "role": "pr-review", + "intent": "review-change", + "content_id": "sha256:5d0c", + "status": "completed", + "conclusion": "success", + "outcome": null, + "started_at": "2026-10-11T12:05:00Z", + "began_at": "2026-10-11T12:05:04Z", + "engine_started": "no" +} diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 16eb6b27..22ff9dcb 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -1338,27 +1338,22 @@ def test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run {"name": NAME, "status": "in_progress", "conclusion": None}): got, _ = self.read(listing("P-17", 3, rows=change), engines={key: False}) self.assertNotIn("engine_started", got["runs"][0], change) - # What the records before said: the later's word, the earlier's for a Run the later does not list; a true or a - # false alone; and from neither for a Run a record lists twice. + # What the last record before said, and no earlier record's word: a true or a false alone, and nothing for a Run + # it lists twice. run = {"run": key[0], "started_at": key[1], "status": "completed", "conclusion": "failure", "outcome": "environment"} other = dict(run, run="r_01JA70") later = {"runs": [dict(run, engine_started=True)]} self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, later]), {key: True}) - self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": [other]}]), - {key: False}) - self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": [run]}]), {}) - self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": []}, later]), - {key: True}) - self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": []}]), {key: False}) - self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": []}, {"runs": []}]), {}) + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": [other]}]), {}) + self.assertEqual(records.engines([later, {"runs": [run]}]), {}) for said in (None, "false", 0, 1): self.assertEqual(records.engines([{"runs": [dict(run, engine_started=said)]}]), {}, said) self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)] * 2}]), {}) for change in ({"run": None}, {"started_at": None}): self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False, **change)]}]), {}, change) self.assertEqual((records.engines(None), records.engines({"runs": [run]}), records.engines([{"runs": None}]), - records.engines([{"runs": ["x"]}])), ({}, {}, {}, {})) + records.engines([{"runs": ["x"]}]), records.engines([])), ({}, {}, {}, {}, {})) def began(self, status, *jobs, **kw): """`(the one record, whether the list was whole)` of a Run GitHub lists with `status`, its jobs read as given.""" @@ -1694,6 +1689,9 @@ def test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1(self): "o/r", 7, self.RUN, started=told), ("item", True)) self.assertEqual(told, []) self.assertEqual(self.read(outcome_zip(dict(outcome_of(self.RUN, "item"), engine_started="no"))), (None, False)) + # A caller that does not ask is told nothing, and the outcome is read all the same. + self.assertEqual(self.read(outcome_zip(dict(outcome_of(self.RUN, "environment"), engine_started=False))), + ("environment", True)) # A listing that gives no digest is read by its size alone. zipped = outcome_zip(outcome_of(self.RUN, "item")) self.assertEqual(self.read(zipped, digest=None), ("item", True)) diff --git a/tests/test_controller.py b/tests/test_controller.py index 29c58071..98412590 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -1187,8 +1187,7 @@ def runs(*args, **kw): self.assertEqual(code, 0) self.assertEqual(efforts["began"], {("r_01JA6Z", "2026-10-11T12:05:00Z"): "2026-10-11T12:05:04Z", ("r_01JA5Y", "2026-10-11T12:05:00Z"): "2026-10-11T12:05:09Z"}) - self.assertEqual(efforts["engines"], {("r_01JA6Z", "2026-10-11T12:05:00Z"): False, - ("r_01JA5Y", "2026-10-11T12:05:00Z"): True}) + self.assertEqual(efforts["engines"], {("r_01JA6Z", "2026-10-11T12:05:00Z"): False}) since = "2026-10-11T06:00:00Z" item = lambda n: {"repo": self.HOME, "type": "pull_request", "number": n} identities, apps = spied[0][2], spied[0][3] @@ -1261,7 +1260,7 @@ def whole(transport, item, kept, identities, apps, since): # Beginnings are kept from pass records alone, each from before this pass. With one that is neither, or with # none, the port is still asked, with nothing kept: every Run's jobs are read afresh. for held in ([older, dict(before, now="2026-10-12T12:00:00Z")], [older, dict(before, release=7)], None): - del efforts["began"] + del efforts["began"], efforts["engines"] with unittest.mock.patch.object(entry.facts_, "snapshot", snapshot), \ unittest.mock.patch.object(entry.passes, "records", lambda *a: held), \ unittest.mock.patch.object(entry.seen_, "read", whole), unittest.mock.patch.object(entry.seen_, "pushes", pushes), \ @@ -1270,6 +1269,7 @@ def whole(transport, item, kept, identities, apps, since): unittest.mock.patch.object(entry.passes, "written", written_read): self.run_entry(Hub(self.table(numbers=[7]))) self.assertEqual(efforts["began"], {}, held and held[1]["now"]) + self.assertEqual(efforts["engines"], {}, held and held[1]["now"]) # With no record before it, nothing is known, so everything in the window is read. with unittest.mock.patch.object(entry.facts_, "snapshot", snapshot), \ unittest.mock.patch.object(entry.passes, "records", lambda *a: None), \ diff --git a/tools/mutants.py b/tools/mutants.py index 2c3d48b4..30af5a10 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -11008,16 +11008,16 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' if type(row.get("engine_started")) is bool:\n', ' if "engine_started" in row:\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), ("what a record says of the Engine twice is kept", 'adapters/records.py', 'for key, started in said.items() if keys.count(key) == 1}', 'for key, started in said.items()}', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), - ("every record's word of the Engine is read, not the last two", 'adapters/records.py', - ' for record in (previous[-2:] if isinstance(previous, list) else []):\n', - ' for record in (previous if isinstance(previous, list) else []):\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), - ("the earlier record's word of the Engine stands over the later's", 'adapters/records.py', - ' for record in (previous[-2:] if isinstance(previous, list) else []):\n', - ' for record in (list(reversed(previous[-2:])) if isinstance(previous, list) else []):\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), - ("the earlier record's word of the Engine stands for a Run the later lists", 'adapters/records.py', - ' held = {key: started for key, started in held.items() if key not in keys}\n', '', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), ("a key that is no Run or no start is kept for the Engine", 'adapters/records.py', ' if not (isinstance(key[0], str) and isinstance(key[1], str)):\n continue\n', '', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("the earlier record's word of the Engine is read", 'adapters/records.py', + ' return said_in(previous[-1])[0] if isinstance(previous, list) and previous else {}\n', + ' return said_in(previous[0])[0] if isinstance(previous, list) and previous else {}\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("a caller that does not ask is handed the Engine", 'adapters/artifacts.py', + ' if started is not None and "engine_started" in record:\n', ' if "engine_started" in record:\n', + ('tests.test_adapters.Artifacts.test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1',)), + ("the entry keeps the Engine from records as they were downloaded", 'controller/__main__.py', + 'engines=records.engines(earlier))', 'engines=records.engines(previous))', WIRE_ENTRY), ("the entry keeps nothing of the Engine", 'controller/__main__.py', 'engines=records.engines(earlier))', 'engines=None)', WIRE_ENTRY), ("the Run's first job checks out the tag again", '.github/workflows/run-jobs.yml', From dad973285462102308abd27ce07afc29631b0a5d Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:23:46 -0600 Subject: [PATCH 3/6] Each word an Engine step that ran may give has a mutant of its own (#757's first part, record 0200) Carbon's finding on b02ece00. Co-Authored-By: Claude Opus 5.5 --- ...200-a-runs-outcome-says-whether-its-engine-step-started.md | 2 +- tools/mutants.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md b/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md index 832580b7..da6e916f 100644 --- a/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md +++ b/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md @@ -25,7 +25,7 @@ It costs one thing, accepted. `outcome/1` admits no field it does not name, so a | Boundary | Check | |---|---| -| `apply` says the Engine did not start only where the run job hands on `skipped`, that it started where it hands on any word GitHub gives a step that ran, and nothing otherwise; the live workflow hands the Engine step's outcome on, and gives `apply` every job's results and outputs | `tests/test_steps.py::test_apply_says_whether_the_engine_step_started`, `harness/apply_step.py`, `.github/workflows/run-jobs.yml`, `tools/mutants.py`, mutant: "the run job hands on nothing of its Engine step", mutant: "a skipped Engine step says it started", mutant: "an Engine step that ran says nothing", mutant: "any word but skipped says the Engine started", mutant: "an Engine step cancelled says nothing", mutant: "apply leaves out whether the Engine started", mutant: "apply says the Engine did not start where the job said nothing" | +| `apply` says the Engine did not start only where the run job hands on `skipped`, that it started where it hands on any word GitHub gives a step that ran, and nothing otherwise; the live workflow hands the Engine step's outcome on, and gives `apply` every job's results and outputs | `tests/test_steps.py::test_apply_says_whether_the_engine_step_started`, `harness/apply_step.py`, `.github/workflows/run-jobs.yml`, `tools/mutants.py`, mutant: "the run job hands on nothing of its Engine step", mutant: "a skipped Engine step says it started", mutant: "an Engine step that ran says nothing", mutant: "any word but skipped says the Engine started", mutant: "an Engine step cancelled says nothing", mutant: "an Engine step that failed says nothing", mutant: "an Engine step that succeeded says nothing", mutant: "apply leaves out whether the Engine started", mutant: "apply says the Engine did not start where the job said nothing" | | A job may hand on a step's outcome, and nothing else of a step but its own output | `tests/test_tools.py::test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running`, `tools/check_workflows.py`, `tools/mutants.py`, mutant: "a job's output may be any of its step's", mutant: "a job may not hand on a step's outcome" | | An outcome's `engine_started` is handed on where the caller asks and the outcome says, and an outcome that says it otherwise is no `outcome/1` | `tests/test_adapters.py::test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1`, `adapters/artifacts.py`, `tools/mutants.py`, mutant: "an outcome hands on nothing of the Engine", mutant: "a caller that does not ask is handed the Engine" | | The run record carries it as the outcome said it, and keeps it with a kept outcome: the last record's word alone, a true or a false alone, and nothing for a Run the record lists twice | `tests/test_adapters.py::test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt`, `adapters/records.py`, `tools/mutants.py`, mutant: "a Run's outcome read says nothing of its Engine", mutant: "a kept outcome keeps nothing of the Engine", mutant: "anything said of the Engine is kept", mutant: "what a record says of the Engine twice is kept", mutant: "the earlier record's word of the Engine is read", mutant: "a key that is no Run or no start is kept for the Engine" | diff --git a/tools/mutants.py b/tools/mutants.py index 30af5a10..ffa885c1 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -10987,6 +10987,10 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' if engine in ("success", "failure", "cancelled"):\n', ' if engine is not None:\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), ("an Engine step cancelled says nothing", 'harness/apply_step.py', '("success", "failure", "cancelled")', '("success", "failure")', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("an Engine step that failed says nothing", 'harness/apply_step.py', + '("success", "failure", "cancelled")', '("success", "cancelled")', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("an Engine step that succeeded says nothing", 'harness/apply_step.py', + '("success", "failure", "cancelled")', '("failure", "cancelled")', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), ("apply leaves out whether the Engine started", 'harness/apply_step.py', ' ended = dict(ended, engine_started=started)\n', ' pass\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), ("apply says the Engine did not start where the job said nothing", 'harness/apply_step.py', From ed1a1f64debe1297735c0906645b221c48d693c3 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:28:06 -0600 Subject: [PATCH 4/6] A kept outcome takes what was said of its own Run and attempt alone (#757's first part, record 0200) Astra's finding on dad97328. Co-Authored-By: Claude Opus 5.5 --- ...200-a-runs-outcome-says-whether-its-engine-step-started.md | 2 +- tests/test_adapters.py | 4 ++++ tools/mutants.py | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md b/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md index da6e916f..56be4b22 100644 --- a/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md +++ b/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md @@ -28,5 +28,5 @@ It costs one thing, accepted. `outcome/1` admits no field it does not name, so a | `apply` says the Engine did not start only where the run job hands on `skipped`, that it started where it hands on any word GitHub gives a step that ran, and nothing otherwise; the live workflow hands the Engine step's outcome on, and gives `apply` every job's results and outputs | `tests/test_steps.py::test_apply_says_whether_the_engine_step_started`, `harness/apply_step.py`, `.github/workflows/run-jobs.yml`, `tools/mutants.py`, mutant: "the run job hands on nothing of its Engine step", mutant: "a skipped Engine step says it started", mutant: "an Engine step that ran says nothing", mutant: "any word but skipped says the Engine started", mutant: "an Engine step cancelled says nothing", mutant: "an Engine step that failed says nothing", mutant: "an Engine step that succeeded says nothing", mutant: "apply leaves out whether the Engine started", mutant: "apply says the Engine did not start where the job said nothing" | | A job may hand on a step's outcome, and nothing else of a step but its own output | `tests/test_tools.py::test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running`, `tools/check_workflows.py`, `tools/mutants.py`, mutant: "a job's output may be any of its step's", mutant: "a job may not hand on a step's outcome" | | An outcome's `engine_started` is handed on where the caller asks and the outcome says, and an outcome that says it otherwise is no `outcome/1` | `tests/test_adapters.py::test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1`, `adapters/artifacts.py`, `tools/mutants.py`, mutant: "an outcome hands on nothing of the Engine", mutant: "a caller that does not ask is handed the Engine" | -| The run record carries it as the outcome said it, and keeps it with a kept outcome: the last record's word alone, a true or a false alone, and nothing for a Run the record lists twice | `tests/test_adapters.py::test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt`, `adapters/records.py`, `tools/mutants.py`, mutant: "a Run's outcome read says nothing of its Engine", mutant: "a kept outcome keeps nothing of the Engine", mutant: "anything said of the Engine is kept", mutant: "what a record says of the Engine twice is kept", mutant: "the earlier record's word of the Engine is read", mutant: "a key that is no Run or no start is kept for the Engine" | +| The run record carries it as the outcome said it, and keeps it with a kept outcome, by its own Run and attempt: the last record's word alone, a true or a false alone, and nothing for a Run the record lists twice | `tests/test_adapters.py::test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt`, `adapters/records.py`, `tools/mutants.py`, mutant: "a Run's outcome read says nothing of its Engine", mutant: "a kept outcome keeps nothing of the Engine", mutant: "a kept outcome takes the Engine of another Run", mutant: "anything said of the Engine is kept", mutant: "what a record says of the Engine twice is kept", mutant: "the earlier record's word of the Engine is read", mutant: "a key that is no Run or no start is kept for the Engine" | | The pass's entry hands the records port what the records before said of the Engine, from pass records alone, each from before the pass | `tests/test_controller.py::test_the_entry_gathers_what_reconciliation_reads`, `controller/__main__.py`, `tools/mutants.py`, mutant: "the entry keeps nothing of the Engine", mutant: "the entry keeps the Engine from records as they were downloaded" | diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 22ff9dcb..44bcfddf 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -1331,6 +1331,10 @@ def test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run self.assertEqual((got["runs"][0]["outcome"], got["runs"][0]["engine_started"]), ("environment", want)) got, _ = self.read(rows, kept=kept, engines={}) self.assertNotIn("engine_started", got["runs"][0]) + # What was said of another Run, or of another attempt of this one, says nothing of this one. + for other in (("r_01JA70", key[1]), (key[0], "2026-10-11T12:06:00Z")): + got, _ = self.read(rows, kept=kept, engines={other: False}) + self.assertNotIn("engine_started", got["runs"][0], other) got, _ = self.read(rows, kept=kept) self.assertNotIn("engine_started", got["runs"][0]) # A Run that succeeded, or is not done, has no outcome read, and nothing of the Engine. diff --git a/tools/mutants.py b/tools/mutants.py index ffa885c1..f455d1e2 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -11008,6 +11008,9 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("a kept outcome keeps nothing of the Engine", 'adapters/records.py', ' if (record["run"], record["started_at"]) in engines:\n', ' if False:\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("a kept outcome takes the Engine of another Run", 'adapters/records.py', + ' if (record["run"], record["started_at"]) in engines:\n record["engine_started"] = engines[(record["run"], record["started_at"])]\n', + ' if engines:\n record["engine_started"] = list(engines.values())[0]\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), ("anything said of the Engine is kept", 'adapters/records.py', ' if type(row.get("engine_started")) is bool:\n', ' if "engine_started" in row:\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), ("what a record says of the Engine twice is kept", 'adapters/records.py', From c4818d31966eb2806fe0c7a782dd46ea4e47802a Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:37:40 -0600 Subject: [PATCH 5/6] Record 0200 is 0201, by the Factory Builder's order (#757's first part, record 0201) Co-Authored-By: Claude Opus 5.5 --- ...0201-a-runs-outcome-says-whether-its-engine-step-started.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/decisions/{0200-a-runs-outcome-says-whether-its-engine-step-started.md => 0201-a-runs-outcome-says-whether-its-engine-step-started.md} (99%) diff --git a/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md b/docs/decisions/0201-a-runs-outcome-says-whether-its-engine-step-started.md similarity index 99% rename from docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md rename to docs/decisions/0201-a-runs-outcome-says-whether-its-engine-step-started.md index 56be4b22..af7328f4 100644 --- a/docs/decisions/0200-a-runs-outcome-says-whether-its-engine-step-started.md +++ b/docs/decisions/0201-a-runs-outcome-says-whether-its-engine-step-started.md @@ -1,4 +1,4 @@ -# 0200. A Run's outcome says whether its Engine step started +# 0201. A Run's outcome says whether its Engine step started Status: accepted. The Factory Builder's ruling of 2026-11-12 on "A Run that ended environment does not use up the item's run ceiling" (issue 757): before a Run that ended `environment` may be counted apart from the Run ceiling, the run record must say whether the Run reached the model. This record adds that fact alone. What counts toward which bound is the next record's. From c0337335b23392e33783847d266f4b4a75c9c662 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:16:48 -0600 Subject: [PATCH 6/6] The expression rule names a step's outcome; the controller's fixture gives the Engine's word on a failed Run (#757's first part, record 0201) Co-Authored-By: Claude Opus 5.5 --- ...201-a-runs-outcome-says-whether-its-engine-step-started.md | 2 +- tests/test_controller.py | 4 ++-- tools/check_workflows.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/decisions/0201-a-runs-outcome-says-whether-its-engine-step-started.md b/docs/decisions/0201-a-runs-outcome-says-whether-its-engine-step-started.md index af7328f4..3d34c910 100644 --- a/docs/decisions/0201-a-runs-outcome-says-whether-its-engine-step-started.md +++ b/docs/decisions/0201-a-runs-outcome-says-whether-its-engine-step-started.md @@ -6,7 +6,7 @@ Status: accepted. The Factory Builder's ruling of 2026-11-12 on "A Run that ende | What | Decided | Because | |---|---|---| -| The signal | The `run` job hands on its Engine step's outcome as the output `engine`, `${{ steps.engine.outcome }}` | GitHub skips a step once a step before it has failed, so a Run whose CLI install or read failed shows the Engine step `skipped`. That was every Run of the runner DNS outage of 2026-11-11. The step's outcome is GitHub's own word, set by no Factory code, and one of four | +| The signal | The `run` job hands on its Engine step's outcome as the output `engine`, `${{ steps.engine.outcome }}` | GitHub skips a step once a step before it has failed, so a Run whose CLI install or read failed shows the Engine step `skipped`. That was every Run of the runner DNS outage of 2026-11-11. The step's outcome is GitHub's own word, set by no Factory code, and one of four values | | What `apply` writes | `outcome/1`'s `engine_started`: false where the output is `skipped`; true where it is `success`, `failure` or `cancelled`; left out where it is anything else or missing | A Run whose Engine step ran may have spent on the model, whatever ended it. One whose output cannot be read is read as one that may have: the field is left out, never false | | Where it is written | Every `outcome/1` `apply` writes, whatever its outcome. The fallback the last step writes, `apply_failed`, and the reader's `release_moved` carry none | Only `apply` is given the run job's outputs (`NEEDS`) | | The run record | `engine_started`, a true or a false alone, as the failed Run's outcome said it, and absent where the outcome said nothing. The records port reads it with the outcome, and keeps it with a kept outcome from the last pass record, the one `known` keeps a fault from | An outcome is written once for its attempt, so what it said stands. Kept outcomes are not read again, so the field must be kept with them, or a later pass would lose it | diff --git a/tests/test_controller.py b/tests/test_controller.py index 29563120..4c7e8c97 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -1157,8 +1157,8 @@ def written_read(transport, repo, branch, poller, own, since, known): # When a Run began, as the records before read it from its jobs, is handed to the records port (decision 0193): # the last record's word, and the one before's for a Run the last does not list. before["runs"][0]["began_at"] = "2026-10-11T12:05:04Z" - # So is whether each Run's Engine step started, as the records before read it with its outcome (issue 757). - before["runs"][0]["engine_started"] = False + # So is whether each failed Run's Engine step started, as the records before read it with its outcome (issue 757). + before["runs"][0].update(conclusion="failure", outcome="environment", engine_started=False) listed = entry.records.runs def runs(*args, **kw): diff --git a/tools/check_workflows.py b/tools/check_workflows.py index f1bcc1a0..36d3b9ef 100644 --- a/tools/check_workflows.py +++ b/tools/check_workflows.py @@ -47,7 +47,7 @@ "wake": "a workflow on workflow_run wakes on the completion alone of the workflows it names, never its own, takes " "no input and reads nothing of the run that woke it", "expression": "an expression stands only as the whole of a value under env:, with: or secrets:, or in if:; or as " - "exactly a Run's name, ${{ inputs.name }}, its lane, ${{ inputs.item }}, or a step's output", + "exactly a Run's name, ${{ inputs.name }}, its lane, ${{ inputs.item }}, or a step's output or its outcome", "model": "no step sets the CLI's environment scrub, which only the Engine sets, beside --tools \"\", and a step that " "holds the model's credential is handed no tool list", "lane": "a job that calls the Run's jobs, or an act's, holds its item's lane: its own concurrency group, "