diff --git a/.github/workflows/run-jobs.yml b/.github/workflows/run-jobs.yml index 4d755a9e..76ab3188 100644 --- a/.github/workflows/run-jobs.yml +++ b/.github/workflows/run-jobs.yml @@ -26,6 +26,7 @@ jobs: outputs: commit: ${{ steps.commit.outputs.commit }} digest: ${{ steps.engine.outputs.digest }} + engine: ${{ steps.engine.outcome }} artifact: ${{ steps.result.outputs.artifact-id }} reader: ${{ steps.reader.outputs.artifact-id }} steps: diff --git a/adapters/artifacts.py b/adapters/artifacts.py index 0172ea3d..79859855 100644 --- a/adapters/artifacts.py +++ b/adapters/artifacts.py @@ -114,9 +114,10 @@ def read(transport, repo: str, github_id: int, seen=None, name: str = NAME, memb return held, held is not None -def outcome(transport, repo: str, github_id: int, run: str, seen=None) -> tuple: +def outcome(transport, repo: str, github_id: int, run: str, seen=None, started=None) -> tuple: """`(outcome, whole)` of the Run GitHub numbers `github_id` and the Factory names `run`: the fault its outcome - names, None for a Run that left none, and whether it was read whole. `seen` is handed the list's answer.""" + names, None for a Run that left none, and whether it was read whole. `seen` is handed the list's answer, and + `started` whether the Run's Engine step started, where its outcome says (issue 757).""" held, whole = read(transport, repo, github_id, seen) if held is None: return None, whole @@ -127,6 +128,8 @@ def outcome(transport, repo: str, github_id: int, run: str, seen=None) -> tuple: record = None if not isinstance(record, dict) or validate(record, SCHEMA) or record["run"] != run or record["outcome"] is None: return None, False + if started is not None and "engine_started" in record: + started.append(record["engine_started"]) return record["outcome"], True diff --git a/adapters/records.py b/adapters/records.py index eec5b902..35e1703f 100644 --- a/adapters/records.py +++ b/adapters/records.py @@ -299,8 +299,30 @@ def read(record) -> dict: return {k: v for k, v in last.items() if v is not None or (k in before and before[k] is None)} +def engines(previous: list) -> dict: + """Whether each Run's Engine step started, as the last pass record before this one read it with its outcome, by + its Run id and when its attempt started (issue 757). It is read only beside an outcome `known` keeps, and `known` + keeps a fault from the last record alone, so no earlier record's word is needed. Only a true or a false is kept: + an outcome is written once, so what it said stands for its attempt. A key the record lists twice is not kept, and + nothing is kept from what lists no runs.""" + def said_in(record) -> tuple: + rows = record.get("runs") if isinstance(record, dict) else None + if not isinstance(rows, list): + return {}, [] + said, keys = {}, [] + for row in rows: + key = (row.get("run"), row.get("started_at")) if isinstance(row, dict) else (None, None) + if not (isinstance(key[0], str) and isinstance(key[1], str)): + continue + keys.append(key) + if type(row.get("engine_started")) is bool: + said[key] = row["engine_started"] + return {key: started for key, started in said.items() if keys.count(key) == 1}, keys + return said_in(previous[-1])[0] if isinstance(previous, list) and previous else {} + + def runs(transport, repo: str, branch: str, workflows: list, dispatcher: tuple, per_page: int = PAGE, - sleep=time.sleep, kept=None, began=None, of=None, on=None) -> dict: + sleep=time.sleep, kept=None, began=None, of=None, on=None, engines=None) -> dict: """The run records of the Factory's Role workflows, the ids of counted Runs not yet named, whether the list was read whole, and each read's date and age for the pass record. `dispatcher` is the pass job's own token's account, by its id and type, `(id, "Bot")`: only a run it dispatched is a Run. @@ -314,6 +336,9 @@ def runs(transport, repo: str, branch: str, workflows: list, dispatcher: tuple, GitHub run its name names (issue 338). Nothing calls `known` until the pass's entry, the effects port's next part (#193), hands it the records it placed. + `engine_started`, where a failed Run's outcome says it, is whether its Engine step started; `engines` is what + `engines` read of it from the pass records before, kept with an outcome `kept` holds (issue 757). + `began` is what `begun` read from the pass records before, and None asks no job: no record then says when it began. Given, each Run in progress or completed has its jobs read for `began_at` (`_began`), but one `began` holds, for a row that names when its attempt started, once in this list. A Run past the `JOBS` asked, or whose @@ -322,7 +347,7 @@ def runs(transport, repo: str, branch: str, workflows: list, dispatcher: tuple, whole. The pass hands the open pull requests, so what it asks is its live work, not the list's history (decision 0196). `on`, an item, asks that item's Runs alone, as an act reading its own item again does, and all of them: a Run keeps nothing from one read to the next, so a cap on its item's Runs would never be worked off.""" - kept = kept or {} + kept, engines = kept or {}, engines or {} roles = {".github/workflows/%s" % name for name in workflows} def read() -> tuple: @@ -361,10 +386,15 @@ def read() -> tuple: for record, github_id, keyed in failed: if keyed and (record["run"], record["started_at"]) in kept: record["outcome"] = kept[(record["run"], record["started_at"])] + if (record["run"], record["started_at"]) in engines: + record["engine_started"] = engines[(record["run"], record["started_at"])] continue + told = [] record["outcome"], read_whole = artifacts.outcome( transport, repo, github_id, record["run"], - lambda answer: reads.append(dict(read_at(answer), read="outcome"))) + lambda answer: reads.append(dict(read_at(answer), read="outcome")), told) + if told: + record["engine_started"] = told[0] whole = whole and read_whole asked = 0 for record, github_id, attempt, keyed in ran if began is not None else []: diff --git a/controller/__main__.py b/controller/__main__.py index 8d3caf06..11491601 100644 --- a/controller/__main__.py +++ b/controller/__main__.py @@ -165,6 +165,7 @@ def read(transport, config: dict, run_id: str, previous, now: str, clock, starte try: runs = records.runs(transport, home, branch, run_files(controller), dispatcher(controller), kept=records.known(earlier), began=records.begun(earlier), + engines=records.engines(earlier), of=[{"repo": home, "type": "pull_request", "number": number} for number in numbers]) except Unanswered: runs = None diff --git a/docs/decisions/0201-a-runs-outcome-says-whether-its-engine-step-started.md b/docs/decisions/0201-a-runs-outcome-says-whether-its-engine-step-started.md new file mode 100644 index 00000000..3d34c910 --- /dev/null +++ b/docs/decisions/0201-a-runs-outcome-says-whether-its-engine-step-started.md @@ -0,0 +1,32 @@ +# 0201. A Run's outcome says whether its Engine step started + +Status: accepted. The Factory Builder's ruling of 2026-11-12 on "A Run that ended environment does not use up the item's run ceiling" (issue 757): before a Run that ended `environment` may be counted apart from the Run ceiling, the run record must say whether the Run reached the model. This record adds that fact alone. What counts toward which bound is the next record's. + +## Decision + +| What | Decided | Because | +|---|---|---| +| The signal | The `run` job hands on its Engine step's outcome as the output `engine`, `${{ steps.engine.outcome }}` | GitHub skips a step once a step before it has failed, so a Run whose CLI install or read failed shows the Engine step `skipped`. That was every Run of the runner DNS outage of 2026-11-11. The step's outcome is GitHub's own word, set by no Factory code, and one of four values | +| What `apply` writes | `outcome/1`'s `engine_started`: false where the output is `skipped`; true where it is `success`, `failure` or `cancelled`; left out where it is anything else or missing | A Run whose Engine step ran may have spent on the model, whatever ended it. One whose output cannot be read is read as one that may have: the field is left out, never false | +| Where it is written | Every `outcome/1` `apply` writes, whatever its outcome. The fallback the last step writes, `apply_failed`, and the reader's `release_moved` carry none | Only `apply` is given the run job's outputs (`NEEDS`) | +| The run record | `engine_started`, a true or a false alone, as the failed Run's outcome said it, and absent where the outcome said nothing. The records port reads it with the outcome, and keeps it with a kept outcome from the last pass record, the one `known` keeps a fault from | An outcome is written once for its attempt, so what it said stands. Kept outcomes are not read again, so the field must be kept with them, or a later pass would lose it | +| The workflow check | A job may hand on a step's `outcome` as an output, beside a step's own output | It is one of four words, so it cannot become a command, as `tools/check_workflows.py`'s expression rule requires | +| What reads it | Nothing yet: the Kernel's counting is the next record's | One change, one rule | + +## Why + +`environment` covers a CLI install that failed, as in the outage, and a Run stopped at its step limit after the model read the item. Both reach `apply` as `run_failed`. A count that cannot tell them apart either gives up the cost ceiling to a slow item, or holds every item after an outage. Whether the Engine step started is the one thing that splits them, and only the `run` job knows it. + +It fails closed: a Run says it did not start the Engine only where GitHub said the step was skipped. A job killed before it handed on its outputs, an output missing, or a word GitHub has not given before all leave the field out. An older outcome, written before this change, has none. + +It costs one thing, accepted. `outcome/1` admits no field it does not name, so after a rollback of `next` to a Release cut before this change, that Release reads a newer Run's outcome as no `outcome/1`, and its list of Runs is not whole until those Runs leave the window. Pass records' run records already carry such a field, `began_at` (decision 0193). Not verified on GitHub: that the job output reads `skipped` for a step skipped after an earlier one failed. GitHub's documentation says so, and if it reads otherwise the field is left out. + +## Boundaries + +| Boundary | Check | +|---|---| +| `apply` says the Engine did not start only where the run job hands on `skipped`, that it started where it hands on any word GitHub gives a step that ran, and nothing otherwise; the live workflow hands the Engine step's outcome on, and gives `apply` every job's results and outputs | `tests/test_steps.py::test_apply_says_whether_the_engine_step_started`, `harness/apply_step.py`, `.github/workflows/run-jobs.yml`, `tools/mutants.py`, mutant: "the run job hands on nothing of its Engine step", mutant: "a skipped Engine step says it started", mutant: "an Engine step that ran says nothing", mutant: "any word but skipped says the Engine started", mutant: "an Engine step cancelled says nothing", mutant: "an Engine step that failed says nothing", mutant: "an Engine step that succeeded says nothing", mutant: "apply leaves out whether the Engine started", mutant: "apply says the Engine did not start where the job said nothing" | +| A job may hand on a step's outcome, and nothing else of a step but its own output | `tests/test_tools.py::test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running`, `tools/check_workflows.py`, `tools/mutants.py`, mutant: "a job's output may be any of its step's", mutant: "a job may not hand on a step's outcome" | +| An outcome's `engine_started` is handed on where the caller asks and the outcome says, and an outcome that says it otherwise is no `outcome/1` | `tests/test_adapters.py::test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1`, `adapters/artifacts.py`, `tools/mutants.py`, mutant: "an outcome hands on nothing of the Engine", mutant: "a caller that does not ask is handed the Engine" | +| The run record carries it as the outcome said it, and keeps it with a kept outcome, by its own Run and attempt: the last record's word alone, a true or a false alone, and nothing for a Run the record lists twice | `tests/test_adapters.py::test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt`, `adapters/records.py`, `tools/mutants.py`, mutant: "a Run's outcome read says nothing of its Engine", mutant: "a kept outcome keeps nothing of the Engine", mutant: "a kept outcome takes the Engine of another Run", mutant: "anything said of the Engine is kept", mutant: "what a record says of the Engine twice is kept", mutant: "the earlier record's word of the Engine is read", mutant: "a key that is no Run or no start is kept for the Engine" | +| The pass's entry hands the records port what the records before said of the Engine, from pass records alone, each from before the pass | `tests/test_controller.py::test_the_entry_gathers_what_reconciliation_reads`, `controller/__main__.py`, `tools/mutants.py`, mutant: "the entry keeps nothing of the Engine", mutant: "the entry keeps the Engine from records as they were downloaded" | diff --git a/examples/invalid/run-record.an-engine-start-in-words.json b/examples/invalid/run-record.an-engine-start-in-words.json new file mode 100644 index 00000000..b54b6789 --- /dev/null +++ b/examples/invalid/run-record.an-engine-start-in-words.json @@ -0,0 +1,17 @@ +{ + "run": "r_01JA6Z", + "item": { + "repo": "combsmith/software-factory", + "type": "pull_request", + "number": 42 + }, + "role": "pr-review", + "intent": "review-change", + "content_id": "sha256:5d0c", + "status": "completed", + "conclusion": "success", + "outcome": null, + "started_at": "2026-10-11T12:05:00Z", + "began_at": "2026-10-11T12:05:04Z", + "engine_started": "no" +} diff --git a/harness/apply_step.py b/harness/apply_step.py index 190f593f..50fc22c6 100644 --- a/harness/apply_step.py +++ b/harness/apply_step.py @@ -68,6 +68,26 @@ def needs_met(needs) -> tuple: return None +def engine_started(needs): + """Whether the `run` job's Engine step started, by the outcome the job hands on for it (issue 757), or None. + + GitHub skips the step once a step before it failed, as the CLI's install or the read did: then the Run spent no + model, and this is False. True for an outcome it gives a step that ran. Anything else, an output missing or not + one of these, says nothing, and the Run is read as one that may have reached the model.""" + try: + said = json.loads(needs) if isinstance(needs, str) else None + except ValueError: + said = None + run = said.get("run") if isinstance(said, dict) else None + outputs = run.get("outputs") if isinstance(run, dict) else None + engine = outputs.get("engine") if isinstance(outputs, dict) else None + if engine == "skipped": + return False + if engine in ("success", "failure", "cancelled"): + return True + return None + + def written(argv: list, environ: dict, base, clock, connect) -> tuple: """`(applied/1, outcome/1)` of the Run.""" name, item, pass_id, found, _ = argv @@ -170,6 +190,9 @@ def step(argv: list, environ: dict, base, clock=time.time, connect=Live) -> str: raise Stop("usage: python3 -m harness.apply_step NAME ITEM PASS RESULT OUT") folder = steps.new_folder(base, argv[4]) done, ended = written(argv, environ, base, clock, connect) + started = engine_started(environ.get("NEEDS")) + if started is not None: + ended = dict(ended, engine_started=started) if shared.errors(done, "applied") or shared.errors(ended, "outcome"): raise Stop("records_unreadable") steps.write(folder, "applied.json", done) diff --git a/schemas/outcome.schema.json b/schemas/outcome.schema.json index e959e9d9..943c3a25 100644 --- a/schemas/outcome.schema.json +++ b/schemas/outcome.schema.json @@ -37,6 +37,10 @@ "null" ], "pattern": "^[a-z_]+$" + }, + "engine_started": { + "description": "Whether the run job's Engine step started, by the outcome the job hands on for it: false where GitHub skipped it because a step before it failed, and the Run spent no model; true where it ran. Absent where the job said neither, and read then as a Run that may have reached the model (issue 757)", + "type": "boolean" } } } diff --git a/schemas/run-record.schema.json b/schemas/run-record.schema.json index b5ac8247..5c5e656b 100644 --- a/schemas/run-record.schema.json +++ b/schemas/run-record.schema.json @@ -81,6 +81,10 @@ "skipped" ] }, + "engine_started": { + "description": "Present where a failed Run's outcome/1 says it: whether the run job's Engine step started. False where GitHub skipped it because a step before it failed, so the Run spent no model; absent where the outcome said neither, and read then as a Run that may have reached the model (issue 757)", + "type": "boolean" + }, "rerun_by_dispatcher": { "description": "Present on a re-run alone, or where GitHub gives no attempt that can be read: whether the pass job's own token, the dispatcher's account, re-ran it, as GitHub's triggering_actor says. A re-run runs nothing (decision 0058); one by that token is a write no Effect records, and alarms (decision 0063); null where its attempt or re-runner cannot be read, and reconciliation is short", "type": [ diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 823e1446..ad359c87 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -1058,7 +1058,7 @@ def read(self, *found, **kw): slept = [] got = records.runs(github.Replay(list(found)), canary(), "main", [WORKFLOW], kw.get("dispatcher", DISPATCHER), per_page=kw.get("per_page", 10), sleep=slept.append, kept=kw.get("kept"), - began=kw.get("began")) + began=kw.get("began"), engines=kw.get("engines")) return got, slept def test_a_run_listed_under_its_workflows_name_is_unidentified_after_one_more_read(self): @@ -1308,6 +1308,57 @@ def record(got): self.assertEqual((records.known(None), records.known(record(first)), records.known([{"runs": None}]), records.known([{"runs": ["x"]}])), ({}, {}, {}, {})) + def test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt(self): # issue 757 + # A failed Run's outcome says whether its Engine step started; the run record carries it as the outcome said it, + # and leaves it out where the outcome said nothing. + done = {"name": NAME, "status": "completed", "conclusion": "failure"} + rows = listing("P-17", 3, rows=done) + github_id = rows["response"]["body"]["workflow_runs"][0]["id"] + key = ("r_01JA6Z", rows["response"]["body"]["workflow_runs"][0]["run_started_at"]) + for said in (False, True): + zipped = outcome_zip(dict(outcome_of("r_01JA6Z", "environment", "run_failed"), engine_started=said)) + got, _ = self.read(rows, *stored(canary(), github_id, zipped)) + self.assertEqual((got["runs"][0]["outcome"], got["runs"][0]["engine_started"], got["complete"]), + ("environment", said, True), said) + self.assertEqual(records.engines([{"runs": got["runs"]}]), {key: said}) + got, _ = self.read(rows, *stored(canary(), github_id, outcome_zip(outcome_of("r_01JA6Z", "environment")))) + self.assertNotIn("engine_started", got["runs"][0]) + self.assertEqual(records.engines([{"runs": got["runs"]}]), {}) + # An outcome kept from the records before is not read again, and what they said of the Engine is kept with it. + kept = {key: "environment"} + for engines, want in (({key: False}, False), ({key: True}, True)): + got, _ = self.read(rows, kept=kept, engines=engines) + self.assertEqual((got["runs"][0]["outcome"], got["runs"][0]["engine_started"]), ("environment", want)) + got, _ = self.read(rows, kept=kept, engines={}) + self.assertNotIn("engine_started", got["runs"][0]) + # What was said of another Run, or of another attempt of this one, says nothing of this one. + for other in (("r_01JA70", key[1]), (key[0], "2026-10-11T12:06:00Z")): + got, _ = self.read(rows, kept=kept, engines={other: False}) + self.assertNotIn("engine_started", got["runs"][0], other) + got, _ = self.read(rows, kept=kept) + self.assertNotIn("engine_started", got["runs"][0]) + # A Run that succeeded, or is not done, has no outcome read, and nothing of the Engine. + for change in ({"name": NAME, "status": "completed", "conclusion": "success"}, + {"name": NAME, "status": "in_progress", "conclusion": None}): + got, _ = self.read(listing("P-17", 3, rows=change), engines={key: False}) + self.assertNotIn("engine_started", got["runs"][0], change) + # What the last record before said, and no earlier record's word: a true or a false alone, and nothing for a Run + # it lists twice. + run = {"run": key[0], "started_at": key[1], "status": "completed", "conclusion": "failure", + "outcome": "environment"} + other = dict(run, run="r_01JA70") + later = {"runs": [dict(run, engine_started=True)]} + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, later]), {key: True}) + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)]}, {"runs": [other]}]), {}) + self.assertEqual(records.engines([later, {"runs": [run]}]), {}) + for said in (None, "false", 0, 1): + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=said)]}]), {}, said) + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False)] * 2}]), {}) + for change in ({"run": None}, {"started_at": None}): + self.assertEqual(records.engines([{"runs": [dict(run, engine_started=False, **change)]}]), {}, change) + self.assertEqual((records.engines(None), records.engines({"runs": [run]}), records.engines([{"runs": None}]), + records.engines([{"runs": ["x"]}]), records.engines([])), ({}, {}, {}, {}, {})) + def began(self, status, *jobs, **kw): """`(the one record, whether the list was whole)` of a Run GitHub lists with `status`, its jobs read as given.""" row = listing("P-17", 3, rows=dict({"name": NAME, "status": status, "run_attempt": 1, @@ -1699,6 +1750,22 @@ def read(self, zipped, **listed): def test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1(self): for fault in ("item", "capacity", "environment", "superseded"): self.assertEqual(self.read(outcome_zip(outcome_of(self.RUN, fault))), (fault, True), fault) + # Whether its Engine step started is handed to `started`, where it says; an outcome that says it otherwise is no + # outcome/1 (issue 757). + for said in (True, False): + told = [] + zipped = outcome_zip(dict(outcome_of(self.RUN, "environment"), engine_started=said)) + self.assertEqual(artifacts.outcome(github.Replay(stored("o/r", 7, zipped)), "o/r", 7, self.RUN, started=told), + ("environment", True)) + self.assertEqual(told, [said]) + told = [] + self.assertEqual(artifacts.outcome(github.Replay(stored("o/r", 7, outcome_zip(outcome_of(self.RUN, "item")))), + "o/r", 7, self.RUN, started=told), ("item", True)) + self.assertEqual(told, []) + self.assertEqual(self.read(outcome_zip(dict(outcome_of(self.RUN, "item"), engine_started="no"))), (None, False)) + # A caller that does not ask is told nothing, and the outcome is read all the same. + self.assertEqual(self.read(outcome_zip(dict(outcome_of(self.RUN, "environment"), engine_started=False))), + ("environment", True)) # A listing that gives no digest is read by its size alone. zipped = outcome_zip(outcome_of(self.RUN, "item")) self.assertEqual(self.read(zipped, digest=None), ("item", True)) diff --git a/tests/test_controller.py b/tests/test_controller.py index 1f423829..4c7e8c97 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -1157,10 +1157,12 @@ def written_read(transport, repo, branch, poller, own, since, known): # When a Run began, as the records before read it from its jobs, is handed to the records port (decision 0193): # the last record's word, and the one before's for a Run the last does not list. before["runs"][0]["began_at"] = "2026-10-11T12:05:04Z" + # So is whether each failed Run's Engine step started, as the records before read it with its outcome (issue 757). + before["runs"][0].update(conclusion="failure", outcome="environment", engine_started=False) listed = entry.records.runs def runs(*args, **kw): - efforts["began"], efforts["of"] = kw["began"], kw["of"] + efforts["began"], efforts["engines"], efforts["of"] = kw["began"], kw["engines"], kw["of"] return listed(*args, **kw) before["reconciled"] = {"runs": [{"id": "r_01JA00", "at": "2026-10-12T11:00:00Z"}], "passes": [{"id": 40, "attempt": 1, "at": "2026-10-12T11:00:00Z"}], "keys": [], "items": [], @@ -1168,8 +1170,8 @@ def runs(*args, **kw): # The last of two records says what has left: the one before it names another that has not been served since. older = dict(before, now="2026-10-12T10:00:00Z", reconciled=dict(before["reconciled"], owed=[], runs=[]), pull_requests=[dict(served[0], facts=dict(served[0]["facts"], item=dict(served[0]["facts"]["item"], number=12)))], - runs=[dict(before["runs"][0], began_at="2026-10-11T12:05:01Z"), - dict(before["runs"][0], run="r_01JA5Y", began_at="2026-10-11T12:05:09Z")]) + runs=[dict(before["runs"][0], began_at="2026-10-11T12:05:01Z", engine_started=True), + dict(before["runs"][0], run="r_01JA5Y", began_at="2026-10-11T12:05:09Z", engine_started=True)]) table = self.table(numbers=[7, 8]) pull = copy.deepcopy(exchanges("P-17")[4]["response"]) pull["headers"] = {"date": "Mon, 12 Oct 2026 11:59:30 GMT", "age": "2"} @@ -1185,6 +1187,7 @@ def runs(*args, **kw): self.assertEqual(code, 0) self.assertEqual(efforts["began"], {("r_01JA6Z", "2026-10-11T12:05:00Z"): "2026-10-11T12:05:04Z", ("r_01JA5Y", "2026-10-11T12:05:00Z"): "2026-10-11T12:05:09Z"}) + self.assertEqual(efforts["engines"], {("r_01JA6Z", "2026-10-11T12:05:00Z"): False}) # The port asks when a Run began only of the open pull requests' Runs, which are listed first (decision 0196). self.assertEqual(efforts["of"], [{"repo": self.HOME, "type": "pull_request", "number": n} for n in (7, 8)]) since = "2026-10-11T06:00:00Z" @@ -1259,7 +1262,7 @@ def whole(transport, item, kept, identities, apps, since): # Beginnings are kept from pass records alone, each from before this pass. With one that is neither, or with # none, the port is still asked, with nothing kept: every Run's jobs are read afresh. for held in ([older, dict(before, now="2026-10-12T12:00:00Z")], [older, dict(before, release=7)], None): - del efforts["began"] + del efforts["began"], efforts["engines"] with unittest.mock.patch.object(entry.facts_, "snapshot", snapshot), \ unittest.mock.patch.object(entry.passes, "records", lambda *a: held), \ unittest.mock.patch.object(entry.seen_, "read", whole), unittest.mock.patch.object(entry.seen_, "pushes", pushes), \ @@ -1268,6 +1271,7 @@ def whole(transport, item, kept, identities, apps, since): unittest.mock.patch.object(entry.passes, "written", written_read): self.run_entry(Hub(self.table(numbers=[7]))) self.assertEqual(efforts["began"], {}, held and held[1]["now"]) + self.assertEqual(efforts["engines"], {}, held and held[1]["now"]) # With no record before it, nothing is known, so everything in the window is read. with unittest.mock.patch.object(entry.facts_, "snapshot", snapshot), \ unittest.mock.patch.object(entry.passes, "records", lambda *a: None), \ diff --git a/tests/test_steps.py b/tests/test_steps.py index 8a538558..b3e52f34 100644 --- a/tests/test_steps.py +++ b/tests/test_steps.py @@ -599,6 +599,38 @@ def test_apply_waits_for_its_jobs(self): self.assertEqual((code, ended["outcome"], ended["reason"], calls["minted"]), (0, "environment", want, []), needs) + def test_apply_says_whether_the_engine_step_started(self): # issue 757 + # The run job hands on its Engine step's outcome. GitHub skips that step once a step before it failed, as the + # CLI's install did in the outage: then the Run spent no model, and its outcome says `engine_started` false. + # The run job hands on its Engine step's outcome, and `apply` is given every job's results and outputs. + live = (ROOT / ".github" / "workflows" / "run-jobs.yml").read_text() + for line in (" engine: ${{ steps.engine.outcome }}\n", " - name: Run the Engine\n id: engine\n", + " NEEDS: ${{ toJSON(needs) }}\n"): + self.assertEqual(live.count(line), 1, line) + + def needs(run_result, engine, verify="skipped"): + run = {"result": run_result, "outputs": {} if engine is None else {"engine": engine}} + return json.dumps({"run": run, "verify": {"result": verify, "outputs": {}}}) + for engine, run_result, started in (("skipped", "failure", False), ("failure", "failure", True), + ("cancelled", "cancelled", True), ("success", "failure", True)): + code, _, ended, calls = self.go({"NEEDS": needs(run_result, engine)}) + self.assertEqual((code, ended["outcome"], ended["reason"], ended.get("engine_started"), calls["minted"]), + (0, "environment", "run_failed", started, []), engine) + # A Run whose run job succeeded reached the Engine, whatever ends it after. + code, _, ended, _ = self.go({"NEEDS": needs("success", "success", verify="failure")}) + self.assertEqual((ended["reason"], ended.get("engine_started")), ("verify_failed", True)) + code, printed, ended, _ = self.go({"NEEDS": needs("success", "success", verify="success")}) + self.assertEqual((code, ended["outcome"], ended.get("engine_started")), (0, None, True), printed) + # What does not say the Engine step was skipped, or ran, says nothing: the field is left out, and the Run is read + # as one that may have reached the model. + for engine in (None, "", "neutral", 0, True, ["skipped"]): + code, _, ended, _ = self.go({"NEEDS": needs("failure", engine)}) + self.assertEqual((ended["reason"], "engine_started" in ended), ("run_failed", False), engine) + odd = json.dumps({"run": {"result": "failure", "outputs": "skipped"}, "verify": {"result": "skipped"}}) + for said in (odd, json.dumps({"run": {"result": "failure"}, "verify": {"result": "skipped"}}), "not json", None): + code, _, ended, _ = self.go({"NEEDS": said}) + self.assertNotIn("engine_started", ended, said) + def test_apply_reads_with_reads(self): code, printed, ended, calls = self.go() self.assertEqual((code, ended["outcome"]), (0, None), printed) diff --git a/tests/test_tools.py b/tests/test_tools.py index fb3e784b..39fcb38f 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -3542,6 +3542,13 @@ def test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running(self): self.assertNotIn("expression", {rule for rule, _ in check_workflows.lint(outputs)}) self.assertIn("expression", {rule for rule, _ in check_workflows.lint(outputs.replace( "${{ steps.run.outputs.digest }}", "${{ github.event.pull_request.title }}"))}) + # A step's outcome is one of four words GitHub gives, so a job may hand it on (issue 757); nothing else of a step. + self.assertNotIn("expression", {rule for rule, _ in check_workflows.lint(outputs.replace( + "${{ steps.run.outputs.digest }}", "${{ steps.run.outcome }}"))}) + for other in ("${{ steps.run.conclusion }}", "${{ steps.run.outcome }}x", "${{ steps.run.outcome.x }}", + "${{ steps.run.outcome || github.event.pull_request.title }}"): + self.assertIn("expression", {rule for rule, _ in check_workflows.lint(outputs.replace( + "${{ steps.run.outputs.digest }}", other))}, other) def run_tree(self, text: str, account, name: str = "run.yml") -> list: config = json.loads((ROOT / "config" / "controller.json").read_text()) diff --git a/tools/check_workflows.py b/tools/check_workflows.py index 3d97aa29..4ee8ca3e 100644 --- a/tools/check_workflows.py +++ b/tools/check_workflows.py @@ -47,7 +47,7 @@ "wake": "a workflow on workflow_run wakes on the completion alone of the workflows it names, never its own, takes " "no input and reads nothing of the run that woke it", "expression": "an expression stands only as the whole of a value under env:, with: or secrets:, or in if:; or as " - "exactly a Run's name, ${{ inputs.name }}, its lane, ${{ inputs.item }}, or a step's output", + "exactly a Run's name, ${{ inputs.name }}, its lane, ${{ inputs.item }}, or a step's output or its outcome", "model": "no step sets the CLI's environment scrub, which only the Engine sets, beside --tools \"\", and a step that " "holds the model's credential is handed no tool list", "lane": "a job that calls the Run's jobs, or an act's, holds its item's lane: its own concurrency group, " @@ -98,13 +98,14 @@ WAKES, RUN_WORKFLOW = "poller.yml", "sf-run" # The Run's lane (issue 196's ruling 6): one concurrency group for each Work item, on the one job that calls the Run's # jobs, so that it holds from run to apply and never cancels what runs. And the three places an expression may stand -# whole beyond env:, with: and secrets:: the run's name, the lane, and a job's output of one of its steps. +# whole beyond env:, with: and secrets:: the run's name, the lane, and a job's output of one of its steps: a step's own +# output, or its outcome, which GitHub gives as one of four words (issue 757). RUN_JOBS = "run-jobs.yml" APPROVE_JOB = "approve-job.yml" QUEUE_JOB = "queue-job.yml" LANE = "${{ inputs.item }}" RUN_NAME = "${{ inputs.name }}" -OUTPUT = re.compile(r"^\$\{\{ steps\.[a-z][\w-]*\.outputs\.[a-z][\w-]* \}\}$") +OUTPUT = re.compile(r"^\$\{\{ steps\.[a-z][\w-]*\.(?:outputs\.[a-z][\w-]*|outcome) \}\}$") # The Run's shim, as the ruling gives it in full: its dispatcher is the pass's own token, by the account controller/1 # names for it, on the run's first attempt alone, since a re-run keeps the dispatcher's id though a person started it; # and a Pause of the repo or the org starts no Run, any value but unset or exactly false a Pause (decision 0003; I-34). @@ -334,6 +335,7 @@ outputs: commit: ${{ steps.commit.outputs.commit }} digest: ${{ steps.engine.outputs.digest }} + engine: ${{ steps.engine.outcome }} artifact: ${{ steps.result.outputs.artifact-id }} reader: ${{ steps.reader.outputs.artifact-id }} steps: diff --git a/tools/mutants.py b/tools/mutants.py index 95ebb562..60ddd6b3 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -4426,8 +4426,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ' key = (record["run"], record["started_at"])\n attempts[key] = attempts.get(key, 0) + 1\n ids.setdefault(record["run"], []).append(row["id"])\n if record["status"] == "completed" and record["conclusion"] != "success":\n failed.append((record, row["id"], bool(row.get("run_started_at"))))\n', ' key = (record["run"], record["started_at"])\n ids.setdefault(record["run"], []).append(row["id"])\n if record["status"] == "completed" and record["conclusion"] != "success":\n attempts[key] = attempts.get(key, 0) + 1\n failed.append((record, row["id"], bool(row.get("run_started_at"))))\n', ('tests.test_adapters.Records.test_an_outcome_is_read_once_and_kept_by_its_run_and_attempt',)), ('an outcome read is not stamped', 'adapters/records.py', - ' lambda answer: reads.append(dict(read_at(answer), read="outcome")))', - ' lambda answer: None)', ('tests.test_adapters',)), + ' lambda answer: reads.append(dict(read_at(answer), read="outcome")), told)', + ' lambda answer: None, told)', ('tests.test_adapters',)), # Decision 0193: when a Run began is read from its jobs, under P-64's rule. ('a job that waits says when its Run began', 'adapters/records.py', 'if j.get("status") == "in_progress"\n', @@ -4604,8 +4604,8 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("the entry asks the jobs of every Run", 'controller/__main__.py', 'of=[{"repo": home, "type": "pull_request", "number": number} for number in numbers])', 'of=None)', WIRE_ENTRY), ("the run records are read though the open pull requests were not read whole", 'controller/__main__.py', - ' if not whole:\n raise Stop("pull_requests_unreadable")\n earlier = pass_.earlier(reads_.previous_records(previous), now) or []\n try:\n runs = records.runs(transport, home, branch, run_files(controller), dispatcher(controller),\n kept=records.known(earlier), began=records.begun(earlier),\n of=[{"repo": home, "type": "pull_request", "number": number} for number in numbers])\n except Unanswered:\n runs = None\n', - ' earlier = pass_.earlier(reads_.previous_records(previous), now) or []\n try:\n runs = records.runs(transport, home, branch, run_files(controller), dispatcher(controller),\n kept=records.known(earlier), began=records.begun(earlier),\n of=[{"repo": home, "type": "pull_request", "number": number} for number in numbers])\n except Unanswered:\n runs = None\n if not whole:\n raise Stop("pull_requests_unreadable")\n', ("tests.test_controller.Entry.test_what_the_entry_cannot_read_writes_nothing",)), + ' if not whole:\n raise Stop("pull_requests_unreadable")\n earlier = pass_.earlier(reads_.previous_records(previous), now) or []\n try:\n runs = records.runs(transport, home, branch, run_files(controller), dispatcher(controller),\n kept=records.known(earlier), began=records.begun(earlier),\n engines=records.engines(earlier),\n of=[{"repo": home, "type": "pull_request", "number": number} for number in numbers])\n except Unanswered:\n runs = None\n', + ' earlier = pass_.earlier(reads_.previous_records(previous), now) or []\n try:\n runs = records.runs(transport, home, branch, run_files(controller), dispatcher(controller),\n kept=records.known(earlier), began=records.begun(earlier),\n engines=records.engines(earlier),\n of=[{"repo": home, "type": "pull_request", "number": number} for number in numbers])\n except Unanswered:\n runs = None\n if not whole:\n raise Stop("pull_requests_unreadable")\n', ("tests.test_controller.Entry.test_what_the_entry_cannot_read_writes_nothing",)), ("a read of the items given is whole whatever jobs of their Runs cannot be read", 'adapters/records.py', ' whole = whole and answered\n', ' whole = whole and (answered or of is not None)\n', BOUNDED), ("an item is given by its repo and number, whatever its type", 'adapters/records.py', @@ -11136,6 +11136,57 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: ("the pass checks out the tag again", '.github/workflows/pass.yml', ' ref: ${{ job.workflow_sha }}\n', ' ref: factory/v0-next\n', ('tests.test_tools.Workflows.test_the_pass_uploads_its_record_before_it_dispatches',)), + # Issue 757: the run record says whether a Run's Engine step started, from the outcome apply writes. + ("the run job hands on nothing of its Engine step", '.github/workflows/run-jobs.yml', + ' engine: ${{ steps.engine.outcome }}\n', '', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("a skipped Engine step says it started", 'harness/apply_step.py', + ' if engine == "skipped":\n return False\n', ' if engine == "skipped":\n return True\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("an Engine step that ran says nothing", 'harness/apply_step.py', + ' if engine in ("success", "failure", "cancelled"):\n', ' if False:\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("any word but skipped says the Engine started", 'harness/apply_step.py', + ' if engine in ("success", "failure", "cancelled"):\n', ' if engine is not None:\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("an Engine step cancelled says nothing", 'harness/apply_step.py', + '("success", "failure", "cancelled")', '("success", "failure")', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("an Engine step that failed says nothing", 'harness/apply_step.py', + '("success", "failure", "cancelled")', '("success", "cancelled")', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("an Engine step that succeeded says nothing", 'harness/apply_step.py', + '("success", "failure", "cancelled")', '("failure", "cancelled")', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("apply leaves out whether the Engine started", 'harness/apply_step.py', + ' ended = dict(ended, engine_started=started)\n', ' pass\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("apply says the Engine did not start where the job said nothing", 'harness/apply_step.py', + ' if started is not None:\n ended = dict(ended, engine_started=started)\n', + ' if True:\n ended = dict(ended, engine_started=bool(started))\n', ('tests.test_steps.Apply.test_apply_says_whether_the_engine_step_started',)), + ("a job's output may be any of its step's", 'tools/check_workflows.py', + '\\.(?:outputs\\.[a-z][\\w-]*|outcome) \\}\\}$")', '\\.(?:outputs\\.[a-z][\\w-]*|[a-z]+) \\}\\}$")', ('tests.test_tools.Workflows.test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running',)), + ("a job may not hand on a step's outcome", 'tools/check_workflows.py', + '\\.(?:outputs\\.[a-z][\\w-]*|outcome) \\}\\}$")', '\\.(?:outputs\\.[a-z][\\w-]*) \\}\\}$")', ('tests.test_tools.Workflows.test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running',)), + ("an outcome hands on nothing of the Engine", 'adapters/artifacts.py', + ' started.append(record["engine_started"])\n', ' pass\n', ('tests.test_adapters.Artifacts.test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1',)), + ("a Run's outcome read says nothing of its Engine", 'adapters/records.py', + ' if told:\n record["engine_started"] = told[0]\n', + ' if False:\n record["engine_started"] = told[0]\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("a kept outcome keeps nothing of the Engine", 'adapters/records.py', + ' if (record["run"], record["started_at"]) in engines:\n', + ' if False:\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("a kept outcome takes the Engine of another Run", 'adapters/records.py', + ' if (record["run"], record["started_at"]) in engines:\n record["engine_started"] = engines[(record["run"], record["started_at"])]\n', + ' if engines:\n record["engine_started"] = list(engines.values())[0]\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("anything said of the Engine is kept", 'adapters/records.py', + ' if type(row.get("engine_started")) is bool:\n', ' if "engine_started" in row:\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("what a record says of the Engine twice is kept", 'adapters/records.py', + 'for key, started in said.items() if keys.count(key) == 1}', 'for key, started in said.items()}', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("a key that is no Run or no start is kept for the Engine", 'adapters/records.py', + ' if not (isinstance(key[0], str) and isinstance(key[1], str)):\n continue\n', '', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("the earlier record's word of the Engine is read", 'adapters/records.py', + ' return said_in(previous[-1])[0] if isinstance(previous, list) and previous else {}\n', + ' return said_in(previous[0])[0] if isinstance(previous, list) and previous else {}\n', ('tests.test_adapters.Records.test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt',)), + ("a caller that does not ask is handed the Engine", 'adapters/artifacts.py', + ' if started is not None and "engine_started" in record:\n', ' if "engine_started" in record:\n', + ('tests.test_adapters.Artifacts.test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1',)), + ("the entry keeps the Engine from records as they were downloaded", 'controller/__main__.py', + 'engines=records.engines(earlier),', 'engines=records.engines(previous),', WIRE_ENTRY), + ("the entry keeps nothing of the Engine", 'controller/__main__.py', + 'engines=records.engines(earlier),', 'engines=None,', WIRE_ENTRY), ("the Run's first job checks out the tag again", '.github/workflows/run-jobs.yml', ' ref: ${{ job.workflow_sha }}\n', ' ref: factory/v0-next\n', ('tests.test_tools.Workflows.test_the_commit_a_later_job_checks_out_is_the_releases_own',)),