Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/run-jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ jobs:
outputs:
commit: ${{ steps.commit.outputs.commit }}
digest: ${{ steps.engine.outputs.digest }}
engine: ${{ steps.engine.outcome }}
artifact: ${{ steps.result.outputs.artifact-id }}
reader: ${{ steps.reader.outputs.artifact-id }}
steps:
Expand Down
7 changes: 5 additions & 2 deletions adapters/artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,9 +114,10 @@ def read(transport, repo: str, github_id: int, seen=None, name: str = NAME, memb
return held, held is not None


def outcome(transport, repo: str, github_id: int, run: str, seen=None) -> tuple:
def outcome(transport, repo: str, github_id: int, run: str, seen=None, started=None) -> tuple:
"""`(outcome, whole)` of the Run GitHub numbers `github_id` and the Factory names `run`: the fault its outcome
names, None for a Run that left none, and whether it was read whole. `seen` is handed the list's answer."""
names, None for a Run that left none, and whether it was read whole. `seen` is handed the list's answer, and
`started` whether the Run's Engine step started, where its outcome says (issue 757)."""
held, whole = read(transport, repo, github_id, seen)
if held is None:
return None, whole
Expand All @@ -127,6 +128,8 @@ def outcome(transport, repo: str, github_id: int, run: str, seen=None) -> tuple:
record = None
if not isinstance(record, dict) or validate(record, SCHEMA) or record["run"] != run or record["outcome"] is None:
return None, False
if started is not None and "engine_started" in record:
started.append(record["engine_started"])
return record["outcome"], True


Expand Down
36 changes: 33 additions & 3 deletions adapters/records.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,8 +299,30 @@ def read(record) -> dict:
return {k: v for k, v in last.items() if v is not None or (k in before and before[k] is None)}


def engines(previous: list) -> dict:
"""Whether each Run's Engine step started, as the last pass record before this one read it with its outcome, by
its Run id and when its attempt started (issue 757). It is read only beside an outcome `known` keeps, and `known`
keeps a fault from the last record alone, so no earlier record's word is needed. Only a true or a false is kept:
an outcome is written once, so what it said stands for its attempt. A key the record lists twice is not kept, and
nothing is kept from what lists no runs."""
def said_in(record) -> tuple:
rows = record.get("runs") if isinstance(record, dict) else None
if not isinstance(rows, list):
return {}, []
said, keys = {}, []
for row in rows:
key = (row.get("run"), row.get("started_at")) if isinstance(row, dict) else (None, None)
if not (isinstance(key[0], str) and isinstance(key[1], str)):
continue
keys.append(key)
if type(row.get("engine_started")) is bool:
said[key] = row["engine_started"]
return {key: started for key, started in said.items() if keys.count(key) == 1}, keys
return said_in(previous[-1])[0] if isinstance(previous, list) and previous else {}


def runs(transport, repo: str, branch: str, workflows: list, dispatcher: tuple, per_page: int = PAGE,
sleep=time.sleep, kept=None, began=None, of=None, on=None) -> dict:
sleep=time.sleep, kept=None, began=None, of=None, on=None, engines=None) -> dict:
"""The run records of the Factory's Role workflows, the ids of counted Runs not yet named, whether the list was
read whole, and each read's date and age for the pass record. `dispatcher` is the pass job's own token's account,
by its id and type, `(id, "Bot")`: only a run it dispatched is a Run.
Expand All @@ -314,6 +336,9 @@ def runs(transport, repo: str, branch: str, workflows: list, dispatcher: tuple,
GitHub run its name names (issue 338). Nothing calls `known` until
the pass's entry, the effects port's next part (#193), hands it the records it placed.

`engine_started`, where a failed Run's outcome says it, is whether its Engine step started; `engines` is what
`engines` read of it from the pass records before, kept with an outcome `kept` holds (issue 757).

`began` is what `begun` read from the pass records before, and None asks no job: no record then says when it
began. Given, each Run in progress or completed has its jobs read for `began_at` (`_began`), but one `began`
holds, for a row that names when its attempt started, once in this list. A Run past the `JOBS` asked, or whose
Expand All @@ -322,7 +347,7 @@ def runs(transport, repo: str, branch: str, workflows: list, dispatcher: tuple,
whole. The pass hands the open pull requests, so what it asks is its live work, not the list's history
(decision 0196). `on`, an item, asks that item's Runs alone, as an act reading its own item again does, and all
of them: a Run keeps nothing from one read to the next, so a cap on its item's Runs would never be worked off."""
kept = kept or {}
kept, engines = kept or {}, engines or {}
roles = {".github/workflows/%s" % name for name in workflows}

def read() -> tuple:
Expand Down Expand Up @@ -361,10 +386,15 @@ def read() -> tuple:
for record, github_id, keyed in failed:
if keyed and (record["run"], record["started_at"]) in kept:
record["outcome"] = kept[(record["run"], record["started_at"])]
if (record["run"], record["started_at"]) in engines:
record["engine_started"] = engines[(record["run"], record["started_at"])]
continue
told = []
record["outcome"], read_whole = artifacts.outcome(
transport, repo, github_id, record["run"],
lambda answer: reads.append(dict(read_at(answer), read="outcome")))
lambda answer: reads.append(dict(read_at(answer), read="outcome")), told)
if told:
record["engine_started"] = told[0]
whole = whole and read_whole
asked = 0
for record, github_id, attempt, keyed in ran if began is not None else []:
Expand Down
1 change: 1 addition & 0 deletions controller/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ def read(transport, config: dict, run_id: str, previous, now: str, clock, starte
try:
runs = records.runs(transport, home, branch, run_files(controller), dispatcher(controller),
kept=records.known(earlier), began=records.begun(earlier),
engines=records.engines(earlier),
of=[{"repo": home, "type": "pull_request", "number": number} for number in numbers])
except Unanswered:
runs = None
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# 0201. A Run's outcome says whether its Engine step started

Status: accepted. The Factory Builder's ruling of 2026-11-12 on "A Run that ended environment does not use up the item's run ceiling" (issue 757): before a Run that ended `environment` may be counted apart from the Run ceiling, the run record must say whether the Run reached the model. This record adds that fact alone. What counts toward which bound is the next record's.

## Decision

| What | Decided | Because |
|---|---|---|
| The signal | The `run` job hands on its Engine step's outcome as the output `engine`, `${{ steps.engine.outcome }}` | GitHub skips a step once a step before it has failed, so a Run whose CLI install or read failed shows the Engine step `skipped`. That was every Run of the runner DNS outage of 2026-11-11. The step's outcome is GitHub's own word, set by no Factory code, and one of four values |
| What `apply` writes | `outcome/1`'s `engine_started`: false where the output is `skipped`; true where it is `success`, `failure` or `cancelled`; left out where it is anything else or missing | A Run whose Engine step ran may have spent on the model, whatever ended it. One whose output cannot be read is read as one that may have: the field is left out, never false |
| Where it is written | Every `outcome/1` `apply` writes, whatever its outcome. The fallback the last step writes, `apply_failed`, and the reader's `release_moved` carry none | Only `apply` is given the run job's outputs (`NEEDS`) |
| The run record | `engine_started`, a true or a false alone, as the failed Run's outcome said it, and absent where the outcome said nothing. The records port reads it with the outcome, and keeps it with a kept outcome from the last pass record, the one `known` keeps a fault from | An outcome is written once for its attempt, so what it said stands. Kept outcomes are not read again, so the field must be kept with them, or a later pass would lose it |
| The workflow check | A job may hand on a step's `outcome` as an output, beside a step's own output | It is one of four words, so it cannot become a command, as `tools/check_workflows.py`'s expression rule requires |
| What reads it | Nothing yet: the Kernel's counting is the next record's | One change, one rule |

## Why

`environment` covers a CLI install that failed, as in the outage, and a Run stopped at its step limit after the model read the item. Both reach `apply` as `run_failed`. A count that cannot tell them apart either gives up the cost ceiling to a slow item, or holds every item after an outage. Whether the Engine step started is the one thing that splits them, and only the `run` job knows it.

It fails closed: a Run says it did not start the Engine only where GitHub said the step was skipped. A job killed before it handed on its outputs, an output missing, or a word GitHub has not given before all leave the field out. An older outcome, written before this change, has none.

It costs one thing, accepted. `outcome/1` admits no field it does not name, so after a rollback of `next` to a Release cut before this change, that Release reads a newer Run's outcome as no `outcome/1`, and its list of Runs is not whole until those Runs leave the window. Pass records' run records already carry such a field, `began_at` (decision 0193). Not verified on GitHub: that the job output reads `skipped` for a step skipped after an earlier one failed. GitHub's documentation says so, and if it reads otherwise the field is left out.

## Boundaries

| Boundary | Check |
|---|---|
| `apply` says the Engine did not start only where the run job hands on `skipped`, that it started where it hands on any word GitHub gives a step that ran, and nothing otherwise; the live workflow hands the Engine step's outcome on, and gives `apply` every job's results and outputs | `tests/test_steps.py::test_apply_says_whether_the_engine_step_started`, `harness/apply_step.py`, `.github/workflows/run-jobs.yml`, `tools/mutants.py`, mutant: "the run job hands on nothing of its Engine step", mutant: "a skipped Engine step says it started", mutant: "an Engine step that ran says nothing", mutant: "any word but skipped says the Engine started", mutant: "an Engine step cancelled says nothing", mutant: "an Engine step that failed says nothing", mutant: "an Engine step that succeeded says nothing", mutant: "apply leaves out whether the Engine started", mutant: "apply says the Engine did not start where the job said nothing" |
| A job may hand on a step's outcome, and nothing else of a step but its own output | `tests/test_tools.py::test_a_runs_lane_is_its_item_on_the_job_and_cancels_nothing_running`, `tools/check_workflows.py`, `tools/mutants.py`, mutant: "a job's output may be any of its step's", mutant: "a job may not hand on a step's outcome" |
| An outcome's `engine_started` is handed on where the caller asks and the outcome says, and an outcome that says it otherwise is no `outcome/1` | `tests/test_adapters.py::test_a_runs_outcome_is_read_from_its_one_file_and_held_to_outcome_1`, `adapters/artifacts.py`, `tools/mutants.py`, mutant: "an outcome hands on nothing of the Engine", mutant: "a caller that does not ask is handed the Engine" |
| The run record carries it as the outcome said it, and keeps it with a kept outcome, by its own Run and attempt: the last record's word alone, a true or a false alone, and nothing for a Run the record lists twice | `tests/test_adapters.py::test_whether_the_engine_started_is_read_with_the_outcome_and_kept_by_its_run_and_attempt`, `adapters/records.py`, `tools/mutants.py`, mutant: "a Run's outcome read says nothing of its Engine", mutant: "a kept outcome keeps nothing of the Engine", mutant: "a kept outcome takes the Engine of another Run", mutant: "anything said of the Engine is kept", mutant: "what a record says of the Engine twice is kept", mutant: "the earlier record's word of the Engine is read", mutant: "a key that is no Run or no start is kept for the Engine" |
| The pass's entry hands the records port what the records before said of the Engine, from pass records alone, each from before the pass | `tests/test_controller.py::test_the_entry_gathers_what_reconciliation_reads`, `controller/__main__.py`, `tools/mutants.py`, mutant: "the entry keeps nothing of the Engine", mutant: "the entry keeps the Engine from records as they were downloaded" |
17 changes: 17 additions & 0 deletions examples/invalid/run-record.an-engine-start-in-words.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"run": "r_01JA6Z",
"item": {
"repo": "combsmith/software-factory",
"type": "pull_request",
"number": 42
},
"role": "pr-review",
"intent": "review-change",
"content_id": "sha256:5d0c",
"status": "completed",
"conclusion": "success",
"outcome": null,
"started_at": "2026-10-11T12:05:00Z",
"began_at": "2026-10-11T12:05:04Z",
"engine_started": "no"
}
23 changes: 23 additions & 0 deletions harness/apply_step.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,26 @@ def needs_met(needs) -> tuple:
return None


def engine_started(needs):
"""Whether the `run` job's Engine step started, by the outcome the job hands on for it (issue 757), or None.

GitHub skips the step once a step before it failed, as the CLI's install or the read did: then the Run spent no
model, and this is False. True for an outcome it gives a step that ran. Anything else, an output missing or not
one of these, says nothing, and the Run is read as one that may have reached the model."""
try:
said = json.loads(needs) if isinstance(needs, str) else None
except ValueError:
said = None
run = said.get("run") if isinstance(said, dict) else None
outputs = run.get("outputs") if isinstance(run, dict) else None
engine = outputs.get("engine") if isinstance(outputs, dict) else None
if engine == "skipped":
return False
if engine in ("success", "failure", "cancelled"):
return True
return None


def written(argv: list, environ: dict, base, clock, connect) -> tuple:
"""`(applied/1, outcome/1)` of the Run."""
name, item, pass_id, found, _ = argv
Expand Down Expand Up @@ -170,6 +190,9 @@ def step(argv: list, environ: dict, base, clock=time.time, connect=Live) -> str:
raise Stop("usage: python3 -m harness.apply_step NAME ITEM PASS RESULT OUT")
folder = steps.new_folder(base, argv[4])
done, ended = written(argv, environ, base, clock, connect)
started = engine_started(environ.get("NEEDS"))
if started is not None:
ended = dict(ended, engine_started=started)
Comment thread
don-petry marked this conversation as resolved.
Resolved
if shared.errors(done, "applied") or shared.errors(ended, "outcome"):
raise Stop("records_unreadable")
steps.write(folder, "applied.json", done)
Expand Down
4 changes: 4 additions & 0 deletions schemas/outcome.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@
"null"
],
"pattern": "^[a-z_]+$"
},
"engine_started": {
"description": "Whether the run job's Engine step started, by the outcome the job hands on for it: false where GitHub skipped it because a step before it failed, and the Run spent no model; true where it ran. Absent where the job said neither, and read then as a Run that may have reached the model (issue 757)",
"type": "boolean"
}
}
}
4 changes: 4 additions & 0 deletions schemas/run-record.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,10 @@
"skipped"
]
},
"engine_started": {
"description": "Present where a failed Run's outcome/1 says it: whether the run job's Engine step started. False where GitHub skipped it because a step before it failed, so the Run spent no model; absent where the outcome said neither, and read then as a Run that may have reached the model (issue 757)",
"type": "boolean"
},
"rerun_by_dispatcher": {
"description": "Present on a re-run alone, or where GitHub gives no attempt that can be read: whether the pass job's own token, the dispatcher's account, re-ran it, as GitHub's triggering_actor says. A re-run runs nothing (decision 0058); one by that token is a write no Effect records, and alarms (decision 0063); null where its attempt or re-runner cannot be read, and reconciliation is short",
"type": [
Expand Down
Loading
Loading