From b2de156f196a757adedeaffcf48ca0bef366f9ee Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 09:00:08 +0100 Subject: [PATCH 1/8] perf(ci): partition complete offline regressions --- .github/workflows/ci.yaml | 181 ++++++++++++++++++++------- AGENTS.md | 2 +- docs/adr/ci-regression-partitions.md | 56 +++++++++ scripts/tests/test-policy-ci-job.sh | 101 +++++++++++++++ 4 files changed, 292 insertions(+), 48 deletions(-) create mode 100644 docs/adr/ci-regression-partitions.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d3cb2d3110..a763ff87ac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1163,29 +1163,6 @@ jobs: shellcheck scripts/tests/test-guard-consumer-discovery-conservation.sh ./scripts/guard-consumer-discovery-conservation.sh - # The report names the two revisions per consumer; the generator (#3550) turns them - # into the committed approved set the matcher narrowing (#3308) is derived from. Its - # cluster read and its pin read are both stubbed here, so this proves the fail-closed - # shape โ€” an unresolved consumer writes nothing โ€” not any particular revision. - - name: ๐Ÿ” Validate the approved-revisions generator - run: | - # -x: the generator SOURCES the report, and without it SC1091 is reported and the run exits 1. - shellcheck -x scripts/generate-publish-workflow-approved-revisions.sh - shellcheck -x scripts/tests/test-publish-workflow-approved-revisions.sh - bash scripts/tests/test-publish-workflow-approved-revisions.sh - shellcheck -x scripts/bootstrap-tenant-source.sh scripts/tenant-publication-binding.lib.sh scripts/tests/test-tenant-publication-binding.sh scripts/tests/test-bootstrap-tenant-source.sh - bash scripts/tests/test-tenant-publication-binding.sh - bash scripts/tests/test-bootstrap-tenant-source.sh - shellcheck -x scripts/canonical-tenant-source.lib.sh scripts/tests/test-canonical-source-bootstrap.sh scripts/tests/test-canonical-bootstrap-output.sh - bash scripts/tests/test-canonical-source-bootstrap.sh - shellcheck -x scripts/tests/test-canonical-constructor-attribution.sh - bash scripts/tests/test-canonical-constructor-attribution.sh - shellcheck -x scripts/tests/test-canonical-generator-output.sh - bash scripts/tests/test-canonical-generator-output.sh - bash scripts/tests/test-canonical-bootstrap-output.sh - shellcheck -x scripts/tests/test-canonical-publisher-regeneration.sh - bash scripts/tests/test-canonical-publisher-regeneration.sh - # The pin guard above proves each matcher pins a FIXED revision; this one proves each # per-consumer matcher pins the revision PAIR the committed approved set names for that # consumer, and refuses any revision outside it (#3551, AC4 of #3308). Unconditional for @@ -1357,6 +1334,79 @@ jobs: bash scripts/tests/test-verify-wedding-backup-cutover.sh go test ./scripts/verify-wedding-dedicated-restore + validate-publication-generator: + name: ๐Ÿ” Validate Publication Generator + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: ๐Ÿ“‘ Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + + - name: โš™๏ธ Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: โš™๏ธ Initialize the approved kubectl for native regressions + timeout-minutes: 5 + shell: bash + env: + KUBECTL_VERSION: "v1.36.2" + KUBECTL_SHA256: "1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82" + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/kubectl" + curl -fsSL \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ + -o "$kubectl_path" + printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check + sudo install "$kubectl_path" /usr/local/bin/kubectl + echo /usr/local/bin >> "$GITHUB_PATH" + export PATH="/usr/local/bin:$PATH" + kubectl version --client + + - name: โŽˆ Install checksum-verified native Helm + shell: bash + env: + HELM_VERSION: v4.2.4 + HELM_SHA256: c306b46f719b0a4da32d0f78ee21bf90ce8d602f15b22ab753f0674d1670a7f3 + run: | + set -euo pipefail + native_dir="$(mktemp -d)" + archive="helm-${HELM_VERSION}-linux-amd64.tar.gz" + curl --fail --location --retry 3 --output "$native_dir/$archive" "https://get.helm.sh/$archive" + printf '%s %s\n' "$HELM_SHA256" "$native_dir/$archive" | sha256sum --check - + tar -xzf "$native_dir/$archive" -C "$native_dir" linux-amd64/helm + printf 'KYVERNO_LOG_HELM_BIN=%s\n' "$native_dir/linux-amd64/helm" >> "$GITHUB_ENV" + + # The report names the two revisions per consumer; the generator (#3550) turns them + # into the committed approved set the matcher narrowing (#3308) is derived from. Its + # cluster read and its pin read are both stubbed here, so this proves the fail-closed + # shape โ€” an unresolved consumer writes nothing โ€” not any particular revision. + - name: ๐Ÿ” Validate the approved-revisions generator + run: | + # -x: the generator SOURCES the report, and without it SC1091 is reported and the run exits 1. + shellcheck -x scripts/generate-publish-workflow-approved-revisions.sh + shellcheck -x scripts/tests/test-publish-workflow-approved-revisions.sh + bash scripts/tests/test-publish-workflow-approved-revisions.sh + shellcheck -x scripts/bootstrap-tenant-source.sh scripts/tenant-publication-binding.lib.sh scripts/tests/test-tenant-publication-binding.sh scripts/tests/test-bootstrap-tenant-source.sh + bash scripts/tests/test-tenant-publication-binding.sh + bash scripts/tests/test-bootstrap-tenant-source.sh + shellcheck -x scripts/canonical-tenant-source.lib.sh scripts/tests/test-canonical-source-bootstrap.sh scripts/tests/test-canonical-bootstrap-output.sh + bash scripts/tests/test-canonical-source-bootstrap.sh + shellcheck -x scripts/tests/test-canonical-constructor-attribution.sh + bash scripts/tests/test-canonical-constructor-attribution.sh + shellcheck -x scripts/tests/test-canonical-generator-output.sh + bash scripts/tests/test-canonical-generator-output.sh + bash scripts/tests/test-canonical-bootstrap-output.sh + shellcheck -x scripts/tests/test-canonical-publisher-regeneration.sh + bash scripts/tests/test-canonical-publisher-regeneration.sh + validate-consumer-discovery: name: ๐Ÿ” Validate Consumer Discovery Regressions runs-on: ubuntu-latest @@ -1690,10 +1740,15 @@ jobs: run: ksail --config ksail.prod.yaml workload validate validate-cluster-policies: - name: โš–๏ธ Validate Cluster Policies + name: โš–๏ธ Validate Cluster Policies (${{ matrix.suite }}) needs: [changes] if: github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true' runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 3 + matrix: + suite: [fixtures, tenant, services, placement, storage, observability] permissions: contents: read steps: @@ -1725,28 +1780,19 @@ jobs: with: release: ${{ steps.kyverno-version.outputs.release }} - - name: โš–๏ธ Validate cluster policies - # Evaluate the complete policy census and its negative controls offline. - # No cluster access or secrets are required. + - name: โš–๏ธ Validate cluster policies (fixtures) + if: matrix.suite == 'fixtures' run: | kyverno test ./tests shellcheck scripts/validate-kyverno-fixture-evaluation.sh scripts/tests/test-validate-kyverno-fixture-evaluation.sh bash scripts/tests/test-validate-kyverno-fixture-evaluation.sh bash scripts/validate-kyverno-fixture-evaluation.sh tests + + - name: โš–๏ธ Validate cluster policies (tenant) + if: matrix.suite == 'tenant' + run: | shellcheck scripts/tests/test-tenant-webapp-boundary.sh bash scripts/tests/test-tenant-webapp-boundary.sh - shellcheck scripts/tests/test-webapp-container-security.sh - bash scripts/tests/test-webapp-container-security.sh - shellcheck scripts/tests/test-hcloud-volume-priority.sh - bash scripts/tests/test-hcloud-volume-priority.sh - shellcheck scripts/tests/test-tetragon-agent-priority.sh - bash scripts/tests/test-tetragon-agent-priority.sh - shellcheck scripts/tests/test-audit-privileged-rbac.sh - bash scripts/tests/test-audit-privileged-rbac.sh - shellcheck scripts/tests/test-auto-vpa-generating-policy-exclusion.sh - bash scripts/tests/test-auto-vpa-generating-policy-exclusion.sh - shellcheck scripts/tests/test-arc-controller-availability.sh - bash scripts/tests/test-arc-controller-availability.sh shellcheck scripts/tests/test-restrict-tenant-secret-stores.sh bash scripts/tests/test-restrict-tenant-secret-stores.sh shellcheck scripts/tests/test-tenant-controller-namespace-coverage.sh @@ -1760,12 +1806,12 @@ jobs: bash scripts/tests/test-add-default-deny-shape.sh shellcheck scripts/tests/test-restrict-homepage-service-groups.sh bash scripts/tests/test-restrict-homepage-service-groups.sh - shellcheck scripts/tests/test-kyverno-umami-mutation-rbac.sh - bash scripts/tests/test-kyverno-umami-mutation-rbac.sh - shellcheck scripts/tests/test-kyverno-github-team-report-rbac.sh - bash scripts/tests/test-kyverno-github-team-report-rbac.sh shellcheck scripts/tests/test-tenant-route-hostname-boundary.sh bash scripts/tests/test-tenant-route-hostname-boundary.sh + + - name: โš–๏ธ Validate cluster policies (services) + if: matrix.suite == 'services' + run: | shellcheck scripts/tests/test-actual-budget-auth-route.sh bash scripts/tests/test-actual-budget-auth-route.sh shellcheck scripts/tests/test-umami-sso-dashboard-route.sh @@ -1783,12 +1829,32 @@ jobs: shellcheck scripts/tests/test-umami-tracking-probe.sh bash scripts/tests/test-umami-tracking-probe.sh shellcheck scripts/verify-umami-tracking.sh + + - name: โš–๏ธ Validate cluster policies (placement) + if: matrix.suite == 'placement' + run: | + shellcheck scripts/tests/test-webapp-container-security.sh + bash scripts/tests/test-webapp-container-security.sh + shellcheck scripts/tests/test-hcloud-volume-priority.sh + bash scripts/tests/test-hcloud-volume-priority.sh + shellcheck scripts/tests/test-tetragon-agent-priority.sh + bash scripts/tests/test-tetragon-agent-priority.sh + shellcheck scripts/tests/test-auto-vpa-generating-policy-exclusion.sh + bash scripts/tests/test-auto-vpa-generating-policy-exclusion.sh + shellcheck scripts/tests/test-arc-controller-availability.sh + bash scripts/tests/test-arc-controller-availability.sh shellcheck scripts/tests/test-provider-drc-pod-seccomp.sh bash scripts/tests/test-provider-drc-pod-seccomp.sh - shellcheck scripts/tests/test-add-baseline-context.sh - bash scripts/tests/test-add-baseline-context.sh shellcheck scripts/tests/test-label-baseline-workers.sh bash scripts/tests/test-label-baseline-workers.sh + shellcheck scripts/tests/test-hcloud-baseline-worker-affinity.sh + bash scripts/tests/test-hcloud-baseline-worker-affinity.sh + + - name: โš–๏ธ Validate cluster policies (storage) + if: matrix.suite == 'storage' + run: | + shellcheck scripts/tests/test-add-baseline-context.sh + bash scripts/tests/test-add-baseline-context.sh shellcheck scripts/tests/test-longhorn-registrar-prestop-policy.sh bash scripts/tests/test-longhorn-registrar-prestop-policy.sh shellcheck scripts/tests/test-csi-sidecar-baseline-context.sh @@ -1797,11 +1863,19 @@ jobs: bash scripts/tests/test-longhorn-csi-plugin-baseline-context.sh shellcheck scripts/tests/test-longhorn-engine-image-baseline-context.sh bash scripts/tests/test-longhorn-engine-image-baseline-context.sh - shellcheck scripts/tests/test-hcloud-baseline-worker-affinity.sh - bash scripts/tests/test-hcloud-baseline-worker-affinity.sh shellcheck scripts/guard-longhorn-ui-baseline-context.sh scripts/tests/test-guard-longhorn-ui-baseline-context.sh scripts/tests/test-longhorn-ui-baseline-context.sh bash scripts/tests/test-guard-longhorn-ui-baseline-context.sh bash scripts/tests/test-longhorn-ui-baseline-context.sh + + - name: โš–๏ธ Validate cluster policies (observability) + if: matrix.suite == 'observability' + run: | + shellcheck scripts/tests/test-audit-privileged-rbac.sh + bash scripts/tests/test-audit-privileged-rbac.sh + shellcheck scripts/tests/test-kyverno-umami-mutation-rbac.sh + bash scripts/tests/test-kyverno-umami-mutation-rbac.sh + shellcheck scripts/tests/test-kyverno-github-team-report-rbac.sh + bash scripts/tests/test-kyverno-github-team-report-rbac.sh shellcheck scripts/tests/test-cilium-hubble-baseline-context.sh bash scripts/tests/test-cilium-hubble-baseline-context.sh shellcheck scripts/guard-coroot-baseline-context.sh scripts/tests/test-guard-coroot-baseline-context.sh @@ -1809,6 +1883,7 @@ jobs: shellcheck scripts/admit-coroot-baseline-context.sh scripts/tests/test-admit-coroot-baseline-context.sh bash scripts/tests/test-admit-coroot-baseline-context.sh - name: ๐Ÿ” Verify the controller-owned tenant pull secret + if: matrix.suite == 'observability' run: | shellcheck scripts/tests/test-tenant-kro-pull-secret.sh bash scripts/tests/test-tenant-kro-pull-secret.sh @@ -3880,6 +3955,7 @@ jobs: [ changes, validate-repository-guards, + validate-publication-generator, validate-go-regressions, validate-webapp-compiler, validate-consumer-discovery, @@ -4241,6 +4317,7 @@ jobs: [ changes, validate-repository-guards, + validate-publication-generator, validate-go-regressions, validate-webapp-compiler, validate-consumer-discovery, @@ -4340,6 +4417,7 @@ jobs: job-results: >- ${{ needs.changes.result }} ${{ needs.validate-repository-guards.result }} + ${{ needs.validate-publication-generator.result }} ${{ needs.validate-go-regressions.result }} ${{ needs.validate-webapp-compiler.result }} ${{ needs.validate-consumer-discovery.result }} @@ -4369,6 +4447,15 @@ jobs: exit 1 fi + - name: ๐Ÿ” Require publication generator validation + env: + GENERATOR_RESULT: ${{ needs.validate-publication-generator.result }} + run: | + if [ "$GENERATOR_RESULT" != success ]; then + echo "::error::Publication generator did not succeed ($GENERATOR_RESULT)" + exit 1 + fi + - name: ๐Ÿงช Require complete Go regressions env: REGRESSION_RESULT: ${{ needs.validate-go-regressions.result }} diff --git a/AGENTS.md b/AGENTS.md index 48b5fa884b..3b7e19bb24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -128,7 +128,7 @@ Timing evidence alone does not establish Coroot request attribution or recovery. CI runs **static manifest validation** on PRs that touch k8s-related paths (`k8s/**`, `ksail*.yaml`, `.sops.yaml`, `talos*/**`, the naming configuration `.github/manifest-naming.yaml`, the validation scripts `scripts/validate-embedded-json/` / `scripts/validate-embedded-json.sh` / `scripts/generate-kubescape-exceptions/`, or `ci.yaml` โ€” the authoritative list is the `k8s` filter in `.github/workflows/ci.yaml`) โ€” the `validate` job in `.github/workflows/ci.yaml` first json-parses every registered embedded-JSON ConfigMap key via [`scripts/validate-embedded-json.sh`](scripts/validate-embedded-json.sh) (keys listed in the Go command's `registeredKeys` or ending in `.json` โ€” schema validation treats such blobs as opaque strings, so a stray comma would otherwise ship silently; run `bash scripts/validate-embedded-json.sh` locally when touching one, from any directory via the script's absolute path), then runs `ksail workload validate` for both the local and prod overlays plus a Kubescape scan (`scripts/generate-kubescape-exceptions` converts the `ClusterSecurityException` CRs into Kubescape's exceptions format, then `ksail workload scan --framework nsa,mitre --exceptions --compliance-threshold ` gates on the combined score โ€” the exact floor lives in `ci.yaml`). It is fast, needs no secrets (so it runs on fork PRs too), and starts no cluster. PRs touching `talos/**` or `talos-local/**` additionally run the `validate-talos` job: it renders the machine config with every patch applied (placeholder values stand in for env-expanded secrets like `${WG_SERVER_PRIVATE_KEY}`) and `talosctl validate`s the result, so a broken patch or an empty env expansion fails the PR event instead of the merge group's deploy (#2477). There is **no longer a full-cluster system test**: the local Docker cluster is a thin manual test-bed (see [Local Development Cluster](#local-development-cluster)), not a CI prod stand-in. -The complete cluster-policy fixtures and their offline regression controls run concurrently with manifest validation in the `validate-cluster-policies` pull-request job, under the same `k8s` path selection. The required aggregate waits for this job and rejects an unsuccessful or unexpectedly skipped selected family. Run `bash scripts/tests/test-policy-ci-job.sh` to exercise that result boundary; policy evaluation uses the shared Kyverno CLI version from `.github/scripts/kyverno-version.sh` and requires no cluster or secrets. +The complete cluster-policy fixtures and offline regressions run in fixed suites of the `validate-cluster-policies` PR matrix, under the existing `k8s` path selection. At most three suites run concurrently in isolated checkouts; every regression step selects exactly one declared suite. The publication-generator regressions run independently of the unconditional repository guards. Required-result aggregation and production deployment wait for both jobs. Run `bash scripts/tests/test-policy-ci-job.sh` for routing and result-boundary controls; see the [partition decision](docs/adr/ci-regression-partitions.md). Policy evaluation uses the shared Kyverno CLI version from `.github/scripts/kyverno-version.sh` and needs no cluster or secrets. The Talos gate renders both roles, applies the reviewed KSail post-installer kernel-argument fold, then validates both. Production's extension path clears `extraKernelArgs` and pins the UKI command line; the local path without extensions is unchanged. The unconditional pin check and real offline positive/invalid-patch controls are documented in [`scripts/reconcile-talos-kernel-args/README.md`](scripts/reconcile-talos-kernel-args/README.md). The pin check compares the pinned KSail release's fold inputs by content with the audited sets, so a KSail bump needs a new source audit of this fold only when one of those inputs changed. diff --git a/docs/adr/ci-regression-partitions.md b/docs/adr/ci-regression-partitions.md new file mode 100644 index 0000000000..795bef8b72 --- /dev/null +++ b/docs/adr/ci-regression-partitions.md @@ -0,0 +1,56 @@ +# Partition complete offline CI regressions + +## Decision + +Run the complete cluster-policy regressions as explicit matrix suites in +`ci.yaml`. Each suite receives an isolated checkout and the same pinned tools and +read-only token. At most three suites run concurrently. Fail-fast is disabled, so +a failed suite does not cancel the other diagnostics. + +Cluster policies have fixtures, tenant, services, placement, storage and +observability suites. Every validation step belongs to exactly one declared +suite. The publication-generator suite runs in its own required job with the same +checkout, compiler and native tool setup. The other repository guards retain their +unconditional steps, including the checks that independently protect CI wiring. +Regressions that share one shell environment remain in the same step. + +The existing selection predicates remain authoritative. The required aggregate +consumes the complete matrix result and refuses failed, cancelled, missing or +unexpectedly skipped selected validation. It explicitly requires generator +success. Production deployment waits for both the generator and its existing +merge-group prerequisites. + +## Evidence and alternatives + +Successful run [40272630712](https://github.com/devantler-tech/platform/actions/runs/40272630712) +took 17m25s for repository guards and 21m06s for cluster policies. The generator +controls occupied 7m48s of the serial guard job. Manifest validation took 14m14s; +its unchanged duration and shared runner queues still bound overall feedback. + +[GitHub's matrix documentation](https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/run-job-variations) +defines independent job combinations, fail-fast and concurrency limits. The +inference is that overlapping independent suites can shorten the critical path; +only a complete first-attempt workflow measurement establishes the actual result. +The measurement includes setup and runner queue time. + +Running all scripts concurrently inside one checkout would require proving that +their temporary files, environment changes and fixture mutations never interfere. +Separate hosted jobs provide that isolation. Caching prior verdicts, omitting +tests and increasing timeouts do not meet the feedback or coverage requirement. + +## Verification and recovery + +`scripts/tests/test-policy-ci-job.sh` checks the authored partition and exercises +missing, duplicate and unknown suites, unselected and repeated regressions, +ignored failures and premature cancellation. The generator and existing policy +result controls exercise success and every selected non-success result. + +Before accepting a scheduling edit, compare its complete command inventory and +step inputs with the reviewed baseline. A passing partition check establishes +complete routing of the authored steps; it does not prove that a deleted command +was an authorized removal. Review and native CI retain that responsibility. + +Observe every matrix suite and the required aggregate at the candidate and merged +source. Record actual durations on #5362; a green workflow does not by itself +prove the 15-minute target. Recovery restores serial scheduling through a reviewed +change while retaining the same validation commands and required-result gates. diff --git a/scripts/tests/test-policy-ci-job.sh b/scripts/tests/test-policy-ci-job.sh index 09dc8fa37e..95c4230cf3 100755 --- a/scripts/tests/test-policy-ci-job.sh +++ b/scripts/tests/test-policy-ci-job.sh @@ -16,6 +16,107 @@ if ! jq -e --arg selection "github.event_name == 'pull_request' && needs.changes echo 'FAIL: independent policy job scheduling changed' >&2 exit 1 fi + +# Exercise the authored matrix routing, not a second list of validation commands. +# Every non-setup step must run in exactly one declared suite. An omitted or +# duplicate suite, a misspelled condition, or an unconditional regression would +# otherwise lose coverage or run the same tests repeatedly. +cat >"${work}/partition.jq" <<'JQ' + def partition($job; $groups; $setup): + ($job.strategy.matrix.suite // []) as $suites | + [$job.steps[] | select(.name as $name | ($setup | index($name)) == null)] as $checks | + ($job["runs-on"] == "ubuntu-latest") and + ($job.permissions == {contents: "read"}) and + (($job["continue-on-error"] // false) == false) and + ($job.strategy["fail-fast"] == false) and + ($job.strategy["max-parallel"] == 3) and + (($job.strategy.matrix // {} | keys) == ["suite"]) and + (($suites | sort) == ($groups | sort)) and + (($suites | unique | length) == ($suites | length)) and + all($setup[]; . as $name | + ([$job.steps[] | select(.name == $name and .if == null)] | length) == 1) and + all($job.steps[]; ((.["continue-on-error"] // false) == false)) and + all($checks[]; . as $step | + ([$suites[] | select($step.if == ("matrix.suite == \u0027" + . + "\u0027"))] | length) == 1) and + all($suites[]; . as $suite | + ([$checks[] | select(.if == ("matrix.suite == \u0027" + $suite + "\u0027"))] | length) > 0); + partition(.jobs["validate-cluster-policies"]; + ["fixtures", "tenant", "services", "placement", "storage", "observability"]; + ["๐Ÿ“‘ Checkout", "โš™๏ธ Setup Go", "๐Ÿ“Œ Install the reviewed YAML parser", + "โš–๏ธ Read Kyverno CLI version", "โš–๏ธ Install Kyverno CLI"]) +JQ + +partition_result() { + local input="$1" result=0 + jq -e -f "${work}/partition.jq" "$input" >"${work}/partition-result" || result=$? + case "$result" in + 0 | 1) return "$result" ;; + *) + echo 'FAIL: CI partition observation could not be evaluated' >&2 + return 2 + ;; + esac +} + +yq -o=json '.' "$workflow" >"${work}/workflow.json" +if ! partition_result "${work}/workflow.json"; then + echo 'FAIL: CI matrices must retain exactly one scheduled copy of every validation step' >&2 + exit 1 +fi + +reject_partition() { + local name="$1" mutation="$2" result=0 + jq "$mutation" "${work}/workflow.json" >"${work}/mutated.json" + partition_result "${work}/mutated.json" || result=$? + if [ "$result" -ne 1 ]; then + printf 'FAIL: %s must reject incomplete routing, got %s\n' "$name" "$result" >&2 + exit 1 + fi +} + +job=validate-cluster-policies +reject_partition "missing suite" "del(.jobs[\"$job\"].strategy.matrix.suite[0])" +reject_partition "duplicate suite" \ + ".jobs[\"$job\"].strategy.matrix.suite += [.jobs[\"$job\"].strategy.matrix.suite[0]]" +reject_partition "unknown suite" ".jobs[\"$job\"].strategy.matrix.suite += [\"unknown\"]" +reject_partition "early cancellation" ".jobs[\"$job\"].strategy[\"fail-fast\"] = true" +reject_partition "ignored failure" ".jobs[\"$job\"][\"continue-on-error\"] = true" +reject_partition "unselected regression" \ + "(.jobs[\"$job\"].steps[] | select(.if != null) | .if) = \"matrix.suite == 'unknown'\"" +reject_partition "repeated regression" \ + "(.jobs[\"$job\"].steps[] | select(.if != null)) |= del(.if)" + +echo 'PASS: the policy matrix preserves one scheduled copy of every regression; 7 broken routings refused' + +jq -e ' + .jobs["validate-publication-generator"] as $job | + ($job.permissions == {contents: "read"}) and + ($job | has("if") | not) and + (($job["continue-on-error"] // false) == false) and + all($job.steps[]; .if == null and ((.["continue-on-error"] // false) == false)) and + ([$job.steps[] | select(.name == "๐Ÿ” Validate the approved-revisions generator" + and .if == null and (.run | type == "string" and length > 0))] | length) == 1 and + (.jobs["ci-required-checks"].needs | index("validate-publication-generator") != null) and + (.jobs["deploy-prod"].needs | index("validate-publication-generator") != null) +' "${work}/workflow.json" >/dev/null +generator_step="$(jq -c '[.steps[] | select(.name == "๐Ÿ” Require publication generator validation")]' <<<"${gate}")" +jq -e 'length == 1 and .[0].env == { + GENERATOR_RESULT: "${{ needs.validate-publication-generator.result }}" +} and (.[0] | has("if") | not) + and ((.[0]["continue-on-error"] // false) == false) + and (.[0].run | type == "string" and length > 0)' <<<"${generator_step}" >/dev/null +jq -r '.[0].run' <<<"${generator_step}" >"${work}/generator-gate.sh" +for result in success failure cancelled skipped pending unknown ''; do + actual=0 + GENERATOR_RESULT="$result" bash -e "${work}/generator-gate.sh" >"${work}/generator-output" 2>&1 || actual=$? + expected=1 + [ "$result" != success ] || expected=0 + if [ "$actual" -ne "$expected" ]; then + printf 'FAIL: generator result=%s expected=%s actual=%s\n' "$result" "$expected" "$actual" >&2 + exit 1 + fi +done +echo 'PASS: the generator is required before deployment and every non-success result is refused' if ! jq -e '.needs | index("validate-cluster-policies") != null' <<<"${gate}" >/dev/null; then echo 'FAIL: required aggregate must wait for the independent policy job' >&2 exit 1 From 377732b1e13c7c23b49e0c2b0ee2f2ce4c70d490 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 09:35:02 +0100 Subject: [PATCH 2/8] perf(ci): move independent manifest regressions off the critical path --- .github/workflows/ci.yaml | 105 ++++++++++++---- AGENTS.md | 2 +- docs/adr/ci-regression-partitions.md | 22 ++++ scripts/tests/test-policy-ci-job.sh | 45 +++++++ scripts/validate-dr-signing/main_test.go | 2 + .../canonical_source_workflow_test.go | 113 ++++++++++++++---- .../coverage_test.go | 5 + .../tenant_bind_test.go | 35 +++++- 8 files changed, 279 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a763ff87ac..2c4c5e25bf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2135,13 +2135,6 @@ jobs: shellcheck scripts/tests/test-business-disclosure-reader.sh bash scripts/tests/test-business-disclosure-reader.sh - - name: ๐Ÿค Validate staged OpenBao standby repair - if: needs.changes.outputs.k8s == 'true' - run: | - shellcheck scripts/tests/test-openbao-standby-canary.sh - shellcheck scripts/tests/test-openbao-strategy-transition.sh - bash scripts/tests/test-openbao-standby-canary.sh - - name: ๐Ÿงฏ Validate daily and break-glass API boundaries if: needs.changes.outputs.k8s == 'true' # Daily Kubernetes/Talos access must stay read-only, while the root @@ -2982,14 +2975,6 @@ jobs: restore-keys: | kubeconform-schema-cache-v1- - - name: ๐Ÿ”’ Validate Crossplane provider egress policy - if: needs.changes.outputs.k8s == 'true' - # This test exercises KSail's in-process Helm renderer, so it must run - # after the pinned KSail setup and schema-cache restore above. The - # reviewed FQDN set must remain narrow while provider traffic stays on - # the direct L3/L4 path that completes upstream TLS handshakes. - run: bash scripts/tests/test-crossplane-egress-policy.sh - - name: ๐Ÿ” Validate Kyverno review admission boundaries if: needs.changes.outputs.k8s == 'true' # Render the pinned chart and exercise its match conditions against an @@ -3041,13 +3026,6 @@ jobs: shellcheck scripts/tests/test-private-mail-server.sh bash scripts/tests/test-private-mail-server.sh - - name: โœ‰๏ธ Prove native mail accounts and recipient restrictions - if: needs.changes.outputs.k8s == 'true' - timeout-minutes: 10 - run: | - shellcheck scripts/tests/test-private-mail-configuration.sh - bash scripts/tests/test-private-mail-configuration.sh - - name: ๐Ÿ›ก๏ธ Validate Kubescape self-hosted scan persistence if: needs.changes.outputs.k8s == 'true' # The in-cluster scanner has no SaaS backend. Versions before v4.0.12 @@ -3351,6 +3329,73 @@ jobs: shellcheck scripts/summarize-sarif-findings.sh bash scripts/summarize-sarif-findings.sh "${RUNNER_TEMP}/kubescape.raw.sarif" + validate-manifest-regressions: + name: ๐Ÿงช Validate Independent Manifest Regressions + needs: + - changes + if: github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: ๐Ÿ“‘ Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: โš™๏ธ Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + - name: ๐Ÿ“Œ Install the reviewed YAML parser + if: needs.changes.outputs.k8s == 'true' + env: + GOBIN: ${{ runner.temp }}/platform-validation-tools + run: | + go install github.com/mikefarah/yq/v4@v4.54.1 + printf '%s\n' "$GOBIN" >> "$GITHUB_PATH" + PATH="$GOBIN:$PATH" yq --version + - name: โš–๏ธ Read Kyverno CLI version + if: needs.changes.outputs.k8s == 'true' + id: kyverno-version + run: bash .github/scripts/kyverno-version.sh + - name: โš–๏ธ Install Kyverno CLI + if: needs.changes.outputs.k8s == 'true' + uses: kyverno/action-install-cli@fcee92fca5c883169ef9927acf543e0b5fc58289 # v0.2.0 + with: + release: ${{ steps.kyverno-version.outputs.release }} + - name: โš™๏ธ Setup KSail + shell: bash + env: + # renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?.+)$ + KSAIL_VERSION: "7.203.26" + GITHUB_TOKEN: ${{ github.token }} + run: | + .github/scripts/setup-ksail.sh + - name: ๐Ÿ“ฅ Restore kubeconform schema cache + id: kubeconform-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/ksail/kubeconform + key: kubeconform-schema-cache-v1-render-regressions-${{ github.run_id }} + restore-keys: | + kubeconform-schema-cache-v1- + - name: ๐Ÿค Validate staged OpenBao standby repair + if: needs.changes.outputs.k8s == 'true' + run: | + shellcheck scripts/tests/test-openbao-standby-canary.sh + shellcheck scripts/tests/test-openbao-strategy-transition.sh + bash scripts/tests/test-openbao-standby-canary.sh + - name: ๐Ÿ”’ Validate Crossplane provider egress policy + if: needs.changes.outputs.k8s == 'true' + run: bash scripts/tests/test-crossplane-egress-policy.sh + - name: โœ‰๏ธ Prove native mail accounts and recipient restrictions + if: needs.changes.outputs.k8s == 'true' + timeout-minutes: 10 + run: | + shellcheck scripts/tests/test-private-mail-configuration.sh + bash scripts/tests/test-private-mail-configuration.sh + validate-flux-bridge: name: ๐Ÿ” Validate Flux Credential Bridge needs: [changes] @@ -4318,6 +4363,7 @@ jobs: changes, validate-repository-guards, validate-publication-generator, + validate-manifest-regressions, validate-go-regressions, validate-webapp-compiler, validate-consumer-discovery, @@ -4418,6 +4464,7 @@ jobs: ${{ needs.changes.result }} ${{ needs.validate-repository-guards.result }} ${{ needs.validate-publication-generator.result }} + ${{ needs.validate-manifest-regressions.result }} ${{ needs.validate-go-regressions.result }} ${{ needs.validate-webapp-compiler.result }} ${{ needs.validate-consumer-discovery.result }} @@ -4456,6 +4503,20 @@ jobs: exit 1 fi + - name: ๐Ÿงช Require selected manifest regressions + env: + EVENT_NAME: ${{ github.event_name }} + SELECTED: ${{ needs.changes.outputs.k8s }} + REGRESSION_RESULT: ${{ needs.validate-manifest-regressions.result }} + run: | + case "$EVENT_NAME:$SELECTED:$REGRESSION_RESULT" in + pull_request:true:success | pull_request:false:skipped | merge_group:*:skipped) ;; + *) + printf 'Manifest regressions did not meet their selection/result boundary: %s:%s:%s\n' "$EVENT_NAME" "$SELECTED" "$REGRESSION_RESULT" >&2 + exit 1 + ;; + esac + - name: ๐Ÿงช Require complete Go regressions env: REGRESSION_RESULT: ${{ needs.validate-go-regressions.result }} diff --git a/AGENTS.md b/AGENTS.md index 3b7e19bb24..92b6047cd5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -128,7 +128,7 @@ Timing evidence alone does not establish Coroot request attribution or recovery. CI runs **static manifest validation** on PRs that touch k8s-related paths (`k8s/**`, `ksail*.yaml`, `.sops.yaml`, `talos*/**`, the naming configuration `.github/manifest-naming.yaml`, the validation scripts `scripts/validate-embedded-json/` / `scripts/validate-embedded-json.sh` / `scripts/generate-kubescape-exceptions/`, or `ci.yaml` โ€” the authoritative list is the `k8s` filter in `.github/workflows/ci.yaml`) โ€” the `validate` job in `.github/workflows/ci.yaml` first json-parses every registered embedded-JSON ConfigMap key via [`scripts/validate-embedded-json.sh`](scripts/validate-embedded-json.sh) (keys listed in the Go command's `registeredKeys` or ending in `.json` โ€” schema validation treats such blobs as opaque strings, so a stray comma would otherwise ship silently; run `bash scripts/validate-embedded-json.sh` locally when touching one, from any directory via the script's absolute path), then runs `ksail workload validate` for both the local and prod overlays plus a Kubescape scan (`scripts/generate-kubescape-exceptions` converts the `ClusterSecurityException` CRs into Kubescape's exceptions format, then `ksail workload scan --framework nsa,mitre --exceptions --compliance-threshold ` gates on the combined score โ€” the exact floor lives in `ci.yaml`). It is fast, needs no secrets (so it runs on fork PRs too), and starts no cluster. PRs touching `talos/**` or `talos-local/**` additionally run the `validate-talos` job: it renders the machine config with every patch applied (placeholder values stand in for env-expanded secrets like `${WG_SERVER_PRIVATE_KEY}`) and `talosctl validate`s the result, so a broken patch or an empty env expansion fails the PR event instead of the merge group's deploy (#2477). There is **no longer a full-cluster system test**: the local Docker cluster is a thin manual test-bed (see [Local Development Cluster](#local-development-cluster)), not a CI prod stand-in. -The complete cluster-policy fixtures and offline regressions run in fixed suites of the `validate-cluster-policies` PR matrix, under the existing `k8s` path selection. At most three suites run concurrently in isolated checkouts; every regression step selects exactly one declared suite. The publication-generator regressions run independently of the unconditional repository guards. Required-result aggregation and production deployment wait for both jobs. Run `bash scripts/tests/test-policy-ci-job.sh` for routing and result-boundary controls; see the [partition decision](docs/adr/ci-regression-partitions.md). Policy evaluation uses the shared Kyverno CLI version from `.github/scripts/kyverno-version.sh` and needs no cluster or secrets. +The complete cluster-policy fixtures and offline regressions run in fixed suites of the `validate-cluster-policies` PR matrix, under the existing `k8s` path selection. At most three suites run concurrently in isolated checkouts; every regression step selects exactly one declared suite. The publication-generator regressions run independently of the unconditional repository guards; required CI and production deployment both wait for the generator. The independent OpenBao, Crossplane egress and native-mail regressions run in the read-only `validate-manifest-regressions` PR job with the same pinned tools and inputs, alongside manifest validation. Required CI refuses an unsuccessful or unexpectedly skipped selected family. Run `bash scripts/tests/test-policy-ci-job.sh` for routing and result-boundary controls; see the [partition decision](docs/adr/ci-regression-partitions.md). Policy evaluation uses the shared Kyverno CLI version from `.github/scripts/kyverno-version.sh` and needs no cluster or secrets. The Talos gate renders both roles, applies the reviewed KSail post-installer kernel-argument fold, then validates both. Production's extension path clears `extraKernelArgs` and pins the UKI command line; the local path without extensions is unchanged. The unconditional pin check and real offline positive/invalid-patch controls are documented in [`scripts/reconcile-talos-kernel-args/README.md`](scripts/reconcile-talos-kernel-args/README.md). The pin check compares the pinned KSail release's fold inputs by content with the audited sets, so a KSail bump needs a new source audit of this fold only when one of those inputs changed. diff --git a/docs/adr/ci-regression-partitions.md b/docs/adr/ci-regression-partitions.md index 795bef8b72..2c4fa121d6 100644 --- a/docs/adr/ci-regression-partitions.md +++ b/docs/adr/ci-regression-partitions.md @@ -14,11 +14,22 @@ checkout, compiler and native tool setup. The other repository guards retain the unconditional steps, including the checks that independently protect CI wiring. Regressions that share one shell environment remain in the same step. +Move the independent OpenBao standby, Crossplane egress and native-mail +regressions out of the serial manifest job into one required PR job. Preserve +their complete steps, timeouts and Kubernetes-path selection. Copy the pinned +compiler, YAML parser, Kyverno, KSail and schema-cache restore prerequisites into +the isolated checkout. This job has only read access to contents; SARIF upload +remains in the existing manifest job. A distinct candidate cache key restores +from the existing schema prefix; cached data never substitutes for evaluation. + The existing selection predicates remain authoritative. The required aggregate consumes the complete matrix result and refuses failed, cancelled, missing or unexpectedly skipped selected validation. It explicitly requires generator success. Production deployment waits for both the generator and its existing merge-group prerequisites. +The independent manifest-regression family remains PR-only, like its original +execution. Required CI accepts success when selected and an intentional skip +otherwise, rejecting unknown selection and every selected non-success result. ## Evidence and alternatives @@ -27,6 +38,15 @@ took 17m25s for repository guards and 21m06s for cluster policies. The generator controls occupied 7m48s of the serial guard job. Manifest validation took 14m14s; its unchanged duration and shared runner queues still bound overall feedback. +The first [scheduling trial](https://github.com/devantler-tech/platform/actions/runs/40289350541) +reduced the guard job to 9m32s and completed its longest policy suite in 12m09s. +That attempt failed because existing validators assumed the old job locations; +it is not evidence of a successful complete feedback loop. Its manifest job took +17m22s. OpenBao standby, Crossplane egress and native mail accounted for 134s, +143s and 129s respectively, motivating the additional independent job. The +remaining validators now check the new owning jobs and their required +dependencies, retaining the existing lost-command and bypass controls. + [GitHub's matrix documentation](https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/run-job-variations) defines independent job combinations, fail-fast and concurrency limits. The inference is that overlapping independent suites can shorten the critical path; @@ -44,6 +64,8 @@ tests and increasing timeouts do not meet the feedback or coverage requirement. missing, duplicate and unknown suites, unselected and repeated regressions, ignored failures and premature cancellation. The generator and existing policy result controls exercise success and every selected non-success result. +The additional manifest-regression boundary exercises selection, success, +failure, cancellation, missing results and unexpected skips. Before accepting a scheduling edit, compare its complete command inventory and step inputs with the reviewed baseline. A passing partition check establishes diff --git a/scripts/tests/test-policy-ci-job.sh b/scripts/tests/test-policy-ci-job.sh index 95c4230cf3..5174311fbf 100755 --- a/scripts/tests/test-policy-ci-job.sh +++ b/scripts/tests/test-policy-ci-job.sh @@ -154,3 +154,48 @@ check merge_group true skipped 0 check merge_group false skipped 0 echo 'PASS: selected policy validation must succeed; intentional non-selection is accepted' + +jq -e --arg selection "github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true'" ' + . as $workflow | + .jobs["validate-manifest-regressions"] as $job | + ($job.needs == ["changes"]) and ($job.if == $selection) and + ($job.permissions == {contents: "read"}) and + (($job["continue-on-error"] // false) == false) and + (["๐Ÿค Validate staged OpenBao standby repair", + "๐Ÿ”’ Validate Crossplane provider egress policy", + "โœ‰๏ธ Prove native mail accounts and recipient restrictions"] | all(.[]; + . as $name | + ([$job.steps[] | select(.name == $name and + .if == "needs.changes.outputs.k8s == '\''true'\''" and + ((.["continue-on-error"] // false) == false) and + (.run | type == "string" and length > 0))] | length) == 1 and + ([$workflow.jobs.validate.steps[]? | select(.name == $name)] | length) == 0)) and + (.jobs["ci-required-checks"].needs | index("validate-manifest-regressions") != null) +' "${work}/workflow.json" >/dev/null +regression_step="$(jq -c '[.steps[] | select(.name == "๐Ÿงช Require selected manifest regressions")]' <<<"${gate}")" +jq -e 'length == 1 and .[0].env == { + EVENT_NAME: "${{ github.event_name }}", + SELECTED: "${{ needs.changes.outputs.k8s }}", + REGRESSION_RESULT: "${{ needs.validate-manifest-regressions.result }}" +} and (.[0] | has("if") | not) + and ((.[0]["continue-on-error"] // false) == false) + and (.[0].run | type == "string" and length > 0)' <<<"${regression_step}" >/dev/null +jq -r '.[0].run' <<<"${regression_step}" >"${work}/regression-gate.sh" +for event in pull_request merge_group; do + for selected in true false unknown ''; do + for result in success failure cancelled skipped pending unknown ''; do + expected=1 actual=0 + case "$event:$selected:$result" in + pull_request:true:success | pull_request:false:skipped | merge_group:*:skipped) expected=0 ;; + esac + EVENT_NAME="$event" SELECTED="$selected" REGRESSION_RESULT="$result" \ + bash -e "${work}/regression-gate.sh" >"${work}/regression-output" 2>&1 || actual=$? + if [ "$actual" -ne "$expected" ]; then + printf 'FAIL: manifest regressions event=%s selected=%s result=%s expected=%s actual=%s\n' \ + "$event" "$selected" "$result" "$expected" "$actual" >&2 + exit 1 + fi + done + done +done +echo 'PASS: all three manifest regressions run once and selected non-success outcomes are refused' diff --git a/scripts/validate-dr-signing/main_test.go b/scripts/validate-dr-signing/main_test.go index 2d9625b35b..76c2b693f6 100644 --- a/scripts/validate-dr-signing/main_test.go +++ b/scripts/validate-dr-signing/main_test.go @@ -1600,6 +1600,7 @@ func TestMergeQueueContractGateIsEnforced(t *testing.T) { [ changes, validate-repository-guards, + validate-publication-generator, validate-go-regressions, validate-webapp-compiler, validate-consumer-discovery, @@ -1617,6 +1618,7 @@ func TestMergeQueueContractGateIsEnforced(t *testing.T) { [ changes, validate-repository-guards, + validate-publication-generator, validate-go-regressions, validate-webapp-compiler, validate-consumer-discovery, diff --git a/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go b/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go index 4bafb1a9c0..992cc209d9 100644 --- a/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go +++ b/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go @@ -3,20 +3,21 @@ package main import ( "fmt" "os" + "slices" "strings" "testing" "gopkg.in/yaml.v3" ) -var canonicalCICommands = []string{ - "bash scripts/tests/test-canonical-source-bootstrap.sh", - "bash scripts/tests/test-canonical-constructor-attribution.sh", - "bash scripts/tests/test-canonical-generator-output.sh", - "bash scripts/tests/test-canonical-bootstrap-output.sh", - "bash scripts/tests/test-canonical-publisher-regeneration.sh", - "bash scripts/tests/test-tenant-source-discovery.sh", - "./scripts/guard-consumer-discovery-conservation.sh", +var canonicalCICommands = map[string]string{ + "bash scripts/tests/test-canonical-source-bootstrap.sh": "validate-publication-generator", + "bash scripts/tests/test-canonical-constructor-attribution.sh": "validate-publication-generator", + "bash scripts/tests/test-canonical-generator-output.sh": "validate-publication-generator", + "bash scripts/tests/test-canonical-bootstrap-output.sh": "validate-publication-generator", + "bash scripts/tests/test-canonical-publisher-regeneration.sh": "validate-publication-generator", + "bash scripts/tests/test-tenant-source-discovery.sh": "validate-repository-guards", + "./scripts/guard-consumer-discovery-conservation.sh": "validate-repository-guards", } var canonicalCIPaths = []string{ @@ -67,7 +68,8 @@ func canonicalCIWorkflow(t *testing.T) workflow { func mutateCanonicalCommand(t *testing.T, wf *workflow, command string, mutate func(*step)) { t.Helper() - j := wf.Jobs["validate-repository-guards"] + owner := canonicalCICommands[command] + j := wf.Jobs[owner] found := 0 for i := range j.Steps { if strings.Contains(j.Steps[i].Run, command) { @@ -78,7 +80,7 @@ func mutateCanonicalCommand(t *testing.T, wf *workflow, command string, mutate f if found != 1 { t.Fatalf("mutation requires exactly one native command block, got %d for %s", found, command) } - wf.Jobs["validate-repository-guards"] = j + wf.Jobs[owner] = j } func mutateCanonicalFilter(t *testing.T, wf *workflow, mutate func(map[string][]string)) { @@ -108,7 +110,7 @@ func mutateCanonicalFilter(t *testing.T, wf *workflow, mutate func(map[string][] } func TestCanonicalSourceCIRejectsLostOrNeutralizedCommands(t *testing.T) { - for _, command := range canonicalCICommands { + for command := range canonicalCICommands { for name, replacement := range map[string]string{ "removed": "", "comment": "# " + command, "echo": "echo " + command, "conditional": "if false; then " + command + "; fi", @@ -278,31 +280,92 @@ func TestCanonicalSourceCIRejectsGateAndFilterBypasses(t *testing.T) { for _, scope := range []string{"job", "step"} { wf = canonicalCIWorkflow(t) wf.Defaults.Run.Shell = "bash {0}" - j = wf.Jobs["validate-repository-guards"] - j.Defaults.Run.Shell = "bash" - if scope == "step" { - j.Defaults.Run.Shell = "bash {0}" - } - for i := range j.Steps { + for _, name := range []string{"validate-repository-guards", "validate-publication-generator"} { + j = wf.Jobs[name] + j.Defaults.Run.Shell = "bash" if scope == "step" { - j.Steps[i].Shell = "bash" - } else { - j.Steps[i].Shell = "" + j.Defaults.Run.Shell = "bash {0}" + } + for i := range j.Steps { + if scope == "step" { + j.Steps[i].Shell = "bash" + } else { + j.Steps[i].Shell = "" + } } + wf.Jobs[name] = j } - wf.Jobs["validate-repository-guards"] = j if err := canonicalSourceCIContract(wf); err != nil { t.Fatalf("rejected safe %s shell override: %v", scope, err) } } } +func TestCanonicalGeneratorCIRejectsLostRequiredJob(t *testing.T) { + for _, mutation := range []string{"missing", "conditional", "tolerated", "unsafe-shell", "deploy-prod", "ci-required-checks"} { + t.Run(mutation, func(t *testing.T) { + wf := canonicalCIWorkflow(t) + if err := canonicalSourceCIContract(wf); err != nil { + t.Fatalf("positive control: %v", err) + } + name := "validate-publication-generator" + j := wf.Jobs[name] + want := "require unconditional " + name + switch mutation { + case "missing": + delete(wf.Jobs, name) + case "conditional": + j.If = "false" + wf.Jobs[name] = j + case "tolerated": + j.ContinueOnError = true + wf.Jobs[name] = j + case "unsafe-shell": + j.Defaults.Run.Shell = "bash {0}" + wf.Jobs[name] = j + want = "canonical source CI must enforce exactly one" + default: + consumer := wf.Jobs[mutation] + needs, valid := consumer.needsOf() + if !valid || !slices.Contains(needs, name) { + t.Fatal("mutation must remove an actual required dependency") + } + kept := make([]any, 0, len(needs)-1) + for _, dependency := range needs { + if dependency != name { + kept = append(kept, dependency) + } + } + consumer.Needs = kept + wf.Jobs[mutation] = consumer + want = mutation + " must require the publication generator" + } + err := canonicalSourceCIContract(wf) + if err == nil { + t.Fatal("accepted lost generator enforcement") + } + if !strings.Contains(err.Error(), want) { + t.Fatalf("wrong refusal: want %q in %v", want, err) + } + }) + } +} + func canonicalSourceCIContract(wf workflow) error { - guard, ok := wf.Jobs["validate-repository-guards"] - if !ok || guard.If != "" || errorIsTolerated(guard.ContinueOnError) { - return fmt.Errorf("canonical source contracts require the unconditional repository guard job") + for _, name := range []string{"validate-repository-guards", "validate-publication-generator"} { + guard, ok := wf.Jobs[name] + if !ok || guard.If != "" || errorIsTolerated(guard.ContinueOnError) { + return fmt.Errorf("canonical source contracts require unconditional %s", name) + } + } + for _, consumer := range []string{"deploy-prod", "ci-required-checks"} { + needs, valid := wf.Jobs[consumer].needsOf() + if !valid || !slices.Contains(needs, "validate-publication-generator") { + return fmt.Errorf("%s must require the publication generator", consumer) + } } - for _, command := range canonicalCICommands { + for command, owner := range canonicalCICommands { + guard := wf.Jobs[owner] count := 0 for _, s := range guard.Steps { shell := s.Shell diff --git a/scripts/validate-eks-ci-role-policy/coverage_test.go b/scripts/validate-eks-ci-role-policy/coverage_test.go index 7d652669ac..862481a6b8 100644 --- a/scripts/validate-eks-ci-role-policy/coverage_test.go +++ b/scripts/validate-eks-ci-role-policy/coverage_test.go @@ -58,6 +58,11 @@ type job struct { ContinueOnError any `yaml:"continue-on-error"` Permissions any `yaml:"permissions"` Defaults runDefaults `yaml:"defaults"` + Strategy struct { + Matrix map[string]any `yaml:"matrix"` + FailFast any `yaml:"fail-fast"` + MaxParallel any `yaml:"max-parallel"` + } `yaml:"strategy"` } type runDefaults struct { diff --git a/scripts/validate-eks-ci-role-policy/tenant_bind_test.go b/scripts/validate-eks-ci-role-policy/tenant_bind_test.go index 0e73fac432..6ff3c56cc3 100644 --- a/scripts/validate-eks-ci-role-policy/tenant_bind_test.go +++ b/scripts/validate-eks-ci-role-policy/tenant_bind_test.go @@ -26,6 +26,19 @@ func tenantPullSecretCIContract(wf workflow) error { strings.Join(strings.Fields(j.If), " ") != "github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true'" { return fmt.Errorf("lost selected policy job enforcement") } + suites, valid := j.Strategy.Matrix["suite"].([]any) + want := map[string]bool{"fixtures": true, "tenant": true, "services": true, "placement": true, "storage": true, "observability": true} + if !valid || len(j.Strategy.Matrix) != 1 || len(suites) != len(want) || + j.Strategy.FailFast != false || j.Strategy.MaxParallel != 3 { + return fmt.Errorf("lost complete policy matrix routing") + } + for _, item := range suites { + name, isString := item.(string) + if !isString || !want[name] { + return fmt.Errorf("unknown or duplicate policy suite") + } + delete(want, name) + } count := 0 for _, s := range j.Steps { shell := s.Shell @@ -35,7 +48,8 @@ func tenantPullSecretCIContract(wf workflow) error { if shell == "" { shell = wf.Defaults.Run.Shell } - if s.Uses == "" && s.enforces() && shellKeepsErrexit(shell) { + if s.Uses == "" && s.If == "matrix.suite == 'observability'" && + !errorIsTolerated(s.ContinueOnError) && shellKeepsErrexit(shell) { count += canonicalCommandCount(s.Run, "bash scripts/tests/test-tenant-kro-pull-secret.sh") } } @@ -50,7 +64,10 @@ func TestTenantPullSecretAdmissionRemainsInCI(t *testing.T) { if err := tenantPullSecretCIContract(wf); err != nil { t.Fatal(err) } - for _, mutation := range []string{"missing", "skip", "tolerate", "comment", "mask", "unsafe-shell", "job-skip", "job-tolerate"} { + for _, mutation := range []string{ + "missing", "skip", "tolerate", "comment", "mask", "unsafe-shell", "job-skip", "job-tolerate", + "wrong-suite", "unconditional", "matrix-missing", "matrix-duplicate", "matrix-exclude", "matrix-unbounded", "matrix-cancel", + } { t.Run(mutation, func(t *testing.T) { w := canonicalCIWorkflow(t) j := w.Jobs["validate-cluster-policies"] @@ -78,6 +95,20 @@ func TestTenantPullSecretAdmissionRemainsInCI(t *testing.T) { j.If = "false" case "job-tolerate": j.ContinueOnError = true + case "wrong-suite": + s.If = "matrix.suite == 'unknown'" + case "unconditional": + s.If = "" + case "matrix-missing": + delete(j.Strategy.Matrix, "suite") + case "matrix-duplicate": + j.Strategy.Matrix["suite"] = []any{"fixtures", "tenant", "services", "placement", "storage", "storage"} + case "matrix-exclude": + j.Strategy.Matrix["exclude"] = []any{map[string]any{"suite": "observability"}} + case "matrix-unbounded": + j.Strategy.MaxParallel = 6 + case "matrix-cancel": + j.Strategy.FailFast = true } } if found != 1 { From 94f18dfe6116efded4c35adaaccd32157d720da2 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 10:32:29 +0100 Subject: [PATCH 3/8] perf(ci): bound rollout fixtures and balance manifest checks --- .github/workflows/ci.yaml | 82 ++++++------ AGENTS.md | 2 +- docs/adr/ci-regression-partitions.md | 25 +++- scripts/tests/lib/fixture-workers.lib.sh | 114 +++++++++++++++++ scripts/tests/test-data-product-ui-rollout.sh | 36 +++++- scripts/tests/test-fixture-workers.sh | 119 ++++++++++++++++++ scripts/tests/test-policy-ci-job.sh | 8 +- 7 files changed, 337 insertions(+), 49 deletions(-) create mode 100644 scripts/tests/lib/fixture-workers.lib.sh create mode 100644 scripts/tests/test-fixture-workers.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2c4c5e25bf..ff1804e216 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -209,6 +209,8 @@ jobs: - 'scripts/tests/test-data-product-controller-trial.sh' - 'scripts/tests/test-data-product-ui-host.sh' - 'scripts/tests/test-data-product-ui-rollout.sh' + - 'scripts/tests/test-fixture-workers.sh' + - 'scripts/tests/lib/fixture-workers.lib.sh' - 'scripts/tests/test-data-product-runtime-digests.sh' - 'scripts/verify-data-product-ui-rollout.sh' - 'scripts/tests/test-oauth2-proxy-sso-redirects.sh' @@ -1820,7 +1822,8 @@ jobs: bash scripts/tests/test-data-product-controller-trial.sh shellcheck scripts/tests/test-data-product-ui-host.sh bash scripts/tests/test-data-product-ui-host.sh - shellcheck scripts/verify-data-product-ui-rollout.sh scripts/tests/test-data-product-ui-rollout.sh + shellcheck -x scripts/verify-data-product-ui-rollout.sh scripts/tests/test-data-product-ui-rollout.sh scripts/tests/lib/fixture-workers.lib.sh scripts/tests/test-fixture-workers.sh + bash scripts/tests/test-fixture-workers.sh bash scripts/tests/test-data-product-ui-rollout.sh shellcheck scripts/tests/test-data-product-runtime-digests.sh bash scripts/tests/test-data-product-runtime-digests.sh @@ -2557,14 +2560,6 @@ jobs: shellcheck scripts/tests/test-coroot-alert-autosuppressor.sh bash scripts/tests/test-coroot-alert-autosuppressor.sh - - name: ๐Ÿฉบ Validate declarative Coroot application health - if: needs.changes.outputs.k8s == 'true' - # Exact reviewed app signals receive per-app thresholds; any novel log - # pattern resets its threshold so the Applications view fails visible. - run: | - shellcheck scripts/tests/test-coroot-application-health-reconciler.sh - bash scripts/tests/test-coroot-application-health-reconciler.sh - - name: ๐Ÿ“ฆ Validate Coroot helper image caching if: needs.changes.outputs.k8s == 'true' # A pinned digest on a warm node must not depend on Docker Hub token @@ -2667,15 +2662,6 @@ jobs: # finding (#3545), plus the wiring that puts it before the promotion. run: bash scripts/tests/test-verify-matcher-accepts-digest.sh - - name: ๐Ÿ” Validate the shared-publish-workflow pin guard - if: needs.changes.outputs.k8s == 'true' - # The guard fails by PASSING: a widened cosign matcher still verifies, so - # nothing else in CI notices. This pins the shapes it must reject โ€” above - # all the SHA-or-tag alternation removed in #3022, which is the exact - # prior contents of all eight subjects and would otherwise revert in - # silence โ€” plus the floor and the wiring that reaches it. - run: bash scripts/tests/test-shared-publish-workflow-pin-guard.sh - - name: ๐Ÿงท Validate checkov skip reasons are quoted and intact if: needs.changes.outputs.k8s == 'true' # A `checkov.io/skipN` whose reason parses away still suppresses its @@ -2768,18 +2754,6 @@ jobs: bash scripts/guard-baseline-context-optin-scan-scope.sh bash scripts/tests/test-guard-baseline-context-optin-scan-scope.sh - - name: ๐Ÿ—บ๏ธ Validate every unscanned namespace is reviewed - if: needs.changes.outputs.k8s == 'true' - # A namespace in the operator's excludeNamespaces produces no scan result, and the - # posture score cannot tell that apart from a clean namespace. So the exclusions must - # match scripts/kubescape-unscanned-namespaces.tsv exactly, each row with a reason, - # and a namespace can only leave scan coverage through a reviewed change (#3215). - # The base HelmRelease is the only place the field may be set: an overlay patch or a - # valuesFrom that sets it would narrow the deployed scope while this read the base. - run: | - bash scripts/guard-kubescape-scan-scope.sh - bash scripts/tests/test-guard-kubescape-scan-scope.sh - - name: ๐Ÿงฎ Test the Kubescape result-coverage check if: needs.changes.outputs.k8s == 'true' # The check compares live Kubescape results against the objects that should have one @@ -2975,15 +2949,6 @@ jobs: restore-keys: | kubeconform-schema-cache-v1- - - name: ๐Ÿ” Validate Kyverno review admission boundaries - if: needs.changes.outputs.k8s == 'true' - # Render the pinned chart and exercise its match conditions against an - # isolated real API server. Ordinary policy requests must still fail - # closed; only the three already-filtered virtual reviews skip dispatch. - run: | - shellcheck scripts/tests/test-kyverno-auth-reviews.sh - bash scripts/tests/test-kyverno-auth-reviews.sh - - name: ๐Ÿง  Validate Kyverno admission VPA safety if: needs.changes.outputs.k8s == 'true' # Render the production VPA and the actual Kyverno chart Deployment so @@ -3396,6 +3361,45 @@ jobs: shellcheck scripts/tests/test-private-mail-configuration.sh bash scripts/tests/test-private-mail-configuration.sh + - name: ๐Ÿ” Validate Kyverno review admission boundaries + if: needs.changes.outputs.k8s == 'true' + # Render the pinned chart and exercise its match conditions against an + # isolated real API server. Ordinary policy requests must still fail + # closed; only the three already-filtered virtual reviews skip dispatch. + run: | + shellcheck scripts/tests/test-kyverno-auth-reviews.sh + bash scripts/tests/test-kyverno-auth-reviews.sh + + - name: ๐Ÿ—บ๏ธ Validate every unscanned namespace is reviewed + if: needs.changes.outputs.k8s == 'true' + # A namespace in the operator's excludeNamespaces produces no scan result, and the + # posture score cannot tell that apart from a clean namespace. So the exclusions must + # match scripts/kubescape-unscanned-namespaces.tsv exactly, each row with a reason, + # and a namespace can only leave scan coverage through a reviewed change (#3215). + # The base HelmRelease is the only place the field may be set: an overlay patch or a + # valuesFrom that sets it would narrow the deployed scope while this read the base. + run: | + bash scripts/guard-kubescape-scan-scope.sh + bash scripts/tests/test-guard-kubescape-scan-scope.sh + + - name: ๐Ÿ” Validate the shared-publish-workflow pin guard + if: needs.changes.outputs.k8s == 'true' + # The guard fails by PASSING: a widened cosign matcher still verifies, so + # nothing else in CI notices. This pins the shapes it must reject โ€” above + # all the SHA-or-tag alternation removed in #3022, which is the exact + # prior contents of all eight subjects and would otherwise revert in + # silence โ€” plus the floor and the wiring that reaches it. + run: bash scripts/tests/test-shared-publish-workflow-pin-guard.sh + + - name: ๐Ÿฉบ Validate declarative Coroot application health + if: needs.changes.outputs.k8s == 'true' + # Exact reviewed app signals receive per-app thresholds; any novel log + # pattern resets its threshold so the Applications view fails visible. + run: | + shellcheck scripts/tests/test-coroot-application-health-reconciler.sh + bash scripts/tests/test-coroot-application-health-reconciler.sh + + validate-flux-bridge: name: ๐Ÿ” Validate Flux Credential Bridge needs: [changes] diff --git a/AGENTS.md b/AGENTS.md index 92b6047cd5..8154f855aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -128,7 +128,7 @@ Timing evidence alone does not establish Coroot request attribution or recovery. CI runs **static manifest validation** on PRs that touch k8s-related paths (`k8s/**`, `ksail*.yaml`, `.sops.yaml`, `talos*/**`, the naming configuration `.github/manifest-naming.yaml`, the validation scripts `scripts/validate-embedded-json/` / `scripts/validate-embedded-json.sh` / `scripts/generate-kubescape-exceptions/`, or `ci.yaml` โ€” the authoritative list is the `k8s` filter in `.github/workflows/ci.yaml`) โ€” the `validate` job in `.github/workflows/ci.yaml` first json-parses every registered embedded-JSON ConfigMap key via [`scripts/validate-embedded-json.sh`](scripts/validate-embedded-json.sh) (keys listed in the Go command's `registeredKeys` or ending in `.json` โ€” schema validation treats such blobs as opaque strings, so a stray comma would otherwise ship silently; run `bash scripts/validate-embedded-json.sh` locally when touching one, from any directory via the script's absolute path), then runs `ksail workload validate` for both the local and prod overlays plus a Kubescape scan (`scripts/generate-kubescape-exceptions` converts the `ClusterSecurityException` CRs into Kubescape's exceptions format, then `ksail workload scan --framework nsa,mitre --exceptions --compliance-threshold ` gates on the combined score โ€” the exact floor lives in `ci.yaml`). It is fast, needs no secrets (so it runs on fork PRs too), and starts no cluster. PRs touching `talos/**` or `talos-local/**` additionally run the `validate-talos` job: it renders the machine config with every patch applied (placeholder values stand in for env-expanded secrets like `${WG_SERVER_PRIVATE_KEY}`) and `talosctl validate`s the result, so a broken patch or an empty env expansion fails the PR event instead of the merge group's deploy (#2477). There is **no longer a full-cluster system test**: the local Docker cluster is a thin manual test-bed (see [Local Development Cluster](#local-development-cluster)), not a CI prod stand-in. -The complete cluster-policy fixtures and offline regressions run in fixed suites of the `validate-cluster-policies` PR matrix, under the existing `k8s` path selection. At most three suites run concurrently in isolated checkouts; every regression step selects exactly one declared suite. The publication-generator regressions run independently of the unconditional repository guards; required CI and production deployment both wait for the generator. The independent OpenBao, Crossplane egress and native-mail regressions run in the read-only `validate-manifest-regressions` PR job with the same pinned tools and inputs, alongside manifest validation. Required CI refuses an unsuccessful or unexpectedly skipped selected family. Run `bash scripts/tests/test-policy-ci-job.sh` for routing and result-boundary controls; see the [partition decision](docs/adr/ci-regression-partitions.md). Policy evaluation uses the shared Kyverno CLI version from `.github/scripts/kyverno-version.sh` and needs no cluster or secrets. +The complete cluster-policy fixtures and offline regressions run in fixed suites of the `validate-cluster-policies` PR matrix, under the existing `k8s` path selection. At most three suites run concurrently in isolated checkouts; every regression step selects exactly one declared suite. The publication-generator regressions run independently of the unconditional repository guards; required CI and production deployment both wait for the generator. Seven independent manifest regressions run in the read-only `validate-manifest-regressions` PR job with the same pinned tools and inputs, alongside manifest validation. The rollout fixture's independent mutation families use two isolated workers, retaining all cases, assertions and deadlines; foreground cases and shared-input changes wait for them. Required CI refuses an unsuccessful or unexpectedly skipped selected family. Run `bash scripts/tests/test-policy-ci-job.sh` for routing and result-boundary controls and `bash scripts/tests/test-fixture-workers.sh` for worker failure and completion controls; see the [partition decision](docs/adr/ci-regression-partitions.md). Policy evaluation uses the shared Kyverno CLI version from `.github/scripts/kyverno-version.sh` and needs no cluster or secrets. The Talos gate renders both roles, applies the reviewed KSail post-installer kernel-argument fold, then validates both. Production's extension path clears `extraKernelArgs` and pins the UKI command line; the local path without extensions is unchanged. The unconditional pin check and real offline positive/invalid-patch controls are documented in [`scripts/reconcile-talos-kernel-args/README.md`](scripts/reconcile-talos-kernel-args/README.md). The pin check compares the pinned KSail release's fold inputs by content with the audited sets, so a KSail bump needs a new source audit of this fold only when one of those inputs changed. diff --git a/docs/adr/ci-regression-partitions.md b/docs/adr/ci-regression-partitions.md index 2c4fa121d6..112ad908c2 100644 --- a/docs/adr/ci-regression-partitions.md +++ b/docs/adr/ci-regression-partitions.md @@ -14,7 +14,8 @@ checkout, compiler and native tool setup. The other repository guards retain the unconditional steps, including the checks that independently protect CI wiring. Regressions that share one shell environment remain in the same step. -Move the independent OpenBao standby, Crossplane egress and native-mail +Move the independent OpenBao standby, Crossplane egress, native-mail, Kyverno +review-admission, unscanned-namespace, publisher-pin and Coroot application-health regressions out of the serial manifest job into one required PR job. Preserve their complete steps, timeouts and Kubernetes-path selection. Copy the pinned compiler, YAML parser, Kyverno, KSail and schema-cache restore prerequisites into @@ -31,6 +32,14 @@ The independent manifest-regression family remains PR-only, like its original execution. Required CI accepts success when selected and an intentional skip otherwise, rejecting unknown selection and every selected non-success result. +The rollout regression's four snapshot-mutation families use two test-only +workers. Each case owns its directory and reads an immutable seed. Foreground +cases drain the workers first; shared policy changes have an explicit barrier. +Every assigned case must return successfully and leave its exact completion +receipt. Interruption, failure, missing receipts and duplicate assignments fail +the suite, and cleanup joins the workers before removing shared scratch. The +production verifier and all fixture assertions and real deadlines stay intact. + ## Evidence and alternatives Successful run [40272630712](https://github.com/devantler-tech/platform/actions/runs/40272630712) @@ -47,6 +56,13 @@ it is not evidence of a successful complete feedback loop. Its manifest job took remaining validators now check the new owning jobs and their required dependencies, retaining the existing lost-command and bypass controls. +The next [complete green trial](https://github.com/devantler-tech/platform/actions/runs/40293296500) +took 17m12s including queue and setup. Manifest validation took 11m52s and its +parallel regression job 6m23s, leaving room to rebalance four unchanged checks. +The services policy suite took 12m11s; the rollout regression alone consumed +11m10s. These observations motivate the bounded fixture workers and rebalancing; +the complete feedback target remains unproven until another native run finishes. + [GitHub's matrix documentation](https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/run-job-variations) defines independent job combinations, fail-fast and concurrency limits. The inference is that overlapping independent suites can shorten the critical path; @@ -55,7 +71,9 @@ The measurement includes setup and runner queue time. Running all scripts concurrently inside one checkout would require proving that their temporary files, environment changes and fixture mutations never interfere. -Separate hosted jobs provide that isolation. Caching prior verdicts, omitting +Separate hosted jobs provide that isolation. The narrowly scoped rollout workers +instead prove distinct case directories and immutable inputs, with barriers for +shared changes and serial foreground cases. Caching prior verdicts, omitting tests and increasing timeouts do not meet the feedback or coverage requirement. ## Verification and recovery @@ -66,6 +84,9 @@ ignored failures and premature cancellation. The generator and existing policy result controls exercise success and every selected non-success result. The additional manifest-regression boundary exercises selection, success, failure, cancellation, missing results and unexpected skips. +`scripts/tests/test-fixture-workers.sh` proves actual overlap, complete receipts, +serial operation and rejection of failures, early errors, duplicate or unknown +cases, missing or corrupt receipts, interruption and invalid worker bounds. Before accepting a scheduling edit, compare its complete command inventory and step inputs with the reviewed baseline. A passing partition check establishes diff --git a/scripts/tests/lib/fixture-workers.lib.sh b/scripts/tests/lib/fixture-workers.lib.sh new file mode 100644 index 0000000000..e185dcd207 --- /dev/null +++ b/scripts/tests/lib/fixture-workers.lib.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Test-only workers. Callbacks own distinct case directories and read immutable +# seed data. Drain before a foreground case or any shared-input mutation. + +fixture_pool_error() { + printf 'fixture-workers: %s\n' "$*" >&2 + return 2 +} + +fixture_pool_init() { + case "$2" in 1 | 2) ;; *) + fixture_pool_error 'workers must be 1 or 2' + return 2 + ;; + esac + FIXTURE_POOL_ROOT="$1/fixture-workers" + mkdir "$FIXTURE_POOL_ROOT" "$FIXTURE_POOL_ROOT/logs" "$FIXTURE_POOL_ROOT/receipts" || return 2 + FIXTURE_POOL_JOBS=$2 FIXTURE_POOL_PENDING=0 FIXTURE_POOL_FAILED=0 FIXTURE_POOL_INTERRUPTED=0 + FIXTURE_POOL_FINISHED=0 FIXTURE_POOL_VERDICT=0 + : >"$FIXTURE_POOL_ROOT/expected" + : >"$FIXTURE_POOL_ROOT/pending" + : >"$FIXTURE_POOL_ROOT/callbacks" +} + +fixture_pool_register() { + if ! [[ "$1" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || ! declare -F "$1" >/dev/null; then + fixture_pool_error 'callback is not a declared function' + return 2 + fi + if grep -qxF "$1" "$FIXTURE_POOL_ROOT/callbacks"; then + fixture_pool_error 'duplicate callback' + return 2 + fi + printf '%s\n' "$1" >>"$FIXTURE_POOL_ROOT/callbacks" +} + +fixture_pool_interrupt() { FIXTURE_POOL_INTERRUPTED=1; } + +fixture_pool_drain() { + [ -n "${FIXTURE_POOL_ROOT:-}" ] || return 0 + local pid name result + while read -r pid name; do + [ -n "$pid" ] || continue + while :; do + result=0 + wait "$pid" || result=$? + if [ "$result" = 0 ]; then break; fi + if [ "$result" = 127 ] || ! kill -0 "$pid" 2>/dev/null; then + FIXTURE_POOL_FAILED=1 + break + fi + done + cat "$FIXTURE_POOL_ROOT/logs/$name" || FIXTURE_POOL_FAILED=1 + done <"$FIXTURE_POOL_ROOT/pending" + : >"$FIXTURE_POOL_ROOT/pending" + FIXTURE_POOL_PENDING=0 + [ "$FIXTURE_POOL_INTERRUPTED" = 0 ] || return 2 + [ "$FIXTURE_POOL_FAILED" = 0 ] +} + +fixture_pool_submit() { + local callback=$1 name=$2 pid + shift + [ "$FIXTURE_POOL_FINISHED" = 0 ] && [ "$FIXTURE_POOL_INTERRUPTED" = 0 ] || return 2 + grep -qxF "$callback" "$FIXTURE_POOL_ROOT/callbacks" || { + fixture_pool_error 'unregistered callback' + return 2 + } + [[ "$name" =~ ^[a-zA-Z0-9_-]+$ ]] || { + fixture_pool_error 'invalid case identifier' + return 2 + } + if grep -qxF "$name" "$FIXTURE_POOL_ROOT/expected"; then + fixture_pool_error 'duplicate case assignment' + return 2 + fi + printf '%s\n' "$name" >>"$FIXTURE_POOL_ROOT/expected" + ( + # The parent owns shared scratch; a worker must never run its EXIT cleanup. + trap - EXIT HUP INT TERM + set -euo pipefail + export FIXTURE_POOL_WORKER=1 + "$callback" "$@" + printf '%s\n' "$name" >"$FIXTURE_POOL_ROOT/receipts/$name" + ) >"$FIXTURE_POOL_ROOT/logs/$name" 2>&1 & + pid=$! + printf '%s %s\n' "$pid" "$name" >>"$FIXTURE_POOL_ROOT/pending" + FIXTURE_POOL_PENDING=$((FIXTURE_POOL_PENDING + 1)) + [ "$FIXTURE_POOL_INTERRUPTED" = 0 ] || return 2 + if [ "$FIXTURE_POOL_PENDING" -ge "$FIXTURE_POOL_JOBS" ]; then fixture_pool_drain; fi +} + +fixture_pool_finish() { + [ -n "${FIXTURE_POOL_ROOT:-}" ] || return 0 + if [ "$FIXTURE_POOL_FINISHED" = 1 ]; then return "$FIXTURE_POOL_VERDICT"; fi + local result=0 name count receipts=0 file + fixture_pool_drain || result=$? + while IFS= read -r name; do + if ! printf '%s\n' "$name" | cmp -s - "$FIXTURE_POOL_ROOT/receipts/$name"; then result=1; fi + done <"$FIXTURE_POOL_ROOT/expected" + count="$(wc -l <"$FIXTURE_POOL_ROOT/expected" | tr -d ' ')" + for file in "$FIXTURE_POOL_ROOT/receipts/"*; do + [ -f "$file" ] || continue + receipts=$((receipts + 1)) + done + [ "$receipts" = "$count" ] || result=1 + if [ "$FIXTURE_POOL_INTERRUPTED" != 0 ]; then result=2; fi + FIXTURE_POOL_FINISHED=1 FIXTURE_POOL_VERDICT=$result + if [ "$result" != 0 ]; then + printf 'fixture-workers: incomplete or failed cases\n' >&2 + return "$result" + fi + printf 'FIXTURE-WORKERS COMPLETE cases=%s completed=%s workers=%s\n' "$count" "$receipts" "$FIXTURE_POOL_JOBS" +} diff --git a/scripts/tests/test-data-product-ui-rollout.sh b/scripts/tests/test-data-product-ui-rollout.sh index 8e27d23bf9..57ffcfea94 100755 --- a/scripts/tests/test-data-product-ui-rollout.sh +++ b/scripts/tests/test-data-product-ui-rollout.sh @@ -3,8 +3,20 @@ set -euo pipefail umask 077 root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" readonly script="${root_dir}/scripts/verify-data-product-ui-rollout.sh" +# shellcheck source=scripts/tests/lib/fixture-workers.lib.sh +source "${root_dir}/scripts/tests/lib/fixture-workers.lib.sh" scratch="$(mktemp -d)" -trap 'rm -rf "${scratch}"' EXIT +fixture_pool_init "$scratch" 2 +cleanup() { + local result=$? workers_result=0 + trap - EXIT + fixture_pool_finish >/dev/null || workers_result=$? + rm -rf "${scratch}" + if [[ "$result" == 0 ]]; then result=$workers_result; fi + exit "$result" +} +trap cleanup EXIT +trap fixture_pool_interrupt HUP INT TERM # fail reports the violated fixture invariant and stops the regression suite. fail() { printf 'FAIL: %s\n' "$1" >&2 @@ -346,6 +358,10 @@ chmod +x "${scratch}/bin/kubectl" "${scratch}/bin/curl" expected_urls=14 # run_case exercises one fixture and checks its verdict, cleanup and sanitized output. run_case() { + # Foreground cases and shared-input changes never overlap the two workers. + # The child deliberately reads its own exported flag; the parent stays unset. + # shellcheck disable=SC2031 + if [[ ${FIXTURE_POOL_WORKER:-0} != 1 ]]; then fixture_pool_drain; fi local name=$1 expected=$2 mode=${3:-} result=0 timeout_seconds=${4:-15} # Healthy fixtures include the expanded sequential public probes and full # snapshots. Keep rejection/cleanup deadline controls separately bounded. @@ -526,7 +542,7 @@ for mode in dns-failure tls-failure tls-handshake-failure request-timeout; do ru run_case transfer-timeout fail transfer-timeout 15 # mutate_case verifies that a dormant-profile mutation reaches snapshot rejection. -mutate_case() { +mutate_case_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" cp "${scratch}/healthy/"*.json "${scratch}/$name/" @@ -534,6 +550,8 @@ mutate_case() { run_case "$name" fail '' "${4:-3}" [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } +fixture_pool_register mutate_case_impl +mutate_case() { fixture_pool_submit mutate_case_impl "$@"; } mutate_case ui-deployment-missing-contract deployment-ui-kit '.spec.template.spec.containers[0].env |= map(select(.name!="UI_CONTRACT_ENABLED"))' mutate_case ui-deployment-disabled-appearance deployment-ui-kit '.spec.template.spec.containers[0].env[1].value="false"' @@ -573,7 +591,7 @@ done run_case publisher-unplanned-route fail publisher-unplanned-route 15 # publisher_mutation starts from the separately declared enabled profile. -publisher_mutation() { +publisher_mutation_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" cp "${scratch}/declared-publisher-true/"*.json "${scratch}/$name/" @@ -581,6 +599,8 @@ publisher_mutation() { run_case "$name" fail '' 3 [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } +fixture_pool_register publisher_mutation_impl +publisher_mutation() { fixture_pool_submit publisher_mutation_impl "$@"; } publisher_mutation declared-publisher-missing-deployment deployment-ui-kit '.spec.template.spec.containers[0].env |= map(select(.name!="PUBLISHER_PREFLIGHT_ENABLED"))' publisher_mutation declared-publisher-missing-pod pods '.items[4].spec.containers[0].env |= map(select(.name!="PUBLISHER_PREFLIGHT_ENABLED"))' publisher_mutation declared-publisher-missing-replicaset replicasets '.items |= map(if .spec.template.spec.containers[0].name=="ui-kit" then .spec.template.spec.containers[0].env |= map(select(.name!="PUBLISHER_PREFLIGHT_ENABLED")) else . end)' @@ -678,7 +698,7 @@ jq '.metadata.name="ksail-releases" | .metadata.uid="release-product-uid" | expected_urls=17 run_case ksail pass # ksail_mutation requires a real snapshot-predicate rejection for each control. -ksail_mutation() { +ksail_mutation_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" cp "${scratch}/ksail/"*.json "${scratch}/$name/" @@ -686,6 +706,8 @@ ksail_mutation() { run_case "$name" fail '' 3 [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } +fixture_pool_register ksail_mutation_impl +ksail_mutation() { fixture_pool_submit ksail_mutation_impl "$@"; } ksail_mutation ksail-disabled-live helm '.spec.values.demoProduct.ksailReleases.enabled=false' ksail_mutation ksail-stale-product release-product '.status.observedGeneration=2' ksail_mutation ksail-failed-product release-product '.status.conditions[0].status="False"' @@ -715,7 +737,7 @@ expected_urls=14 jq -e '.retaken == 1 and .settled == ["apps.metadata.labels"]' "${scratch}/active-settled-label/stdout" >/dev/null || fail 'active-settled-label: the active report did not name the change that settled' # active_mutation verifies that an active-profile mutation reaches snapshot rejection. -active_mutation() { +active_mutation_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" cp "${scratch}/active/"*.json "${scratch}/$name/" @@ -723,6 +745,8 @@ active_mutation() { run_case "$name" fail '' "${4:-3}" [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } +fixture_pool_register active_mutation_impl +active_mutation() { fixture_pool_submit active_mutation_impl "$@"; } active_mutation active-stale-contract-condition product '.status.conditions[2].observedGeneration=2' active_mutation active-false-connector product '.status.conditions[1].status="False"' active_mutation active-missing-contract-condition product '.status.conditions |= map(select(.type!="ContractsReady"))' @@ -982,6 +1006,7 @@ for version in v1 ''; do done mkdir "${scratch}/declared-policy-update" cp "${scratch}/healthy/"*.json "${scratch}/declared-policy-update/" +fixture_pool_drain cp "${scratch}/apps-verify.json" "${scratch}/saved-policy.json" jq '.matchOIDCIdentity[0].subject="new-explicit-declared-publisher"' "${scratch}/saved-policy.json" >"${scratch}/apps-verify.json" jq --slurpfile policy "${scratch}/apps-verify.json" '.spec.verify=$policy[0]' "${scratch}/healthy/root.json" >"${scratch}/declared-policy-update/root.json" @@ -996,4 +1021,5 @@ run_case bounded-preflight fail preflight-timeout 1 # Two seconds of deliberate preflight plus a three-second startup/cleanup budget # reaches the hanging plugin; the child still sleeps far beyond the deadline. run_case bounded-credential-plugin fail hang 5 +fixture_pool_finish printf 'PASS: rollout readback behavior and bounded, read-only request scopes\n' diff --git a/scripts/tests/test-fixture-workers.sh b/scripts/tests/test-fixture-workers.sh new file mode 100644 index 0000000000..b2dced408d --- /dev/null +++ b/scripts/tests/test-fixture-workers.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +cat >"$work/driver" <<'DRIVER' +#!/usr/bin/env bash +set -euo pipefail +source "$POOL_LIBRARY" +fixture_pool_init "$FIXTURE" "${JOBS:-2}" +cleanup_driver() { + local result=$? + trap - EXIT + fixture_pool_finish >/dev/null 2>&1 || true + exit "$result" +} +trap cleanup_driver EXIT +trap fixture_pool_interrupt HUP INT TERM +work_case() { + local name=$1 + mkdir "$FIXTURE/active-$name" + [ "$(find "$FIXTURE" -maxdepth 1 -type d -name 'active-*' | wc -l | tr -d ' ')" -le 2 ] + [ "$name" != failed ] || return 17 + if [ "$name" = early-error ]; then false; printf 'unsafe\n' >"$FIXTURE/after-error"; fi + if [ "$MODE" = good ] && [ "$FIXTURE_POOL_JOBS" = 2 ]; then + # The first worker must observe its peer before either completes. This + # proves overlap rather than relying on a timing-only speed assertion. + local started=$SECONDS + until [ "$(find "$FIXTURE" -maxdepth 1 -type d -name 'active-*' | wc -l | tr -d ' ')" = 2 ]; do + [ "$((SECONDS - started))" -lt 5 ] || return 19 + sleep 0.02 + done + fi + sleep 0.15 + [ -d "$FIXTURE" ] + printf '%s\n' "$name" >"$FIXTURE/result-$name" + rmdir "$FIXTURE/active-$name" +} +fixture_pool_register work_case +case "$MODE" in +good) + for name in alpha beta gamma delta; do fixture_pool_submit work_case "$name"; done + ;; +failed | early-error) fixture_pool_submit work_case "$MODE" ;; +missing | corrupt) + fixture_pool_submit work_case alpha + fixture_pool_drain + if [ "$MODE" = missing ]; then rm "$FIXTURE/fixture-workers/receipts/alpha"; + else printf 'other\n' >"$FIXTURE/fixture-workers/receipts/alpha"; fi + ;; +duplicate) + fixture_pool_submit work_case alpha + fixture_pool_submit work_case alpha + ;; +unregistered) fixture_pool_submit unknown alpha ;; +interrupted) + fixture_pool_submit work_case alpha + fixture_pool_interrupt + ;; +signalled) + fixture_pool_submit work_case alpha + kill -TERM "$$" + ;; +esac +fixture_pool_finish +DRIVER +chmod +x "$work/driver" +library="$root/scripts/tests/lib/fixture-workers.lib.sh" +for mode in good failed early-error missing corrupt duplicate unregistered interrupted signalled; do + mkdir "$work/$mode" + result=0 + POOL_LIBRARY="$library" FIXTURE="$work/$mode" MODE="$mode" \ + "$BASH" "$work/driver" >"$work/$mode.log" 2>&1 || result=$? + if [ "$mode" = good ]; then + if [ "$result" != 0 ] || ! grep -q 'FIXTURE-WORKERS COMPLETE cases=4 completed=4 workers=2' "$work/$mode.log"; then + cat "$work/$mode.log" >&2 + exit 1 + fi + for name in alpha beta gamma delta; do [ "$(cat "$work/$mode/result-$name")" = "$name" ]; done + elif [ "$result" = 0 ]; then + printf 'FAIL: %s was accepted\n' "$mode" >&2 + exit 1 + else + case "$mode" in + duplicate) reason='duplicate case assignment' ;; + unregistered) reason='unregistered callback' ;; + *) reason='incomplete or failed cases' ;; + esac + grep -q "$reason" "$work/$mode.log" || { + cat "$work/$mode.log" >&2 + exit 1 + } + if grep -q 'FIXTURE-WORKERS COMPLETE' "$work/$mode.log"; then + echo 'FAIL: unsuccessful work emitted a completion verdict' >&2 + exit 1 + fi + fi + case "$mode" in + interrupted | signalled) [ "$(cat "$work/$mode/result-alpha")" = alpha ] ;; + esac + [ ! -e "$work/$mode/after-error" ] || { + echo 'FAIL: worker ignored early failure' >&2 + exit 1 + } +done +mkdir "$work/serial" +POOL_LIBRARY="$library" FIXTURE="$work/serial" MODE=good JOBS=1 \ + "$BASH" "$work/driver" >"$work/serial.log" 2>&1 +grep -q 'FIXTURE-WORKERS COMPLETE cases=4 completed=4 workers=1' "$work/serial.log" +for jobs in 0 3 unknown; do + mkdir "$work/jobs-$jobs" + if POOL_LIBRARY="$library" FIXTURE="$work/jobs-$jobs" MODE=good JOBS="$jobs" \ + "$BASH" "$work/driver" >"$work/jobs-$jobs.log" 2>&1; then + echo 'FAIL: invalid worker bound accepted' >&2 + exit 1 + fi + grep -q 'workers must be 1 or 2' "$work/jobs-$jobs.log" +done +printf 'PASS: complete isolated workers; failures, early errors, missing/corrupt receipts, duplicate/unknown cases, interruption and invalid bounds refused\n' diff --git a/scripts/tests/test-policy-ci-job.sh b/scripts/tests/test-policy-ci-job.sh index 5174311fbf..1186be3406 100755 --- a/scripts/tests/test-policy-ci-job.sh +++ b/scripts/tests/test-policy-ci-job.sh @@ -163,7 +163,11 @@ jq -e --arg selection "github.event_name == 'pull_request' && needs.changes.outp (($job["continue-on-error"] // false) == false) and (["๐Ÿค Validate staged OpenBao standby repair", "๐Ÿ”’ Validate Crossplane provider egress policy", - "โœ‰๏ธ Prove native mail accounts and recipient restrictions"] | all(.[]; + "โœ‰๏ธ Prove native mail accounts and recipient restrictions", + "๐Ÿ” Validate Kyverno review admission boundaries", + "๐Ÿ—บ๏ธ Validate every unscanned namespace is reviewed", + "๐Ÿ” Validate the shared-publish-workflow pin guard", + "๐Ÿฉบ Validate declarative Coroot application health"] | all(.[]; . as $name | ([$job.steps[] | select(.name == $name and .if == "needs.changes.outputs.k8s == '\''true'\''" and @@ -198,4 +202,4 @@ for event in pull_request merge_group; do done done done -echo 'PASS: all three manifest regressions run once and selected non-success outcomes are refused' +echo 'PASS: all seven manifest regressions run once and selected non-success outcomes are refused' From 9480cf0d03c91dc2e5c5687e06d8e1e8c0fc249a Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 10:49:34 +0100 Subject: [PATCH 4/8] fix(ci): track fixture worker test as executable --- scripts/tests/test-fixture-workers.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/tests/test-fixture-workers.sh diff --git a/scripts/tests/test-fixture-workers.sh b/scripts/tests/test-fixture-workers.sh old mode 100644 new mode 100755 From f684eb06b226f03d605fe85194dc0295d93519ad Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 11:03:51 +0100 Subject: [PATCH 5/8] test(ci): verify failed peers and unexpected fixture receipts --- docs/adr/ci-regression-partitions.md | 11 ++++++++++- scripts/tests/test-fixture-workers.sh | 17 +++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/adr/ci-regression-partitions.md b/docs/adr/ci-regression-partitions.md index 112ad908c2..3fe8da7957 100644 --- a/docs/adr/ci-regression-partitions.md +++ b/docs/adr/ci-regression-partitions.md @@ -63,6 +63,13 @@ The services policy suite took 12m11s; the rollout regression alone consumed 11m10s. These observations motivate the bounded fixture workers and rebalancing; the complete feedback target remains unproven until another native run finishes. +The [bounded-worker trial](https://github.com/devantler-tech/platform/actions/runs/40300652085) +finished in 9m57s, but failed the executable-bit guard because the new worker +test was tracked as mode `100644`. Its services suite passed in 8m02s; both +manifest jobs passed in 9m18s. The mode is corrected to `100755`, and both +executable guards pass. This failed attempt is timing evidence, not successful +feedback acceptance. The follow-up must pass every native gate at its own head. + [GitHub's matrix documentation](https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/run-job-variations) defines independent job combinations, fail-fast and concurrency limits. The inference is that overlapping independent suites can shorten the critical path; @@ -86,7 +93,9 @@ The additional manifest-regression boundary exercises selection, success, failure, cancellation, missing results and unexpected skips. `scripts/tests/test-fixture-workers.sh` proves actual overlap, complete receipts, serial operation and rejection of failures, early errors, duplicate or unknown -cases, missing or corrupt receipts, interruption and invalid worker bounds. +cases, missing, corrupt or unassigned receipts, interruption and invalid worker +bounds. A failing peer must still be joined with its healthy sibling before +cleanup returns; serial mode also checks the observed concurrency bound. Before accepting a scheduling edit, compare its complete command inventory and step inputs with the reviewed baseline. A passing partition check establishes diff --git a/scripts/tests/test-fixture-workers.sh b/scripts/tests/test-fixture-workers.sh index b2dced408d..6a65d03918 100755 --- a/scripts/tests/test-fixture-workers.sh +++ b/scripts/tests/test-fixture-workers.sh @@ -11,7 +11,7 @@ fixture_pool_init "$FIXTURE" "${JOBS:-2}" cleanup_driver() { local result=$? trap - EXIT - fixture_pool_finish >/dev/null 2>&1 || true + fixture_pool_finish >/dev/null || true exit "$result" } trap cleanup_driver EXIT @@ -19,7 +19,7 @@ trap fixture_pool_interrupt HUP INT TERM work_case() { local name=$1 mkdir "$FIXTURE/active-$name" - [ "$(find "$FIXTURE" -maxdepth 1 -type d -name 'active-*' | wc -l | tr -d ' ')" -le 2 ] + [ "$(find "$FIXTURE" -maxdepth 1 -type d -name 'active-*' | wc -l | tr -d ' ')" -le "$FIXTURE_POOL_JOBS" ] [ "$name" != failed ] || return 17 if [ "$name" = early-error ]; then false; printf 'unsafe\n' >"$FIXTURE/after-error"; fi if [ "$MODE" = good ] && [ "$FIXTURE_POOL_JOBS" = 2 ]; then @@ -42,11 +42,16 @@ good) for name in alpha beta gamma delta; do fixture_pool_submit work_case "$name"; done ;; failed | early-error) fixture_pool_submit work_case "$MODE" ;; -missing | corrupt) +peer-failure) + fixture_pool_submit work_case failed + fixture_pool_submit work_case alpha + ;; +missing | corrupt | extra) fixture_pool_submit work_case alpha fixture_pool_drain if [ "$MODE" = missing ]; then rm "$FIXTURE/fixture-workers/receipts/alpha"; - else printf 'other\n' >"$FIXTURE/fixture-workers/receipts/alpha"; fi + elif [ "$MODE" = corrupt ]; then printf 'other\n' >"$FIXTURE/fixture-workers/receipts/alpha"; + else printf 'unassigned\n' >"$FIXTURE/fixture-workers/receipts/unassigned"; fi ;; duplicate) fixture_pool_submit work_case alpha @@ -66,7 +71,7 @@ fixture_pool_finish DRIVER chmod +x "$work/driver" library="$root/scripts/tests/lib/fixture-workers.lib.sh" -for mode in good failed early-error missing corrupt duplicate unregistered interrupted signalled; do +for mode in good failed early-error peer-failure missing corrupt extra duplicate unregistered interrupted signalled; do mkdir "$work/$mode" result=0 POOL_LIBRARY="$library" FIXTURE="$work/$mode" MODE="$mode" \ @@ -96,7 +101,7 @@ for mode in good failed early-error missing corrupt duplicate unregistered inter fi fi case "$mode" in - interrupted | signalled) [ "$(cat "$work/$mode/result-alpha")" = alpha ] ;; + interrupted | signalled | peer-failure) [ "$(cat "$work/$mode/result-alpha")" = alpha ] ;; esac [ ! -e "$work/$mode/after-error" ] || { echo 'FAIL: worker ignored early failure' >&2 From 335352c25eac4d377c67a931746c8bd70a38eb74 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 11:59:33 +0100 Subject: [PATCH 6/8] test(ci): preserve policy failure enforcement and worker diagnostics --- scripts/tests/test-data-product-ui-rollout.sh | 2 +- scripts/tests/test-fixture-workers.sh | 36 +++++++ .../coverage_test.go | 6 +- .../tenant_bind_test.go | 99 +++++++++++++++++-- 4 files changed, 133 insertions(+), 10 deletions(-) diff --git a/scripts/tests/test-data-product-ui-rollout.sh b/scripts/tests/test-data-product-ui-rollout.sh index 57ffcfea94..b057a8a3dd 100755 --- a/scripts/tests/test-data-product-ui-rollout.sh +++ b/scripts/tests/test-data-product-ui-rollout.sh @@ -10,7 +10,7 @@ fixture_pool_init "$scratch" 2 cleanup() { local result=$? workers_result=0 trap - EXIT - fixture_pool_finish >/dev/null || workers_result=$? + fixture_pool_finish >&2 || workers_result=$? rm -rf "${scratch}" if [[ "$result" == 0 ]]; then result=$workers_result; fi exit "$result" diff --git a/scripts/tests/test-fixture-workers.sh b/scripts/tests/test-fixture-workers.sh index 6a65d03918..1939323544 100755 --- a/scripts/tests/test-fixture-workers.sh +++ b/scripts/tests/test-fixture-workers.sh @@ -121,4 +121,40 @@ for jobs in 0 3 unknown; do fi grep -q 'workers must be 1 or 2' "$work/jobs-$jobs.log" done +# Exercise the real caller cleanup without running the long rollout suite. +# A parent failure must retain its status and pending-worker diagnostics. +sed -n '/^cleanup() {$/,/^}$/p' "$root/scripts/tests/test-data-product-ui-rollout.sh" >"$work/cleanup-body" +[ "$(grep -c '^cleanup() {' "$work/cleanup-body")" = 1 ] +cat >"$work/cleanup-driver" <<'CLEANUP' +#!/usr/bin/env bash +set -euo pipefail +fixture_pool_finish() { + [ -d "$scratch" ] || return 98 + printf 'pending-worker diagnostic\n' + return "$WORKER_RESULT" +} +CLEANUP +cat "$work/cleanup-body" >>"$work/cleanup-driver" +cat >>"$work/cleanup-driver" <<'CLEANUP' +trap cleanup EXIT +exit "$PARENT_RESULT" +CLEANUP +for parent_result in 0 11; do + for worker_result in 0 7; do + scratch="$work/cleanup-$parent_result-$worker_result" + mkdir "$scratch" + result=0 expected=$parent_result + if [ "$expected" = 0 ]; then expected=$worker_result; fi + scratch="$scratch" PARENT_RESULT="$parent_result" WORKER_RESULT="$worker_result" \ + "$BASH" "$work/cleanup-driver" >"$work/cleanup-out" 2>"$work/cleanup-err" || result=$? + [ "$result" = "$expected" ] && [ ! -e "$scratch" ] && [ ! -s "$work/cleanup-out" ] || { + echo 'FAIL: caller cleanup lost exit status or shared scratch ordering' >&2 + exit 1 + } + grep -qxF 'pending-worker diagnostic' "$work/cleanup-err" || { + echo 'FAIL: caller cleanup discarded pending-worker diagnostics' >&2 + exit 1 + } + done +done printf 'PASS: complete isolated workers; failures, early errors, missing/corrupt receipts, duplicate/unknown cases, interruption and invalid bounds refused\n' diff --git a/scripts/validate-eks-ci-role-policy/coverage_test.go b/scripts/validate-eks-ci-role-policy/coverage_test.go index 862481a6b8..ead6216822 100644 --- a/scripts/validate-eks-ci-role-policy/coverage_test.go +++ b/scripts/validate-eks-ci-role-policy/coverage_test.go @@ -59,9 +59,9 @@ type job struct { Permissions any `yaml:"permissions"` Defaults runDefaults `yaml:"defaults"` Strategy struct { - Matrix map[string]any `yaml:"matrix"` - FailFast any `yaml:"fail-fast"` - MaxParallel any `yaml:"max-parallel"` + Matrix any `yaml:"matrix"` + FailFast any `yaml:"fail-fast"` + MaxParallel any `yaml:"max-parallel"` } `yaml:"strategy"` } diff --git a/scripts/validate-eks-ci-role-policy/tenant_bind_test.go b/scripts/validate-eks-ci-role-policy/tenant_bind_test.go index 6ff3c56cc3..2e80b75788 100644 --- a/scripts/validate-eks-ci-role-policy/tenant_bind_test.go +++ b/scripts/validate-eks-ci-role-policy/tenant_bind_test.go @@ -26,9 +26,13 @@ func tenantPullSecretCIContract(wf workflow) error { strings.Join(strings.Fields(j.If), " ") != "github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true'" { return fmt.Errorf("lost selected policy job enforcement") } - suites, valid := j.Strategy.Matrix["suite"].([]any) + matrix, valid := j.Strategy.Matrix.(map[string]any) + if !valid { + return fmt.Errorf("lost literal policy matrix routing") + } + suites, valid := matrix["suite"].([]any) want := map[string]bool{"fixtures": true, "tenant": true, "services": true, "placement": true, "storage": true, "observability": true} - if !valid || len(j.Strategy.Matrix) != 1 || len(suites) != len(want) || + if !valid || len(matrix) != 1 || len(suites) != len(want) || j.Strategy.FailFast != false || j.Strategy.MaxParallel != 3 { return fmt.Errorf("lost complete policy matrix routing") } @@ -48,6 +52,9 @@ func tenantPullSecretCIContract(wf workflow) error { if shell == "" { shell = wf.Defaults.Run.Shell } + if s.Run != "" && (errorIsTolerated(s.ContinueOnError) || !shellKeepsErrexit(shell)) { + return fmt.Errorf("policy suite can hide a failed command") + } if s.Uses == "" && s.If == "matrix.suite == 'observability'" && !errorIsTolerated(s.ContinueOnError) && shellKeepsErrexit(shell) { count += canonicalCommandCount(s.Run, "bash scripts/tests/test-tenant-kro-pull-secret.sh") @@ -66,11 +73,15 @@ func TestTenantPullSecretAdmissionRemainsInCI(t *testing.T) { } for _, mutation := range []string{ "missing", "skip", "tolerate", "comment", "mask", "unsafe-shell", "job-skip", "job-tolerate", - "wrong-suite", "unconditional", "matrix-missing", "matrix-duplicate", "matrix-exclude", "matrix-unbounded", "matrix-cancel", + "wrong-suite", "unconditional", "matrix-missing", "matrix-duplicate", "matrix-exclude", "matrix-unbounded", "matrix-cancel", "matrix-expression", "matrix-array", "matrix-null", } { t.Run(mutation, func(t *testing.T) { w := canonicalCIWorkflow(t) j := w.Jobs["validate-cluster-policies"] + matrix, ok := j.Strategy.Matrix.(map[string]any) + if !ok { + t.Fatal("native policy matrix is not a literal mapping") + } found := 0 for i := range j.Steps { s := &j.Steps[i] @@ -100,15 +111,21 @@ func TestTenantPullSecretAdmissionRemainsInCI(t *testing.T) { case "unconditional": s.If = "" case "matrix-missing": - delete(j.Strategy.Matrix, "suite") + delete(matrix, "suite") case "matrix-duplicate": - j.Strategy.Matrix["suite"] = []any{"fixtures", "tenant", "services", "placement", "storage", "storage"} + matrix["suite"] = []any{"fixtures", "tenant", "services", "placement", "storage", "storage"} case "matrix-exclude": - j.Strategy.Matrix["exclude"] = []any{map[string]any{"suite": "observability"}} + matrix["exclude"] = []any{map[string]any{"suite": "observability"}} case "matrix-unbounded": j.Strategy.MaxParallel = 6 case "matrix-cancel": j.Strategy.FailFast = true + case "matrix-expression": + j.Strategy.Matrix = "${{ fromJSON(needs.prepare.outputs.matrix) }}" + case "matrix-array": + j.Strategy.Matrix = []any{"fixtures"} + case "matrix-null": + j.Strategy.Matrix = nil } } if found != 1 { @@ -122,6 +139,76 @@ func TestTenantPullSecretAdmissionRemainsInCI(t *testing.T) { } } +func TestPolicyMatrixRequiresSafeEffectiveShells(t *testing.T) { + for _, scope := range []string{"step", "job", "workflow"} { + t.Run(scope, func(t *testing.T) { + wf := canonicalCIWorkflow(t) + j := wf.Jobs["validate-cluster-policies"] + switch scope { + case "step": + found := false + for i := range j.Steps { + if j.Steps[i].Run != "" && j.Steps[i].If == "matrix.suite == 'fixtures'" { + j.Steps[i].Shell = "bash {0}" + found = true + } + } + if !found { + t.Fatal("no real fixture-suite step selected") + } + case "job": + j.Defaults.Run.Shell = "bash {0}" + case "workflow": + wf.Defaults.Run.Shell = "bash {0}" + } + wf.Jobs["validate-cluster-policies"] = j + if tenantPullSecretCIContract(wf) == nil { + t.Fatal("accepted a native suite that can hide an earlier failure") + } + }) + } + + // A narrower safe default or explicit step shell must override an unsafe + // inherited default. Check the same precedence the real runner resolves. + for _, scope := range []string{"job", "step"} { + t.Run("safe-override-"+scope, func(t *testing.T) { + wf := canonicalCIWorkflow(t) + wf.Defaults.Run.Shell = "bash {0}" + j := wf.Jobs["validate-cluster-policies"] + if scope == "job" { + j.Defaults.Run.Shell = "bash" + } else { + j.Defaults.Run.Shell = "bash {0}" + for i := range j.Steps { + if j.Steps[i].Run != "" { + j.Steps[i].Shell = "bash" + } + } + } + wf.Jobs["validate-cluster-policies"] = j + if err := tenantPullSecretCIContract(wf); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestWorkflowParserAcceptsUnrelatedExpressionMatrix(t *testing.T) { + const source = `on: pull_request +jobs: + unrelated: + strategy: + matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} + runs-on: ubuntu-latest + steps: + - run: echo fixture +` + var wf workflow + if err := yaml.Unmarshal([]byte(source), &wf); err != nil { + t.Fatalf("valid expression matrix disabled the shared workflow parser: %v", err) + } +} + func tenantBindContract(graphBody, roleBody []byte) error { var graph struct { Spec struct { From aa77624bac18dbeb395a459081d7f789afca0d34 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 12:07:13 +0100 Subject: [PATCH 7/8] test(ci): make cleanup assertions explicit for native shellcheck --- scripts/tests/test-fixture-workers.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test-fixture-workers.sh b/scripts/tests/test-fixture-workers.sh index 1939323544..d683bea6c4 100755 --- a/scripts/tests/test-fixture-workers.sh +++ b/scripts/tests/test-fixture-workers.sh @@ -147,10 +147,10 @@ for parent_result in 0 11; do if [ "$expected" = 0 ]; then expected=$worker_result; fi scratch="$scratch" PARENT_RESULT="$parent_result" WORKER_RESULT="$worker_result" \ "$BASH" "$work/cleanup-driver" >"$work/cleanup-out" 2>"$work/cleanup-err" || result=$? - [ "$result" = "$expected" ] && [ ! -e "$scratch" ] && [ ! -s "$work/cleanup-out" ] || { + if [[ "$result" != "$expected" || -e "$scratch" || -s "$work/cleanup-out" ]]; then echo 'FAIL: caller cleanup lost exit status or shared scratch ordering' >&2 exit 1 - } + fi grep -qxF 'pending-worker diagnostic' "$work/cleanup-err" || { echo 'FAIL: caller cleanup discarded pending-worker diagnostics' >&2 exit 1 From deaf2cf4f85aa6e0f7bd37f00ca2e7c68c237a0b Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 12:47:00 +0100 Subject: [PATCH 8/8] docs(ci): explain regression worker and routing contracts --- scripts/tests/lib/fixture-workers.lib.sh | 8 ++++++++ scripts/tests/test-data-product-ui-rollout.sh | 10 ++++++++++ scripts/tests/test-fixture-workers.sh | 3 +++ scripts/tests/test-policy-ci-job.sh | 15 +++++++++------ .../canonical_source_workflow_test.go | 8 ++++++++ .../tenant_bind_test.go | 4 ++++ 6 files changed, 42 insertions(+), 6 deletions(-) diff --git a/scripts/tests/lib/fixture-workers.lib.sh b/scripts/tests/lib/fixture-workers.lib.sh index e185dcd207..b6001c1faf 100644 --- a/scripts/tests/lib/fixture-workers.lib.sh +++ b/scripts/tests/lib/fixture-workers.lib.sh @@ -2,11 +2,13 @@ # Test-only workers. Callbacks own distinct case directories and read immutable # seed data. Drain before a foreground case or any shared-input mutation. +# fixture_pool_error reports a pool contract violation as an infrastructure error. fixture_pool_error() { printf 'fixture-workers: %s\n' "$*" >&2 return 2 } +# fixture_pool_init creates private tracking files for one or two isolated workers. fixture_pool_init() { case "$2" in 1 | 2) ;; *) fixture_pool_error 'workers must be 1 or 2' @@ -22,6 +24,7 @@ fixture_pool_init() { : >"$FIXTURE_POOL_ROOT/callbacks" } +# fixture_pool_register admits each declared callback once before cases are submitted. fixture_pool_register() { if ! [[ "$1" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || ! declare -F "$1" >/dev/null; then fixture_pool_error 'callback is not a declared function' @@ -34,8 +37,10 @@ fixture_pool_register() { printf '%s\n' "$1" >>"$FIXTURE_POOL_ROOT/callbacks" } +# fixture_pool_interrupt records cancellation so joining workers cannot report success. fixture_pool_interrupt() { FIXTURE_POOL_INTERRUPTED=1; } +# fixture_pool_drain joins the current batch and emits every case log before returning failure. fixture_pool_drain() { [ -n "${FIXTURE_POOL_ROOT:-}" ] || return 0 local pid name result @@ -58,6 +63,7 @@ fixture_pool_drain() { [ "$FIXTURE_POOL_FAILED" = 0 ] } +# fixture_pool_submit assigns a unique case to a registered callback and enforces the worker bound. fixture_pool_submit() { local callback=$1 name=$2 pid shift @@ -90,6 +96,8 @@ fixture_pool_submit() { if [ "$FIXTURE_POOL_PENDING" -ge "$FIXTURE_POOL_JOBS" ]; then fixture_pool_drain; fi } +# fixture_pool_finish joins workers and verifies an exact completion receipt for every assigned case. +# Repeated calls preserve the first complete verdict, including interruption and missing receipts. fixture_pool_finish() { [ -n "${FIXTURE_POOL_ROOT:-}" ] || return 0 if [ "$FIXTURE_POOL_FINISHED" = 1 ]; then return "$FIXTURE_POOL_VERDICT"; fi diff --git a/scripts/tests/test-data-product-ui-rollout.sh b/scripts/tests/test-data-product-ui-rollout.sh index b057a8a3dd..90f91b5c9f 100755 --- a/scripts/tests/test-data-product-ui-rollout.sh +++ b/scripts/tests/test-data-product-ui-rollout.sh @@ -7,6 +7,7 @@ readonly script="${root_dir}/scripts/verify-data-product-ui-rollout.sh" source "${root_dir}/scripts/tests/lib/fixture-workers.lib.sh" scratch="$(mktemp -d)" fixture_pool_init "$scratch" 2 +# cleanup joins pending workers before deleting shared scratch and preserves the original failure. cleanup() { local result=$? workers_result=0 trap - EXIT @@ -357,6 +358,7 @@ chmod +x "${scratch}/bin/kubectl" "${scratch}/bin/curl" expected_urls=14 # run_case exercises one fixture and checks its verdict, cleanup and sanitized output. +# run_case drains independent mutations before exercising a foreground verifier scenario. run_case() { # Foreground cases and shared-input changes never overlap the two workers. # The child deliberately reads its own exported flag; the parent stays unset. @@ -542,6 +544,7 @@ for mode in dns-failure tls-failure tls-handshake-failure request-timeout; do ru run_case transfer-timeout fail transfer-timeout 15 # mutate_case verifies that a dormant-profile mutation reaches snapshot rejection. +# mutate_case_impl copies the immutable healthy seed and checks one isolated rollout mutation. mutate_case_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" @@ -551,6 +554,7 @@ mutate_case_impl() { [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } fixture_pool_register mutate_case_impl +# mutate_case schedules a rollout mutation under the bounded worker and receipt contract. mutate_case() { fixture_pool_submit mutate_case_impl "$@"; } mutate_case ui-deployment-missing-contract deployment-ui-kit '.spec.template.spec.containers[0].env |= map(select(.name!="UI_CONTRACT_ENABLED"))' @@ -591,6 +595,7 @@ done run_case publisher-unplanned-route fail publisher-unplanned-route 15 # publisher_mutation starts from the separately declared enabled profile. +# publisher_mutation_impl checks one private copy of the publisher identity fixtures. publisher_mutation_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" @@ -600,6 +605,7 @@ publisher_mutation_impl() { [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } fixture_pool_register publisher_mutation_impl +# publisher_mutation schedules a publisher identity mutation without sharing writable fixtures. publisher_mutation() { fixture_pool_submit publisher_mutation_impl "$@"; } publisher_mutation declared-publisher-missing-deployment deployment-ui-kit '.spec.template.spec.containers[0].env |= map(select(.name!="PUBLISHER_PREFLIGHT_ENABLED"))' publisher_mutation declared-publisher-missing-pod pods '.items[4].spec.containers[0].env |= map(select(.name!="PUBLISHER_PREFLIGHT_ENABLED"))' @@ -698,6 +704,7 @@ jq '.metadata.name="ksail-releases" | .metadata.uid="release-product-uid" | expected_urls=17 run_case ksail pass # ksail_mutation requires a real snapshot-predicate rejection for each control. +# ksail_mutation_impl checks one private copy of the KSail release-product fixtures. ksail_mutation_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" @@ -707,6 +714,7 @@ ksail_mutation_impl() { [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } fixture_pool_register ksail_mutation_impl +# ksail_mutation schedules an independent release-product mutation and completion receipt. ksail_mutation() { fixture_pool_submit ksail_mutation_impl "$@"; } ksail_mutation ksail-disabled-live helm '.spec.values.demoProduct.ksailReleases.enabled=false' ksail_mutation ksail-stale-product release-product '.status.observedGeneration=2' @@ -737,6 +745,7 @@ expected_urls=14 jq -e '.retaken == 1 and .settled == ["apps.metadata.labels"]' "${scratch}/active-settled-label/stdout" >/dev/null || fail 'active-settled-label: the active report did not name the change that settled' # active_mutation verifies that an active-profile mutation reaches snapshot rejection. +# active_mutation_impl checks one private copy of the active declaration and workload fixtures. active_mutation_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" @@ -746,6 +755,7 @@ active_mutation_impl() { [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } fixture_pool_register active_mutation_impl +# active_mutation schedules an active-state mutation within the same strict worker bound. active_mutation() { fixture_pool_submit active_mutation_impl "$@"; } active_mutation active-stale-contract-condition product '.status.conditions[2].observedGeneration=2' active_mutation active-false-connector product '.status.conditions[1].status="False"' diff --git a/scripts/tests/test-fixture-workers.sh b/scripts/tests/test-fixture-workers.sh index d683bea6c4..2936ec81f3 100755 --- a/scripts/tests/test-fixture-workers.sh +++ b/scripts/tests/test-fixture-workers.sh @@ -8,6 +8,7 @@ cat >"$work/driver" <<'DRIVER' set -euo pipefail source "$POOL_LIBRARY" fixture_pool_init "$FIXTURE" "${JOBS:-2}" +# cleanup_driver joins the synthetic pool before its temporary test files can be removed. cleanup_driver() { local result=$? trap - EXIT @@ -16,6 +17,7 @@ cleanup_driver() { } trap cleanup_driver EXIT trap fixture_pool_interrupt HUP INT TERM +# work_case supplies controlled success, early failure, corruption and interruption scenarios. work_case() { local name=$1 mkdir "$FIXTURE/active-$name" @@ -128,6 +130,7 @@ sed -n '/^cleanup() {$/,/^}$/p' "$root/scripts/tests/test-data-product-ui-rollou cat >"$work/cleanup-driver" <<'CLEANUP' #!/usr/bin/env bash set -euo pipefail +# fixture_pool_finish models a pending worker diagnostic while requiring scratch to remain present. fixture_pool_finish() { [ -d "$scratch" ] || return 98 printf 'pending-worker diagnostic\n' diff --git a/scripts/tests/test-policy-ci-job.sh b/scripts/tests/test-policy-ci-job.sh index 1186be3406..6c6537ea2c 100755 --- a/scripts/tests/test-policy-ci-job.sh +++ b/scripts/tests/test-policy-ci-job.sh @@ -46,15 +46,16 @@ cat >"${work}/partition.jq" <<'JQ' "โš–๏ธ Read Kyverno CLI version", "โš–๏ธ Install Kyverno CLI"]) JQ +# partition_result distinguishes a rejected matrix from an unreadable routing observation. partition_result() { local input="$1" result=0 jq -e -f "${work}/partition.jq" "$input" >"${work}/partition-result" || result=$? case "$result" in - 0 | 1) return "$result" ;; - *) - echo 'FAIL: CI partition observation could not be evaluated' >&2 - return 2 - ;; + 0 | 1) return "$result" ;; + *) + echo 'FAIL: CI partition observation could not be evaluated' >&2 + return 2 + ;; esac } @@ -64,6 +65,7 @@ if ! partition_result "${work}/workflow.json"; then exit 1 fi +# reject_partition requires each broken routing to fail for a policy verdict, not a parser error. reject_partition() { local name="$1" mutation="$2" result=0 jq "$mutation" "${work}/workflow.json" >"${work}/mutated.json" @@ -131,6 +133,7 @@ jq -e '.[0].env == { POLICY_RESULT: "${{ needs.validate-cluster-policies.result }}" }' <<<"${step}" >/dev/null +# check exercises the actual aggregate decision with controlled upstream job outcomes. check() { local event="$1" selected="$2" result="$3" expected="$4" actual=0 EVENT_NAME="${event}" SELECTED="${selected}" POLICY_RESULT="${result}" \ @@ -190,7 +193,7 @@ for event in pull_request merge_group; do for result in success failure cancelled skipped pending unknown ''; do expected=1 actual=0 case "$event:$selected:$result" in - pull_request:true:success | pull_request:false:skipped | merge_group:*:skipped) expected=0 ;; + pull_request:true:success | pull_request:false:skipped | merge_group:*:skipped) expected=0 ;; esac EVENT_NAME="$event" SELECTED="$selected" REGRESSION_RESULT="$result" \ bash -e "${work}/regression-gate.sh" >"${work}/regression-output" 2>&1 || actual=$? diff --git a/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go b/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go index 992cc209d9..e823dc67ee 100644 --- a/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go +++ b/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go @@ -53,6 +53,7 @@ func TestCanonicalSourceCIRegistration(t *testing.T) { } } +// canonicalCIWorkflow loads the actual workflow so every mutation starts from shipping source. func canonicalCIWorkflow(t *testing.T) workflow { t.Helper() body, err := os.ReadFile("../../.github/workflows/ci.yaml") @@ -66,6 +67,7 @@ func canonicalCIWorkflow(t *testing.T) workflow { return wf } +// mutateCanonicalCommand changes exactly one registered command in its owning CI job. func mutateCanonicalCommand(t *testing.T, wf *workflow, command string, mutate func(*step)) { t.Helper() owner := canonicalCICommands[command] @@ -83,6 +85,7 @@ func mutateCanonicalCommand(t *testing.T, wf *workflow, command string, mutate f wf.Jobs[owner] = j } +// mutateCanonicalFilter applies a control to the real source-selection configuration. func mutateCanonicalFilter(t *testing.T, wf *workflow, mutate func(map[string][]string)) { t.Helper() j := wf.Jobs["changes"] @@ -109,6 +112,7 @@ func mutateCanonicalFilter(t *testing.T, wf *workflow, mutate func(map[string][] wf.Jobs["changes"] = j } +// TestCanonicalSourceCIRejectsLostOrNeutralizedCommands rejects absent or masked source regressions. func TestCanonicalSourceCIRejectsLostOrNeutralizedCommands(t *testing.T) { for command := range canonicalCICommands { for name, replacement := range map[string]string{ @@ -149,6 +153,7 @@ func TestCanonicalSourceCIRejectsLostOrNeutralizedCommands(t *testing.T) { } } +// TestCanonicalSourceCIRejectsLostSourceSelection protects every registered source trigger. func TestCanonicalSourceCIRejectsLostSourceSelection(t *testing.T) { for _, required := range canonicalCIPaths { for _, move := range []bool{false, true} { @@ -184,6 +189,7 @@ func TestCanonicalSourceCIRejectsLostSourceSelection(t *testing.T) { } } +// TestCanonicalSourceCIRejectsGateAndFilterBypasses checks scheduling, dependency and shell boundaries. func TestCanonicalSourceCIRejectsGateAndFilterBypasses(t *testing.T) { for name, mutate := range map[string]func(*workflow){ "missing guard job": func(w *workflow) { delete(w.Jobs, "validate-repository-guards") }, @@ -301,6 +307,7 @@ func TestCanonicalSourceCIRejectsGateAndFilterBypasses(t *testing.T) { } } +// TestCanonicalGeneratorCIRejectsLostRequiredJob requires generator success before deployment and CI. func TestCanonicalGeneratorCIRejectsLostRequiredJob(t *testing.T) { for _, mutation := range []string{"missing", "conditional", "tolerated", "unsafe-shell", "deploy-prod", "ci-required-checks"} { t.Run(mutation, func(t *testing.T) { @@ -351,6 +358,7 @@ func TestCanonicalGeneratorCIRejectsLostRequiredJob(t *testing.T) { } } +// canonicalSourceCIContract verifies source coverage and the required path to production authorization. func canonicalSourceCIContract(wf workflow) error { for _, name := range []string{"validate-repository-guards", "validate-publication-generator"} { guard, ok := wf.Jobs[name] diff --git a/scripts/validate-eks-ci-role-policy/tenant_bind_test.go b/scripts/validate-eks-ci-role-policy/tenant_bind_test.go index 2e80b75788..8f31627d11 100644 --- a/scripts/validate-eks-ci-role-policy/tenant_bind_test.go +++ b/scripts/validate-eks-ci-role-policy/tenant_bind_test.go @@ -19,6 +19,7 @@ type tenantBindRole struct { } `yaml:"rules"` } +// tenantPullSecretCIContract requires all fixed policy suites and one enforced pull-secret check. func tenantPullSecretCIContract(wf workflow) error { j, ok := wf.Jobs["validate-cluster-policies"] needs, valid := j.needsOf() @@ -66,6 +67,7 @@ func tenantPullSecretCIContract(wf workflow) error { return nil } +// TestTenantPullSecretAdmissionRemainsInCI rejects mutations that omit, misroute or mask admission. func TestTenantPullSecretAdmissionRemainsInCI(t *testing.T) { wf := canonicalCIWorkflow(t) if err := tenantPullSecretCIContract(wf); err != nil { @@ -139,6 +141,7 @@ func TestTenantPullSecretAdmissionRemainsInCI(t *testing.T) { } } +// TestPolicyMatrixRequiresSafeEffectiveShells checks failure propagation and shell-default precedence. func TestPolicyMatrixRequiresSafeEffectiveShells(t *testing.T) { for _, scope := range []string{"step", "job", "workflow"} { t.Run(scope, func(t *testing.T) { @@ -193,6 +196,7 @@ func TestPolicyMatrixRequiresSafeEffectiveShells(t *testing.T) { } } +// TestWorkflowParserAcceptsUnrelatedExpressionMatrix keeps valid unrelated workflows parseable. func TestWorkflowParserAcceptsUnrelatedExpressionMatrix(t *testing.T) { const source = `on: pull_request jobs: