diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8ef62e0a24..8081bea2fd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -209,6 +209,8 @@ jobs: - 'scripts/tests/test-data-product-controller-trial.sh' - 'scripts/tests/test-data-product-ui-host.sh' - 'scripts/tests/test-data-product-ui-rollout.sh' + - 'scripts/tests/test-fixture-workers.sh' + - 'scripts/tests/lib/fixture-workers.lib.sh' - 'scripts/tests/test-data-product-runtime-digests.sh' - 'scripts/verify-data-product-ui-rollout.sh' - 'scripts/tests/test-oauth2-proxy-sso-redirects.sh' @@ -1164,29 +1166,6 @@ jobs: shellcheck scripts/tests/test-guard-consumer-discovery-conservation.sh ./scripts/guard-consumer-discovery-conservation.sh - # The report names the two revisions per consumer; the generator (#3550) turns them - # into the committed approved set the matcher narrowing (#3308) is derived from. Its - # cluster read and its pin read are both stubbed here, so this proves the fail-closed - # shape โ€” an unresolved consumer writes nothing โ€” not any particular revision. - - name: ๐Ÿ” Validate the approved-revisions generator - run: | - # -x: the generator SOURCES the report, and without it SC1091 is reported and the run exits 1. - shellcheck -x scripts/generate-publish-workflow-approved-revisions.sh - shellcheck -x scripts/tests/test-publish-workflow-approved-revisions.sh - bash scripts/tests/test-publish-workflow-approved-revisions.sh - shellcheck -x scripts/bootstrap-tenant-source.sh scripts/tenant-publication-binding.lib.sh scripts/tests/test-tenant-publication-binding.sh scripts/tests/test-bootstrap-tenant-source.sh - bash scripts/tests/test-tenant-publication-binding.sh - bash scripts/tests/test-bootstrap-tenant-source.sh - shellcheck -x scripts/canonical-tenant-source.lib.sh scripts/tests/test-canonical-source-bootstrap.sh scripts/tests/test-canonical-bootstrap-output.sh - bash scripts/tests/test-canonical-source-bootstrap.sh - shellcheck -x scripts/tests/test-canonical-constructor-attribution.sh - bash scripts/tests/test-canonical-constructor-attribution.sh - shellcheck -x scripts/tests/test-canonical-generator-output.sh - bash scripts/tests/test-canonical-generator-output.sh - bash scripts/tests/test-canonical-bootstrap-output.sh - shellcheck -x scripts/tests/test-canonical-publisher-regeneration.sh - bash scripts/tests/test-canonical-publisher-regeneration.sh - # The pin guard above proves each matcher pins a FIXED revision; this one proves each # per-consumer matcher pins the revision PAIR the committed approved set names for that # consumer, and refuses any revision outside it (#3551, AC4 of #3308). Unconditional for @@ -1358,6 +1337,79 @@ jobs: bash scripts/tests/test-verify-wedding-backup-cutover.sh go test ./scripts/verify-wedding-dedicated-restore + validate-publication-generator: + name: ๐Ÿ” Validate Publication Generator + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: ๐Ÿ“‘ Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + + - name: โš™๏ธ Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: โš™๏ธ Initialize the approved kubectl for native regressions + timeout-minutes: 5 + shell: bash + env: + KUBECTL_VERSION: "v1.36.2" + KUBECTL_SHA256: "1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82" + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/kubectl" + curl -fsSL \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ + -o "$kubectl_path" + printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check + sudo install "$kubectl_path" /usr/local/bin/kubectl + echo /usr/local/bin >> "$GITHUB_PATH" + export PATH="/usr/local/bin:$PATH" + kubectl version --client + + - name: โŽˆ Install checksum-verified native Helm + shell: bash + env: + HELM_VERSION: v4.2.4 + HELM_SHA256: c306b46f719b0a4da32d0f78ee21bf90ce8d602f15b22ab753f0674d1670a7f3 + run: | + set -euo pipefail + native_dir="$(mktemp -d)" + archive="helm-${HELM_VERSION}-linux-amd64.tar.gz" + curl --fail --location --retry 3 --output "$native_dir/$archive" "https://get.helm.sh/$archive" + printf '%s %s\n' "$HELM_SHA256" "$native_dir/$archive" | sha256sum --check - + tar -xzf "$native_dir/$archive" -C "$native_dir" linux-amd64/helm + printf 'KYVERNO_LOG_HELM_BIN=%s\n' "$native_dir/linux-amd64/helm" >> "$GITHUB_ENV" + + # The report names the two revisions per consumer; the generator (#3550) turns them + # into the committed approved set the matcher narrowing (#3308) is derived from. Its + # cluster read and its pin read are both stubbed here, so this proves the fail-closed + # shape โ€” an unresolved consumer writes nothing โ€” not any particular revision. + - name: ๐Ÿ” Validate the approved-revisions generator + run: | + # -x: the generator SOURCES the report, and without it SC1091 is reported and the run exits 1. + shellcheck -x scripts/generate-publish-workflow-approved-revisions.sh + shellcheck -x scripts/tests/test-publish-workflow-approved-revisions.sh + bash scripts/tests/test-publish-workflow-approved-revisions.sh + shellcheck -x scripts/bootstrap-tenant-source.sh scripts/tenant-publication-binding.lib.sh scripts/tests/test-tenant-publication-binding.sh scripts/tests/test-bootstrap-tenant-source.sh + bash scripts/tests/test-tenant-publication-binding.sh + bash scripts/tests/test-bootstrap-tenant-source.sh + shellcheck -x scripts/canonical-tenant-source.lib.sh scripts/tests/test-canonical-source-bootstrap.sh scripts/tests/test-canonical-bootstrap-output.sh + bash scripts/tests/test-canonical-source-bootstrap.sh + shellcheck -x scripts/tests/test-canonical-constructor-attribution.sh + bash scripts/tests/test-canonical-constructor-attribution.sh + shellcheck -x scripts/tests/test-canonical-generator-output.sh + bash scripts/tests/test-canonical-generator-output.sh + bash scripts/tests/test-canonical-bootstrap-output.sh + shellcheck -x scripts/tests/test-canonical-publisher-regeneration.sh + bash scripts/tests/test-canonical-publisher-regeneration.sh + validate-consumer-discovery: name: ๐Ÿ” Validate Consumer Discovery Regressions runs-on: ubuntu-latest @@ -1691,10 +1743,15 @@ jobs: run: ksail --config ksail.prod.yaml workload validate validate-cluster-policies: - name: โš–๏ธ Validate Cluster Policies + name: โš–๏ธ Validate Cluster Policies (${{ matrix.suite }}) needs: [changes] if: github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true' runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 3 + matrix: + suite: [fixtures, tenant, services, placement, storage, observability] permissions: contents: read steps: @@ -1726,9 +1783,8 @@ jobs: with: release: ${{ steps.kyverno-version.outputs.release }} - - name: โš–๏ธ Validate cluster policies - # Evaluate the complete policy census and its negative controls offline. - # No cluster access or secrets are required. + - name: โš–๏ธ Validate cluster policies (fixtures) + if: matrix.suite == 'fixtures' run: | kyverno test ./tests shellcheck scripts/validate-kyverno-fixture-evaluation.sh scripts/tests/test-validate-kyverno-fixture-evaluation.sh @@ -1736,20 +1792,12 @@ jobs: bash scripts/validate-kyverno-fixture-evaluation.sh tests shellcheck scripts/tests/test-helm-release-mutation-scope.sh bash scripts/tests/test-helm-release-mutation-scope.sh + + - name: โš–๏ธ Validate cluster policies (tenant) + if: matrix.suite == 'tenant' + run: | shellcheck scripts/tests/test-tenant-webapp-boundary.sh bash scripts/tests/test-tenant-webapp-boundary.sh - shellcheck scripts/tests/test-webapp-container-security.sh - bash scripts/tests/test-webapp-container-security.sh - shellcheck scripts/tests/test-hcloud-volume-priority.sh - bash scripts/tests/test-hcloud-volume-priority.sh - shellcheck scripts/tests/test-tetragon-agent-priority.sh - bash scripts/tests/test-tetragon-agent-priority.sh - shellcheck scripts/tests/test-audit-privileged-rbac.sh - bash scripts/tests/test-audit-privileged-rbac.sh - shellcheck scripts/tests/test-auto-vpa-generating-policy-exclusion.sh - bash scripts/tests/test-auto-vpa-generating-policy-exclusion.sh - shellcheck scripts/tests/test-arc-controller-availability.sh - bash scripts/tests/test-arc-controller-availability.sh shellcheck scripts/tests/test-restrict-tenant-secret-stores.sh bash scripts/tests/test-restrict-tenant-secret-stores.sh shellcheck scripts/tests/test-tenant-controller-namespace-coverage.sh @@ -1763,12 +1811,12 @@ jobs: bash scripts/tests/test-add-default-deny-shape.sh shellcheck scripts/tests/test-restrict-homepage-service-groups.sh bash scripts/tests/test-restrict-homepage-service-groups.sh - shellcheck scripts/tests/test-kyverno-umami-mutation-rbac.sh - bash scripts/tests/test-kyverno-umami-mutation-rbac.sh - shellcheck scripts/tests/test-kyverno-github-team-report-rbac.sh - bash scripts/tests/test-kyverno-github-team-report-rbac.sh shellcheck scripts/tests/test-tenant-route-hostname-boundary.sh bash scripts/tests/test-tenant-route-hostname-boundary.sh + + - name: โš–๏ธ Validate cluster policies (services) + if: matrix.suite == 'services' + run: | shellcheck scripts/tests/test-actual-budget-auth-route.sh bash scripts/tests/test-actual-budget-auth-route.sh shellcheck scripts/tests/test-umami-sso-dashboard-route.sh @@ -1777,7 +1825,8 @@ jobs: bash scripts/tests/test-data-product-controller-trial.sh shellcheck scripts/tests/test-data-product-ui-host.sh bash scripts/tests/test-data-product-ui-host.sh - shellcheck scripts/verify-data-product-ui-rollout.sh scripts/tests/test-data-product-ui-rollout.sh + shellcheck -x scripts/verify-data-product-ui-rollout.sh scripts/tests/test-data-product-ui-rollout.sh scripts/tests/lib/fixture-workers.lib.sh scripts/tests/test-fixture-workers.sh + bash scripts/tests/test-fixture-workers.sh bash scripts/tests/test-data-product-ui-rollout.sh shellcheck scripts/tests/test-data-product-runtime-digests.sh bash scripts/tests/test-data-product-runtime-digests.sh @@ -1786,12 +1835,32 @@ jobs: shellcheck scripts/tests/test-umami-tracking-probe.sh bash scripts/tests/test-umami-tracking-probe.sh shellcheck scripts/verify-umami-tracking.sh + + - name: โš–๏ธ Validate cluster policies (placement) + if: matrix.suite == 'placement' + run: | + shellcheck scripts/tests/test-webapp-container-security.sh + bash scripts/tests/test-webapp-container-security.sh + shellcheck scripts/tests/test-hcloud-volume-priority.sh + bash scripts/tests/test-hcloud-volume-priority.sh + shellcheck scripts/tests/test-tetragon-agent-priority.sh + bash scripts/tests/test-tetragon-agent-priority.sh + shellcheck scripts/tests/test-auto-vpa-generating-policy-exclusion.sh + bash scripts/tests/test-auto-vpa-generating-policy-exclusion.sh + shellcheck scripts/tests/test-arc-controller-availability.sh + bash scripts/tests/test-arc-controller-availability.sh shellcheck scripts/tests/test-provider-drc-pod-seccomp.sh bash scripts/tests/test-provider-drc-pod-seccomp.sh - shellcheck scripts/tests/test-add-baseline-context.sh - bash scripts/tests/test-add-baseline-context.sh shellcheck scripts/tests/test-label-baseline-workers.sh bash scripts/tests/test-label-baseline-workers.sh + shellcheck scripts/tests/test-hcloud-baseline-worker-affinity.sh + bash scripts/tests/test-hcloud-baseline-worker-affinity.sh + + - name: โš–๏ธ Validate cluster policies (storage) + if: matrix.suite == 'storage' + run: | + shellcheck scripts/tests/test-add-baseline-context.sh + bash scripts/tests/test-add-baseline-context.sh shellcheck scripts/tests/test-longhorn-registrar-prestop-policy.sh bash scripts/tests/test-longhorn-registrar-prestop-policy.sh shellcheck scripts/tests/test-csi-sidecar-baseline-context.sh @@ -1800,11 +1869,19 @@ jobs: bash scripts/tests/test-longhorn-csi-plugin-baseline-context.sh shellcheck scripts/tests/test-longhorn-engine-image-baseline-context.sh bash scripts/tests/test-longhorn-engine-image-baseline-context.sh - shellcheck scripts/tests/test-hcloud-baseline-worker-affinity.sh - bash scripts/tests/test-hcloud-baseline-worker-affinity.sh shellcheck scripts/guard-longhorn-ui-baseline-context.sh scripts/tests/test-guard-longhorn-ui-baseline-context.sh scripts/tests/test-longhorn-ui-baseline-context.sh bash scripts/tests/test-guard-longhorn-ui-baseline-context.sh bash scripts/tests/test-longhorn-ui-baseline-context.sh + + - name: โš–๏ธ Validate cluster policies (observability) + if: matrix.suite == 'observability' + run: | + shellcheck scripts/tests/test-audit-privileged-rbac.sh + bash scripts/tests/test-audit-privileged-rbac.sh + shellcheck scripts/tests/test-kyverno-umami-mutation-rbac.sh + bash scripts/tests/test-kyverno-umami-mutation-rbac.sh + shellcheck scripts/tests/test-kyverno-github-team-report-rbac.sh + bash scripts/tests/test-kyverno-github-team-report-rbac.sh shellcheck scripts/tests/test-cilium-hubble-baseline-context.sh bash scripts/tests/test-cilium-hubble-baseline-context.sh shellcheck scripts/guard-coroot-baseline-context.sh scripts/tests/test-guard-coroot-baseline-context.sh @@ -1812,6 +1889,7 @@ jobs: shellcheck scripts/admit-coroot-baseline-context.sh scripts/tests/test-admit-coroot-baseline-context.sh bash scripts/tests/test-admit-coroot-baseline-context.sh - name: ๐Ÿ” Verify the controller-owned tenant pull secret + if: matrix.suite == 'observability' run: | shellcheck scripts/tests/test-tenant-kro-pull-secret.sh bash scripts/tests/test-tenant-kro-pull-secret.sh @@ -2063,13 +2141,6 @@ jobs: shellcheck scripts/tests/test-business-disclosure-reader.sh bash scripts/tests/test-business-disclosure-reader.sh - - name: ๐Ÿค Validate staged OpenBao standby repair - if: needs.changes.outputs.k8s == 'true' - run: | - shellcheck scripts/tests/test-openbao-standby-canary.sh - shellcheck scripts/tests/test-openbao-strategy-transition.sh - bash scripts/tests/test-openbao-standby-canary.sh - - name: ๐Ÿงฏ Validate daily and break-glass API boundaries if: needs.changes.outputs.k8s == 'true' # Daily Kubernetes/Talos access must stay read-only, while the root @@ -2492,14 +2563,6 @@ jobs: shellcheck scripts/tests/test-coroot-alert-autosuppressor.sh bash scripts/tests/test-coroot-alert-autosuppressor.sh - - name: ๐Ÿฉบ Validate declarative Coroot application health - if: needs.changes.outputs.k8s == 'true' - # Exact reviewed app signals receive per-app thresholds; any novel log - # pattern resets its threshold so the Applications view fails visible. - run: | - shellcheck scripts/tests/test-coroot-application-health-reconciler.sh - bash scripts/tests/test-coroot-application-health-reconciler.sh - - name: ๐Ÿ“ฆ Validate Coroot helper image caching if: needs.changes.outputs.k8s == 'true' # A pinned digest on a warm node must not depend on Docker Hub token @@ -2602,15 +2665,6 @@ jobs: # finding (#3545), plus the wiring that puts it before the promotion. run: bash scripts/tests/test-verify-matcher-accepts-digest.sh - - name: ๐Ÿ” Validate the shared-publish-workflow pin guard - if: needs.changes.outputs.k8s == 'true' - # The guard fails by PASSING: a widened cosign matcher still verifies, so - # nothing else in CI notices. This pins the shapes it must reject โ€” above - # all the SHA-or-tag alternation removed in #3022, which is the exact - # prior contents of all eight subjects and would otherwise revert in - # silence โ€” plus the floor and the wiring that reaches it. - run: bash scripts/tests/test-shared-publish-workflow-pin-guard.sh - - name: ๐Ÿงท Validate checkov skip reasons are quoted and intact if: needs.changes.outputs.k8s == 'true' # A `checkov.io/skipN` whose reason parses away still suppresses its @@ -2703,18 +2757,6 @@ jobs: bash scripts/guard-baseline-context-optin-scan-scope.sh bash scripts/tests/test-guard-baseline-context-optin-scan-scope.sh - - name: ๐Ÿ—บ๏ธ Validate every unscanned namespace is reviewed - if: needs.changes.outputs.k8s == 'true' - # A namespace in the operator's excludeNamespaces produces no scan result, and the - # posture score cannot tell that apart from a clean namespace. So the exclusions must - # match scripts/kubescape-unscanned-namespaces.tsv exactly, each row with a reason, - # and a namespace can only leave scan coverage through a reviewed change (#3215). - # The base HelmRelease is the only place the field may be set: an overlay patch or a - # valuesFrom that sets it would narrow the deployed scope while this read the base. - run: | - bash scripts/guard-kubescape-scan-scope.sh - bash scripts/tests/test-guard-kubescape-scan-scope.sh - - name: ๐Ÿงฎ Test the Kubescape result-coverage check if: needs.changes.outputs.k8s == 'true' # The check compares live Kubescape results against the objects that should have one @@ -2910,23 +2952,6 @@ jobs: restore-keys: | kubeconform-schema-cache-v1- - - name: ๐Ÿ”’ Validate Crossplane provider egress policy - if: needs.changes.outputs.k8s == 'true' - # This test exercises KSail's in-process Helm renderer, so it must run - # after the pinned KSail setup and schema-cache restore above. The - # reviewed FQDN set must remain narrow while provider traffic stays on - # the direct L3/L4 path that completes upstream TLS handshakes. - run: bash scripts/tests/test-crossplane-egress-policy.sh - - - name: ๐Ÿ” Validate Kyverno review admission boundaries - if: needs.changes.outputs.k8s == 'true' - # Render the pinned chart and exercise its match conditions against an - # isolated real API server. Ordinary policy requests must still fail - # closed; only the three already-filtered virtual reviews skip dispatch. - run: | - shellcheck scripts/tests/test-kyverno-auth-reviews.sh - bash scripts/tests/test-kyverno-auth-reviews.sh - - name: ๐Ÿง  Validate Kyverno admission VPA safety if: needs.changes.outputs.k8s == 'true' # Render the production VPA and the actual Kyverno chart Deployment so @@ -2969,13 +2994,6 @@ jobs: shellcheck scripts/tests/test-private-mail-server.sh bash scripts/tests/test-private-mail-server.sh - - name: โœ‰๏ธ Prove native mail accounts and recipient restrictions - if: needs.changes.outputs.k8s == 'true' - timeout-minutes: 10 - run: | - shellcheck scripts/tests/test-private-mail-configuration.sh - bash scripts/tests/test-private-mail-configuration.sh - - name: ๐Ÿ›ก๏ธ Validate Kubescape self-hosted scan persistence if: needs.changes.outputs.k8s == 'true' # The in-cluster scanner has no SaaS backend. Versions before v4.0.12 @@ -3279,6 +3297,112 @@ jobs: shellcheck scripts/summarize-sarif-findings.sh bash scripts/summarize-sarif-findings.sh "${RUNNER_TEMP}/kubescape.raw.sarif" + validate-manifest-regressions: + name: ๐Ÿงช Validate Independent Manifest Regressions + needs: + - changes + if: github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: ๐Ÿ“‘ Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: โš™๏ธ Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + - name: ๐Ÿ“Œ Install the reviewed YAML parser + if: needs.changes.outputs.k8s == 'true' + env: + GOBIN: ${{ runner.temp }}/platform-validation-tools + run: | + go install github.com/mikefarah/yq/v4@v4.54.1 + printf '%s\n' "$GOBIN" >> "$GITHUB_PATH" + PATH="$GOBIN:$PATH" yq --version + - name: โš–๏ธ Read Kyverno CLI version + if: needs.changes.outputs.k8s == 'true' + id: kyverno-version + run: bash .github/scripts/kyverno-version.sh + - name: โš–๏ธ Install Kyverno CLI + if: needs.changes.outputs.k8s == 'true' + uses: kyverno/action-install-cli@fcee92fca5c883169ef9927acf543e0b5fc58289 # v0.2.0 + with: + release: ${{ steps.kyverno-version.outputs.release }} + - name: โš™๏ธ Setup KSail + shell: bash + env: + # renovate: datasource=github-releases depName=devantler-tech/ksail extractVersion=^v(?.+)$ + KSAIL_VERSION: "7.203.26" + GITHUB_TOKEN: ${{ github.token }} + run: | + .github/scripts/setup-ksail.sh + - name: ๐Ÿ“ฅ Restore kubeconform schema cache + id: kubeconform-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/ksail/kubeconform + key: kubeconform-schema-cache-v1-render-regressions-${{ github.run_id }} + restore-keys: | + kubeconform-schema-cache-v1- + - name: ๐Ÿค Validate staged OpenBao standby repair + if: needs.changes.outputs.k8s == 'true' + run: | + shellcheck scripts/tests/test-openbao-standby-canary.sh + shellcheck scripts/tests/test-openbao-strategy-transition.sh + bash scripts/tests/test-openbao-standby-canary.sh + - name: ๐Ÿ”’ Validate Crossplane provider egress policy + if: needs.changes.outputs.k8s == 'true' + run: bash scripts/tests/test-crossplane-egress-policy.sh + - name: โœ‰๏ธ Prove native mail accounts and recipient restrictions + if: needs.changes.outputs.k8s == 'true' + timeout-minutes: 10 + run: | + shellcheck scripts/tests/test-private-mail-configuration.sh + bash scripts/tests/test-private-mail-configuration.sh + + - name: ๐Ÿ” Validate Kyverno review admission boundaries + if: needs.changes.outputs.k8s == 'true' + # Render the pinned chart and exercise its match conditions against an + # isolated real API server. Ordinary policy requests must still fail + # closed; only the three already-filtered virtual reviews skip dispatch. + run: | + shellcheck scripts/tests/test-kyverno-auth-reviews.sh + bash scripts/tests/test-kyverno-auth-reviews.sh + + - name: ๐Ÿ—บ๏ธ Validate every unscanned namespace is reviewed + if: needs.changes.outputs.k8s == 'true' + # A namespace in the operator's excludeNamespaces produces no scan result, and the + # posture score cannot tell that apart from a clean namespace. So the exclusions must + # match scripts/kubescape-unscanned-namespaces.tsv exactly, each row with a reason, + # and a namespace can only leave scan coverage through a reviewed change (#3215). + # The base HelmRelease is the only place the field may be set: an overlay patch or a + # valuesFrom that sets it would narrow the deployed scope while this read the base. + run: | + bash scripts/guard-kubescape-scan-scope.sh + bash scripts/tests/test-guard-kubescape-scan-scope.sh + + - name: ๐Ÿ” Validate the shared-publish-workflow pin guard + if: needs.changes.outputs.k8s == 'true' + # The guard fails by PASSING: a widened cosign matcher still verifies, so + # nothing else in CI notices. This pins the shapes it must reject โ€” above + # all the SHA-or-tag alternation removed in #3022, which is the exact + # prior contents of all eight subjects and would otherwise revert in + # silence โ€” plus the floor and the wiring that reaches it. + run: bash scripts/tests/test-shared-publish-workflow-pin-guard.sh + + - name: ๐Ÿฉบ Validate declarative Coroot application health + if: needs.changes.outputs.k8s == 'true' + # Exact reviewed app signals receive per-app thresholds; any novel log + # pattern resets its threshold so the Applications view fails visible. + run: | + shellcheck scripts/tests/test-coroot-application-health-reconciler.sh + bash scripts/tests/test-coroot-application-health-reconciler.sh + + validate-flux-bridge: name: ๐Ÿ” Validate Flux Credential Bridge needs: [changes] @@ -3883,6 +4007,7 @@ jobs: [ changes, validate-repository-guards, + validate-publication-generator, validate-go-regressions, validate-webapp-compiler, validate-consumer-discovery, @@ -4244,6 +4369,8 @@ jobs: [ changes, validate-repository-guards, + validate-publication-generator, + validate-manifest-regressions, validate-go-regressions, validate-webapp-compiler, validate-consumer-discovery, @@ -4343,6 +4470,8 @@ jobs: job-results: >- ${{ needs.changes.result }} ${{ needs.validate-repository-guards.result }} + ${{ needs.validate-publication-generator.result }} + ${{ needs.validate-manifest-regressions.result }} ${{ needs.validate-go-regressions.result }} ${{ needs.validate-webapp-compiler.result }} ${{ needs.validate-consumer-discovery.result }} @@ -4372,6 +4501,29 @@ jobs: exit 1 fi + - name: ๐Ÿ” Require publication generator validation + env: + GENERATOR_RESULT: ${{ needs.validate-publication-generator.result }} + run: | + if [ "$GENERATOR_RESULT" != success ]; then + echo "::error::Publication generator did not succeed ($GENERATOR_RESULT)" + exit 1 + fi + + - name: ๐Ÿงช Require selected manifest regressions + env: + EVENT_NAME: ${{ github.event_name }} + SELECTED: ${{ needs.changes.outputs.k8s }} + REGRESSION_RESULT: ${{ needs.validate-manifest-regressions.result }} + run: | + case "$EVENT_NAME:$SELECTED:$REGRESSION_RESULT" in + pull_request:true:success | pull_request:false:skipped | merge_group:*:skipped) ;; + *) + printf 'Manifest regressions did not meet their selection/result boundary: %s:%s:%s\n' "$EVENT_NAME" "$SELECTED" "$REGRESSION_RESULT" >&2 + exit 1 + ;; + esac + - name: ๐Ÿงช Require complete Go regressions env: REGRESSION_RESULT: ${{ needs.validate-go-regressions.result }} diff --git a/AGENTS.md b/AGENTS.md index 48b5fa884b..8154f855aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -128,7 +128,7 @@ Timing evidence alone does not establish Coroot request attribution or recovery. CI runs **static manifest validation** on PRs that touch k8s-related paths (`k8s/**`, `ksail*.yaml`, `.sops.yaml`, `talos*/**`, the naming configuration `.github/manifest-naming.yaml`, the validation scripts `scripts/validate-embedded-json/` / `scripts/validate-embedded-json.sh` / `scripts/generate-kubescape-exceptions/`, or `ci.yaml` โ€” the authoritative list is the `k8s` filter in `.github/workflows/ci.yaml`) โ€” the `validate` job in `.github/workflows/ci.yaml` first json-parses every registered embedded-JSON ConfigMap key via [`scripts/validate-embedded-json.sh`](scripts/validate-embedded-json.sh) (keys listed in the Go command's `registeredKeys` or ending in `.json` โ€” schema validation treats such blobs as opaque strings, so a stray comma would otherwise ship silently; run `bash scripts/validate-embedded-json.sh` locally when touching one, from any directory via the script's absolute path), then runs `ksail workload validate` for both the local and prod overlays plus a Kubescape scan (`scripts/generate-kubescape-exceptions` converts the `ClusterSecurityException` CRs into Kubescape's exceptions format, then `ksail workload scan --framework nsa,mitre --exceptions --compliance-threshold ` gates on the combined score โ€” the exact floor lives in `ci.yaml`). It is fast, needs no secrets (so it runs on fork PRs too), and starts no cluster. PRs touching `talos/**` or `talos-local/**` additionally run the `validate-talos` job: it renders the machine config with every patch applied (placeholder values stand in for env-expanded secrets like `${WG_SERVER_PRIVATE_KEY}`) and `talosctl validate`s the result, so a broken patch or an empty env expansion fails the PR event instead of the merge group's deploy (#2477). There is **no longer a full-cluster system test**: the local Docker cluster is a thin manual test-bed (see [Local Development Cluster](#local-development-cluster)), not a CI prod stand-in. -The complete cluster-policy fixtures and their offline regression controls run concurrently with manifest validation in the `validate-cluster-policies` pull-request job, under the same `k8s` path selection. The required aggregate waits for this job and rejects an unsuccessful or unexpectedly skipped selected family. Run `bash scripts/tests/test-policy-ci-job.sh` to exercise that result boundary; policy evaluation uses the shared Kyverno CLI version from `.github/scripts/kyverno-version.sh` and requires no cluster or secrets. +The complete cluster-policy fixtures and offline regressions run in fixed suites of the `validate-cluster-policies` PR matrix, under the existing `k8s` path selection. At most three suites run concurrently in isolated checkouts; every regression step selects exactly one declared suite. The publication-generator regressions run independently of the unconditional repository guards; required CI and production deployment both wait for the generator. Seven independent manifest regressions run in the read-only `validate-manifest-regressions` PR job with the same pinned tools and inputs, alongside manifest validation. The rollout fixture's independent mutation families use two isolated workers, retaining all cases, assertions and deadlines; foreground cases and shared-input changes wait for them. Required CI refuses an unsuccessful or unexpectedly skipped selected family. Run `bash scripts/tests/test-policy-ci-job.sh` for routing and result-boundary controls and `bash scripts/tests/test-fixture-workers.sh` for worker failure and completion controls; see the [partition decision](docs/adr/ci-regression-partitions.md). Policy evaluation uses the shared Kyverno CLI version from `.github/scripts/kyverno-version.sh` and needs no cluster or secrets. The Talos gate renders both roles, applies the reviewed KSail post-installer kernel-argument fold, then validates both. Production's extension path clears `extraKernelArgs` and pins the UKI command line; the local path without extensions is unchanged. The unconditional pin check and real offline positive/invalid-patch controls are documented in [`scripts/reconcile-talos-kernel-args/README.md`](scripts/reconcile-talos-kernel-args/README.md). The pin check compares the pinned KSail release's fold inputs by content with the audited sets, so a KSail bump needs a new source audit of this fold only when one of those inputs changed. diff --git a/docs/adr/ci-regression-partitions.md b/docs/adr/ci-regression-partitions.md new file mode 100644 index 0000000000..3fe8da7957 --- /dev/null +++ b/docs/adr/ci-regression-partitions.md @@ -0,0 +1,108 @@ +# Partition complete offline CI regressions + +## Decision + +Run the complete cluster-policy regressions as explicit matrix suites in +`ci.yaml`. Each suite receives an isolated checkout and the same pinned tools and +read-only token. At most three suites run concurrently. Fail-fast is disabled, so +a failed suite does not cancel the other diagnostics. + +Cluster policies have fixtures, tenant, services, placement, storage and +observability suites. Every validation step belongs to exactly one declared +suite. The publication-generator suite runs in its own required job with the same +checkout, compiler and native tool setup. The other repository guards retain their +unconditional steps, including the checks that independently protect CI wiring. +Regressions that share one shell environment remain in the same step. + +Move the independent OpenBao standby, Crossplane egress, native-mail, Kyverno +review-admission, unscanned-namespace, publisher-pin and Coroot application-health +regressions out of the serial manifest job into one required PR job. Preserve +their complete steps, timeouts and Kubernetes-path selection. Copy the pinned +compiler, YAML parser, Kyverno, KSail and schema-cache restore prerequisites into +the isolated checkout. This job has only read access to contents; SARIF upload +remains in the existing manifest job. A distinct candidate cache key restores +from the existing schema prefix; cached data never substitutes for evaluation. + +The existing selection predicates remain authoritative. The required aggregate +consumes the complete matrix result and refuses failed, cancelled, missing or +unexpectedly skipped selected validation. It explicitly requires generator +success. Production deployment waits for both the generator and its existing +merge-group prerequisites. +The independent manifest-regression family remains PR-only, like its original +execution. Required CI accepts success when selected and an intentional skip +otherwise, rejecting unknown selection and every selected non-success result. + +The rollout regression's four snapshot-mutation families use two test-only +workers. Each case owns its directory and reads an immutable seed. Foreground +cases drain the workers first; shared policy changes have an explicit barrier. +Every assigned case must return successfully and leave its exact completion +receipt. Interruption, failure, missing receipts and duplicate assignments fail +the suite, and cleanup joins the workers before removing shared scratch. The +production verifier and all fixture assertions and real deadlines stay intact. + +## Evidence and alternatives + +Successful run [40272630712](https://github.com/devantler-tech/platform/actions/runs/40272630712) +took 17m25s for repository guards and 21m06s for cluster policies. The generator +controls occupied 7m48s of the serial guard job. Manifest validation took 14m14s; +its unchanged duration and shared runner queues still bound overall feedback. + +The first [scheduling trial](https://github.com/devantler-tech/platform/actions/runs/40289350541) +reduced the guard job to 9m32s and completed its longest policy suite in 12m09s. +That attempt failed because existing validators assumed the old job locations; +it is not evidence of a successful complete feedback loop. Its manifest job took +17m22s. OpenBao standby, Crossplane egress and native mail accounted for 134s, +143s and 129s respectively, motivating the additional independent job. The +remaining validators now check the new owning jobs and their required +dependencies, retaining the existing lost-command and bypass controls. + +The next [complete green trial](https://github.com/devantler-tech/platform/actions/runs/40293296500) +took 17m12s including queue and setup. Manifest validation took 11m52s and its +parallel regression job 6m23s, leaving room to rebalance four unchanged checks. +The services policy suite took 12m11s; the rollout regression alone consumed +11m10s. These observations motivate the bounded fixture workers and rebalancing; +the complete feedback target remains unproven until another native run finishes. + +The [bounded-worker trial](https://github.com/devantler-tech/platform/actions/runs/40300652085) +finished in 9m57s, but failed the executable-bit guard because the new worker +test was tracked as mode `100644`. Its services suite passed in 8m02s; both +manifest jobs passed in 9m18s. The mode is corrected to `100755`, and both +executable guards pass. This failed attempt is timing evidence, not successful +feedback acceptance. The follow-up must pass every native gate at its own head. + +[GitHub's matrix documentation](https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/run-job-variations) +defines independent job combinations, fail-fast and concurrency limits. The +inference is that overlapping independent suites can shorten the critical path; +only a complete first-attempt workflow measurement establishes the actual result. +The measurement includes setup and runner queue time. + +Running all scripts concurrently inside one checkout would require proving that +their temporary files, environment changes and fixture mutations never interfere. +Separate hosted jobs provide that isolation. The narrowly scoped rollout workers +instead prove distinct case directories and immutable inputs, with barriers for +shared changes and serial foreground cases. Caching prior verdicts, omitting +tests and increasing timeouts do not meet the feedback or coverage requirement. + +## Verification and recovery + +`scripts/tests/test-policy-ci-job.sh` checks the authored partition and exercises +missing, duplicate and unknown suites, unselected and repeated regressions, +ignored failures and premature cancellation. The generator and existing policy +result controls exercise success and every selected non-success result. +The additional manifest-regression boundary exercises selection, success, +failure, cancellation, missing results and unexpected skips. +`scripts/tests/test-fixture-workers.sh` proves actual overlap, complete receipts, +serial operation and rejection of failures, early errors, duplicate or unknown +cases, missing, corrupt or unassigned receipts, interruption and invalid worker +bounds. A failing peer must still be joined with its healthy sibling before +cleanup returns; serial mode also checks the observed concurrency bound. + +Before accepting a scheduling edit, compare its complete command inventory and +step inputs with the reviewed baseline. A passing partition check establishes +complete routing of the authored steps; it does not prove that a deleted command +was an authorized removal. Review and native CI retain that responsibility. + +Observe every matrix suite and the required aggregate at the candidate and merged +source. Record actual durations on #5362; a green workflow does not by itself +prove the 15-minute target. Recovery restores serial scheduling through a reviewed +change while retaining the same validation commands and required-result gates. diff --git a/scripts/tests/lib/fixture-workers.lib.sh b/scripts/tests/lib/fixture-workers.lib.sh new file mode 100644 index 0000000000..b6001c1faf --- /dev/null +++ b/scripts/tests/lib/fixture-workers.lib.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Test-only workers. Callbacks own distinct case directories and read immutable +# seed data. Drain before a foreground case or any shared-input mutation. + +# fixture_pool_error reports a pool contract violation as an infrastructure error. +fixture_pool_error() { + printf 'fixture-workers: %s\n' "$*" >&2 + return 2 +} + +# fixture_pool_init creates private tracking files for one or two isolated workers. +fixture_pool_init() { + case "$2" in 1 | 2) ;; *) + fixture_pool_error 'workers must be 1 or 2' + return 2 + ;; + esac + FIXTURE_POOL_ROOT="$1/fixture-workers" + mkdir "$FIXTURE_POOL_ROOT" "$FIXTURE_POOL_ROOT/logs" "$FIXTURE_POOL_ROOT/receipts" || return 2 + FIXTURE_POOL_JOBS=$2 FIXTURE_POOL_PENDING=0 FIXTURE_POOL_FAILED=0 FIXTURE_POOL_INTERRUPTED=0 + FIXTURE_POOL_FINISHED=0 FIXTURE_POOL_VERDICT=0 + : >"$FIXTURE_POOL_ROOT/expected" + : >"$FIXTURE_POOL_ROOT/pending" + : >"$FIXTURE_POOL_ROOT/callbacks" +} + +# fixture_pool_register admits each declared callback once before cases are submitted. +fixture_pool_register() { + if ! [[ "$1" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || ! declare -F "$1" >/dev/null; then + fixture_pool_error 'callback is not a declared function' + return 2 + fi + if grep -qxF "$1" "$FIXTURE_POOL_ROOT/callbacks"; then + fixture_pool_error 'duplicate callback' + return 2 + fi + printf '%s\n' "$1" >>"$FIXTURE_POOL_ROOT/callbacks" +} + +# fixture_pool_interrupt records cancellation so joining workers cannot report success. +fixture_pool_interrupt() { FIXTURE_POOL_INTERRUPTED=1; } + +# fixture_pool_drain joins the current batch and emits every case log before returning failure. +fixture_pool_drain() { + [ -n "${FIXTURE_POOL_ROOT:-}" ] || return 0 + local pid name result + while read -r pid name; do + [ -n "$pid" ] || continue + while :; do + result=0 + wait "$pid" || result=$? + if [ "$result" = 0 ]; then break; fi + if [ "$result" = 127 ] || ! kill -0 "$pid" 2>/dev/null; then + FIXTURE_POOL_FAILED=1 + break + fi + done + cat "$FIXTURE_POOL_ROOT/logs/$name" || FIXTURE_POOL_FAILED=1 + done <"$FIXTURE_POOL_ROOT/pending" + : >"$FIXTURE_POOL_ROOT/pending" + FIXTURE_POOL_PENDING=0 + [ "$FIXTURE_POOL_INTERRUPTED" = 0 ] || return 2 + [ "$FIXTURE_POOL_FAILED" = 0 ] +} + +# fixture_pool_submit assigns a unique case to a registered callback and enforces the worker bound. +fixture_pool_submit() { + local callback=$1 name=$2 pid + shift + [ "$FIXTURE_POOL_FINISHED" = 0 ] && [ "$FIXTURE_POOL_INTERRUPTED" = 0 ] || return 2 + grep -qxF "$callback" "$FIXTURE_POOL_ROOT/callbacks" || { + fixture_pool_error 'unregistered callback' + return 2 + } + [[ "$name" =~ ^[a-zA-Z0-9_-]+$ ]] || { + fixture_pool_error 'invalid case identifier' + return 2 + } + if grep -qxF "$name" "$FIXTURE_POOL_ROOT/expected"; then + fixture_pool_error 'duplicate case assignment' + return 2 + fi + printf '%s\n' "$name" >>"$FIXTURE_POOL_ROOT/expected" + ( + # The parent owns shared scratch; a worker must never run its EXIT cleanup. + trap - EXIT HUP INT TERM + set -euo pipefail + export FIXTURE_POOL_WORKER=1 + "$callback" "$@" + printf '%s\n' "$name" >"$FIXTURE_POOL_ROOT/receipts/$name" + ) >"$FIXTURE_POOL_ROOT/logs/$name" 2>&1 & + pid=$! + printf '%s %s\n' "$pid" "$name" >>"$FIXTURE_POOL_ROOT/pending" + FIXTURE_POOL_PENDING=$((FIXTURE_POOL_PENDING + 1)) + [ "$FIXTURE_POOL_INTERRUPTED" = 0 ] || return 2 + if [ "$FIXTURE_POOL_PENDING" -ge "$FIXTURE_POOL_JOBS" ]; then fixture_pool_drain; fi +} + +# fixture_pool_finish joins workers and verifies an exact completion receipt for every assigned case. +# Repeated calls preserve the first complete verdict, including interruption and missing receipts. +fixture_pool_finish() { + [ -n "${FIXTURE_POOL_ROOT:-}" ] || return 0 + if [ "$FIXTURE_POOL_FINISHED" = 1 ]; then return "$FIXTURE_POOL_VERDICT"; fi + local result=0 name count receipts=0 file + fixture_pool_drain || result=$? + while IFS= read -r name; do + if ! printf '%s\n' "$name" | cmp -s - "$FIXTURE_POOL_ROOT/receipts/$name"; then result=1; fi + done <"$FIXTURE_POOL_ROOT/expected" + count="$(wc -l <"$FIXTURE_POOL_ROOT/expected" | tr -d ' ')" + for file in "$FIXTURE_POOL_ROOT/receipts/"*; do + [ -f "$file" ] || continue + receipts=$((receipts + 1)) + done + [ "$receipts" = "$count" ] || result=1 + if [ "$FIXTURE_POOL_INTERRUPTED" != 0 ]; then result=2; fi + FIXTURE_POOL_FINISHED=1 FIXTURE_POOL_VERDICT=$result + if [ "$result" != 0 ]; then + printf 'fixture-workers: incomplete or failed cases\n' >&2 + return "$result" + fi + printf 'FIXTURE-WORKERS COMPLETE cases=%s completed=%s workers=%s\n' "$count" "$receipts" "$FIXTURE_POOL_JOBS" +} diff --git a/scripts/tests/test-data-product-ui-rollout.sh b/scripts/tests/test-data-product-ui-rollout.sh index 8e27d23bf9..90f91b5c9f 100755 --- a/scripts/tests/test-data-product-ui-rollout.sh +++ b/scripts/tests/test-data-product-ui-rollout.sh @@ -3,8 +3,21 @@ set -euo pipefail umask 077 root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" readonly script="${root_dir}/scripts/verify-data-product-ui-rollout.sh" +# shellcheck source=scripts/tests/lib/fixture-workers.lib.sh +source "${root_dir}/scripts/tests/lib/fixture-workers.lib.sh" scratch="$(mktemp -d)" -trap 'rm -rf "${scratch}"' EXIT +fixture_pool_init "$scratch" 2 +# cleanup joins pending workers before deleting shared scratch and preserves the original failure. +cleanup() { + local result=$? workers_result=0 + trap - EXIT + fixture_pool_finish >&2 || workers_result=$? + rm -rf "${scratch}" + if [[ "$result" == 0 ]]; then result=$workers_result; fi + exit "$result" +} +trap cleanup EXIT +trap fixture_pool_interrupt HUP INT TERM # fail reports the violated fixture invariant and stops the regression suite. fail() { printf 'FAIL: %s\n' "$1" >&2 @@ -345,7 +358,12 @@ chmod +x "${scratch}/bin/kubectl" "${scratch}/bin/curl" expected_urls=14 # run_case exercises one fixture and checks its verdict, cleanup and sanitized output. +# run_case drains independent mutations before exercising a foreground verifier scenario. run_case() { + # Foreground cases and shared-input changes never overlap the two workers. + # The child deliberately reads its own exported flag; the parent stays unset. + # shellcheck disable=SC2031 + if [[ ${FIXTURE_POOL_WORKER:-0} != 1 ]]; then fixture_pool_drain; fi local name=$1 expected=$2 mode=${3:-} result=0 timeout_seconds=${4:-15} # Healthy fixtures include the expanded sequential public probes and full # snapshots. Keep rejection/cleanup deadline controls separately bounded. @@ -526,7 +544,8 @@ for mode in dns-failure tls-failure tls-handshake-failure request-timeout; do ru run_case transfer-timeout fail transfer-timeout 15 # mutate_case verifies that a dormant-profile mutation reaches snapshot rejection. -mutate_case() { +# mutate_case_impl copies the immutable healthy seed and checks one isolated rollout mutation. +mutate_case_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" cp "${scratch}/healthy/"*.json "${scratch}/$name/" @@ -534,6 +553,9 @@ mutate_case() { run_case "$name" fail '' "${4:-3}" [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } +fixture_pool_register mutate_case_impl +# mutate_case schedules a rollout mutation under the bounded worker and receipt contract. +mutate_case() { fixture_pool_submit mutate_case_impl "$@"; } mutate_case ui-deployment-missing-contract deployment-ui-kit '.spec.template.spec.containers[0].env |= map(select(.name!="UI_CONTRACT_ENABLED"))' mutate_case ui-deployment-disabled-appearance deployment-ui-kit '.spec.template.spec.containers[0].env[1].value="false"' @@ -573,7 +595,8 @@ done run_case publisher-unplanned-route fail publisher-unplanned-route 15 # publisher_mutation starts from the separately declared enabled profile. -publisher_mutation() { +# publisher_mutation_impl checks one private copy of the publisher identity fixtures. +publisher_mutation_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" cp "${scratch}/declared-publisher-true/"*.json "${scratch}/$name/" @@ -581,6 +604,9 @@ publisher_mutation() { run_case "$name" fail '' 3 [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } +fixture_pool_register publisher_mutation_impl +# publisher_mutation schedules a publisher identity mutation without sharing writable fixtures. +publisher_mutation() { fixture_pool_submit publisher_mutation_impl "$@"; } publisher_mutation declared-publisher-missing-deployment deployment-ui-kit '.spec.template.spec.containers[0].env |= map(select(.name!="PUBLISHER_PREFLIGHT_ENABLED"))' publisher_mutation declared-publisher-missing-pod pods '.items[4].spec.containers[0].env |= map(select(.name!="PUBLISHER_PREFLIGHT_ENABLED"))' publisher_mutation declared-publisher-missing-replicaset replicasets '.items |= map(if .spec.template.spec.containers[0].name=="ui-kit" then .spec.template.spec.containers[0].env |= map(select(.name!="PUBLISHER_PREFLIGHT_ENABLED")) else . end)' @@ -678,7 +704,8 @@ jq '.metadata.name="ksail-releases" | .metadata.uid="release-product-uid" | expected_urls=17 run_case ksail pass # ksail_mutation requires a real snapshot-predicate rejection for each control. -ksail_mutation() { +# ksail_mutation_impl checks one private copy of the KSail release-product fixtures. +ksail_mutation_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" cp "${scratch}/ksail/"*.json "${scratch}/$name/" @@ -686,6 +713,9 @@ ksail_mutation() { run_case "$name" fail '' 3 [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } +fixture_pool_register ksail_mutation_impl +# ksail_mutation schedules an independent release-product mutation and completion receipt. +ksail_mutation() { fixture_pool_submit ksail_mutation_impl "$@"; } ksail_mutation ksail-disabled-live helm '.spec.values.demoProduct.ksailReleases.enabled=false' ksail_mutation ksail-stale-product release-product '.status.observedGeneration=2' ksail_mutation ksail-failed-product release-product '.status.conditions[0].status="False"' @@ -715,7 +745,8 @@ expected_urls=14 jq -e '.retaken == 1 and .settled == ["apps.metadata.labels"]' "${scratch}/active-settled-label/stdout" >/dev/null || fail 'active-settled-label: the active report did not name the change that settled' # active_mutation verifies that an active-profile mutation reaches snapshot rejection. -active_mutation() { +# active_mutation_impl checks one private copy of the active declaration and workload fixtures. +active_mutation_impl() { local name=$1 key=$2 mutation=$3 mkdir "${scratch}/$name" cp "${scratch}/active/"*.json "${scratch}/$name/" @@ -723,6 +754,9 @@ active_mutation() { run_case "$name" fail '' "${4:-3}" [[ -s "${scratch}/$name/rejections" ]] || fail "$name: rejection did not reach snapshot validation" } +fixture_pool_register active_mutation_impl +# active_mutation schedules an active-state mutation within the same strict worker bound. +active_mutation() { fixture_pool_submit active_mutation_impl "$@"; } active_mutation active-stale-contract-condition product '.status.conditions[2].observedGeneration=2' active_mutation active-false-connector product '.status.conditions[1].status="False"' active_mutation active-missing-contract-condition product '.status.conditions |= map(select(.type!="ContractsReady"))' @@ -982,6 +1016,7 @@ for version in v1 ''; do done mkdir "${scratch}/declared-policy-update" cp "${scratch}/healthy/"*.json "${scratch}/declared-policy-update/" +fixture_pool_drain cp "${scratch}/apps-verify.json" "${scratch}/saved-policy.json" jq '.matchOIDCIdentity[0].subject="new-explicit-declared-publisher"' "${scratch}/saved-policy.json" >"${scratch}/apps-verify.json" jq --slurpfile policy "${scratch}/apps-verify.json" '.spec.verify=$policy[0]' "${scratch}/healthy/root.json" >"${scratch}/declared-policy-update/root.json" @@ -996,4 +1031,5 @@ run_case bounded-preflight fail preflight-timeout 1 # Two seconds of deliberate preflight plus a three-second startup/cleanup budget # reaches the hanging plugin; the child still sleeps far beyond the deadline. run_case bounded-credential-plugin fail hang 5 +fixture_pool_finish printf 'PASS: rollout readback behavior and bounded, read-only request scopes\n' diff --git a/scripts/tests/test-fixture-workers.sh b/scripts/tests/test-fixture-workers.sh new file mode 100755 index 0000000000..2936ec81f3 --- /dev/null +++ b/scripts/tests/test-fixture-workers.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +set -euo pipefail +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +cat >"$work/driver" <<'DRIVER' +#!/usr/bin/env bash +set -euo pipefail +source "$POOL_LIBRARY" +fixture_pool_init "$FIXTURE" "${JOBS:-2}" +# cleanup_driver joins the synthetic pool before its temporary test files can be removed. +cleanup_driver() { + local result=$? + trap - EXIT + fixture_pool_finish >/dev/null || true + exit "$result" +} +trap cleanup_driver EXIT +trap fixture_pool_interrupt HUP INT TERM +# work_case supplies controlled success, early failure, corruption and interruption scenarios. +work_case() { + local name=$1 + mkdir "$FIXTURE/active-$name" + [ "$(find "$FIXTURE" -maxdepth 1 -type d -name 'active-*' | wc -l | tr -d ' ')" -le "$FIXTURE_POOL_JOBS" ] + [ "$name" != failed ] || return 17 + if [ "$name" = early-error ]; then false; printf 'unsafe\n' >"$FIXTURE/after-error"; fi + if [ "$MODE" = good ] && [ "$FIXTURE_POOL_JOBS" = 2 ]; then + # The first worker must observe its peer before either completes. This + # proves overlap rather than relying on a timing-only speed assertion. + local started=$SECONDS + until [ "$(find "$FIXTURE" -maxdepth 1 -type d -name 'active-*' | wc -l | tr -d ' ')" = 2 ]; do + [ "$((SECONDS - started))" -lt 5 ] || return 19 + sleep 0.02 + done + fi + sleep 0.15 + [ -d "$FIXTURE" ] + printf '%s\n' "$name" >"$FIXTURE/result-$name" + rmdir "$FIXTURE/active-$name" +} +fixture_pool_register work_case +case "$MODE" in +good) + for name in alpha beta gamma delta; do fixture_pool_submit work_case "$name"; done + ;; +failed | early-error) fixture_pool_submit work_case "$MODE" ;; +peer-failure) + fixture_pool_submit work_case failed + fixture_pool_submit work_case alpha + ;; +missing | corrupt | extra) + fixture_pool_submit work_case alpha + fixture_pool_drain + if [ "$MODE" = missing ]; then rm "$FIXTURE/fixture-workers/receipts/alpha"; + elif [ "$MODE" = corrupt ]; then printf 'other\n' >"$FIXTURE/fixture-workers/receipts/alpha"; + else printf 'unassigned\n' >"$FIXTURE/fixture-workers/receipts/unassigned"; fi + ;; +duplicate) + fixture_pool_submit work_case alpha + fixture_pool_submit work_case alpha + ;; +unregistered) fixture_pool_submit unknown alpha ;; +interrupted) + fixture_pool_submit work_case alpha + fixture_pool_interrupt + ;; +signalled) + fixture_pool_submit work_case alpha + kill -TERM "$$" + ;; +esac +fixture_pool_finish +DRIVER +chmod +x "$work/driver" +library="$root/scripts/tests/lib/fixture-workers.lib.sh" +for mode in good failed early-error peer-failure missing corrupt extra duplicate unregistered interrupted signalled; do + mkdir "$work/$mode" + result=0 + POOL_LIBRARY="$library" FIXTURE="$work/$mode" MODE="$mode" \ + "$BASH" "$work/driver" >"$work/$mode.log" 2>&1 || result=$? + if [ "$mode" = good ]; then + if [ "$result" != 0 ] || ! grep -q 'FIXTURE-WORKERS COMPLETE cases=4 completed=4 workers=2' "$work/$mode.log"; then + cat "$work/$mode.log" >&2 + exit 1 + fi + for name in alpha beta gamma delta; do [ "$(cat "$work/$mode/result-$name")" = "$name" ]; done + elif [ "$result" = 0 ]; then + printf 'FAIL: %s was accepted\n' "$mode" >&2 + exit 1 + else + case "$mode" in + duplicate) reason='duplicate case assignment' ;; + unregistered) reason='unregistered callback' ;; + *) reason='incomplete or failed cases' ;; + esac + grep -q "$reason" "$work/$mode.log" || { + cat "$work/$mode.log" >&2 + exit 1 + } + if grep -q 'FIXTURE-WORKERS COMPLETE' "$work/$mode.log"; then + echo 'FAIL: unsuccessful work emitted a completion verdict' >&2 + exit 1 + fi + fi + case "$mode" in + interrupted | signalled | peer-failure) [ "$(cat "$work/$mode/result-alpha")" = alpha ] ;; + esac + [ ! -e "$work/$mode/after-error" ] || { + echo 'FAIL: worker ignored early failure' >&2 + exit 1 + } +done +mkdir "$work/serial" +POOL_LIBRARY="$library" FIXTURE="$work/serial" MODE=good JOBS=1 \ + "$BASH" "$work/driver" >"$work/serial.log" 2>&1 +grep -q 'FIXTURE-WORKERS COMPLETE cases=4 completed=4 workers=1' "$work/serial.log" +for jobs in 0 3 unknown; do + mkdir "$work/jobs-$jobs" + if POOL_LIBRARY="$library" FIXTURE="$work/jobs-$jobs" MODE=good JOBS="$jobs" \ + "$BASH" "$work/driver" >"$work/jobs-$jobs.log" 2>&1; then + echo 'FAIL: invalid worker bound accepted' >&2 + exit 1 + fi + grep -q 'workers must be 1 or 2' "$work/jobs-$jobs.log" +done +# Exercise the real caller cleanup without running the long rollout suite. +# A parent failure must retain its status and pending-worker diagnostics. +sed -n '/^cleanup() {$/,/^}$/p' "$root/scripts/tests/test-data-product-ui-rollout.sh" >"$work/cleanup-body" +[ "$(grep -c '^cleanup() {' "$work/cleanup-body")" = 1 ] +cat >"$work/cleanup-driver" <<'CLEANUP' +#!/usr/bin/env bash +set -euo pipefail +# fixture_pool_finish models a pending worker diagnostic while requiring scratch to remain present. +fixture_pool_finish() { + [ -d "$scratch" ] || return 98 + printf 'pending-worker diagnostic\n' + return "$WORKER_RESULT" +} +CLEANUP +cat "$work/cleanup-body" >>"$work/cleanup-driver" +cat >>"$work/cleanup-driver" <<'CLEANUP' +trap cleanup EXIT +exit "$PARENT_RESULT" +CLEANUP +for parent_result in 0 11; do + for worker_result in 0 7; do + scratch="$work/cleanup-$parent_result-$worker_result" + mkdir "$scratch" + result=0 expected=$parent_result + if [ "$expected" = 0 ]; then expected=$worker_result; fi + scratch="$scratch" PARENT_RESULT="$parent_result" WORKER_RESULT="$worker_result" \ + "$BASH" "$work/cleanup-driver" >"$work/cleanup-out" 2>"$work/cleanup-err" || result=$? + if [[ "$result" != "$expected" || -e "$scratch" || -s "$work/cleanup-out" ]]; then + echo 'FAIL: caller cleanup lost exit status or shared scratch ordering' >&2 + exit 1 + fi + grep -qxF 'pending-worker diagnostic' "$work/cleanup-err" || { + echo 'FAIL: caller cleanup discarded pending-worker diagnostics' >&2 + exit 1 + } + done +done +printf 'PASS: complete isolated workers; failures, early errors, missing/corrupt receipts, duplicate/unknown cases, interruption and invalid bounds refused\n' diff --git a/scripts/tests/test-policy-ci-job.sh b/scripts/tests/test-policy-ci-job.sh index 09dc8fa37e..6c6537ea2c 100755 --- a/scripts/tests/test-policy-ci-job.sh +++ b/scripts/tests/test-policy-ci-job.sh @@ -16,6 +16,109 @@ if ! jq -e --arg selection "github.event_name == 'pull_request' && needs.changes echo 'FAIL: independent policy job scheduling changed' >&2 exit 1 fi + +# Exercise the authored matrix routing, not a second list of validation commands. +# Every non-setup step must run in exactly one declared suite. An omitted or +# duplicate suite, a misspelled condition, or an unconditional regression would +# otherwise lose coverage or run the same tests repeatedly. +cat >"${work}/partition.jq" <<'JQ' + def partition($job; $groups; $setup): + ($job.strategy.matrix.suite // []) as $suites | + [$job.steps[] | select(.name as $name | ($setup | index($name)) == null)] as $checks | + ($job["runs-on"] == "ubuntu-latest") and + ($job.permissions == {contents: "read"}) and + (($job["continue-on-error"] // false) == false) and + ($job.strategy["fail-fast"] == false) and + ($job.strategy["max-parallel"] == 3) and + (($job.strategy.matrix // {} | keys) == ["suite"]) and + (($suites | sort) == ($groups | sort)) and + (($suites | unique | length) == ($suites | length)) and + all($setup[]; . as $name | + ([$job.steps[] | select(.name == $name and .if == null)] | length) == 1) and + all($job.steps[]; ((.["continue-on-error"] // false) == false)) and + all($checks[]; . as $step | + ([$suites[] | select($step.if == ("matrix.suite == \u0027" + . + "\u0027"))] | length) == 1) and + all($suites[]; . as $suite | + ([$checks[] | select(.if == ("matrix.suite == \u0027" + $suite + "\u0027"))] | length) > 0); + partition(.jobs["validate-cluster-policies"]; + ["fixtures", "tenant", "services", "placement", "storage", "observability"]; + ["๐Ÿ“‘ Checkout", "โš™๏ธ Setup Go", "๐Ÿ“Œ Install the reviewed YAML parser", + "โš–๏ธ Read Kyverno CLI version", "โš–๏ธ Install Kyverno CLI"]) +JQ + +# partition_result distinguishes a rejected matrix from an unreadable routing observation. +partition_result() { + local input="$1" result=0 + jq -e -f "${work}/partition.jq" "$input" >"${work}/partition-result" || result=$? + case "$result" in + 0 | 1) return "$result" ;; + *) + echo 'FAIL: CI partition observation could not be evaluated' >&2 + return 2 + ;; + esac +} + +yq -o=json '.' "$workflow" >"${work}/workflow.json" +if ! partition_result "${work}/workflow.json"; then + echo 'FAIL: CI matrices must retain exactly one scheduled copy of every validation step' >&2 + exit 1 +fi + +# reject_partition requires each broken routing to fail for a policy verdict, not a parser error. +reject_partition() { + local name="$1" mutation="$2" result=0 + jq "$mutation" "${work}/workflow.json" >"${work}/mutated.json" + partition_result "${work}/mutated.json" || result=$? + if [ "$result" -ne 1 ]; then + printf 'FAIL: %s must reject incomplete routing, got %s\n' "$name" "$result" >&2 + exit 1 + fi +} + +job=validate-cluster-policies +reject_partition "missing suite" "del(.jobs[\"$job\"].strategy.matrix.suite[0])" +reject_partition "duplicate suite" \ + ".jobs[\"$job\"].strategy.matrix.suite += [.jobs[\"$job\"].strategy.matrix.suite[0]]" +reject_partition "unknown suite" ".jobs[\"$job\"].strategy.matrix.suite += [\"unknown\"]" +reject_partition "early cancellation" ".jobs[\"$job\"].strategy[\"fail-fast\"] = true" +reject_partition "ignored failure" ".jobs[\"$job\"][\"continue-on-error\"] = true" +reject_partition "unselected regression" \ + "(.jobs[\"$job\"].steps[] | select(.if != null) | .if) = \"matrix.suite == 'unknown'\"" +reject_partition "repeated regression" \ + "(.jobs[\"$job\"].steps[] | select(.if != null)) |= del(.if)" + +echo 'PASS: the policy matrix preserves one scheduled copy of every regression; 7 broken routings refused' + +jq -e ' + .jobs["validate-publication-generator"] as $job | + ($job.permissions == {contents: "read"}) and + ($job | has("if") | not) and + (($job["continue-on-error"] // false) == false) and + all($job.steps[]; .if == null and ((.["continue-on-error"] // false) == false)) and + ([$job.steps[] | select(.name == "๐Ÿ” Validate the approved-revisions generator" + and .if == null and (.run | type == "string" and length > 0))] | length) == 1 and + (.jobs["ci-required-checks"].needs | index("validate-publication-generator") != null) and + (.jobs["deploy-prod"].needs | index("validate-publication-generator") != null) +' "${work}/workflow.json" >/dev/null +generator_step="$(jq -c '[.steps[] | select(.name == "๐Ÿ” Require publication generator validation")]' <<<"${gate}")" +jq -e 'length == 1 and .[0].env == { + GENERATOR_RESULT: "${{ needs.validate-publication-generator.result }}" +} and (.[0] | has("if") | not) + and ((.[0]["continue-on-error"] // false) == false) + and (.[0].run | type == "string" and length > 0)' <<<"${generator_step}" >/dev/null +jq -r '.[0].run' <<<"${generator_step}" >"${work}/generator-gate.sh" +for result in success failure cancelled skipped pending unknown ''; do + actual=0 + GENERATOR_RESULT="$result" bash -e "${work}/generator-gate.sh" >"${work}/generator-output" 2>&1 || actual=$? + expected=1 + [ "$result" != success ] || expected=0 + if [ "$actual" -ne "$expected" ]; then + printf 'FAIL: generator result=%s expected=%s actual=%s\n' "$result" "$expected" "$actual" >&2 + exit 1 + fi +done +echo 'PASS: the generator is required before deployment and every non-success result is refused' if ! jq -e '.needs | index("validate-cluster-policies") != null' <<<"${gate}" >/dev/null; then echo 'FAIL: required aggregate must wait for the independent policy job' >&2 exit 1 @@ -30,6 +133,7 @@ jq -e '.[0].env == { POLICY_RESULT: "${{ needs.validate-cluster-policies.result }}" }' <<<"${step}" >/dev/null +# check exercises the actual aggregate decision with controlled upstream job outcomes. check() { local event="$1" selected="$2" result="$3" expected="$4" actual=0 EVENT_NAME="${event}" SELECTED="${selected}" POLICY_RESULT="${result}" \ @@ -53,3 +157,52 @@ check merge_group true skipped 0 check merge_group false skipped 0 echo 'PASS: selected policy validation must succeed; intentional non-selection is accepted' + +jq -e --arg selection "github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true'" ' + . as $workflow | + .jobs["validate-manifest-regressions"] as $job | + ($job.needs == ["changes"]) and ($job.if == $selection) and + ($job.permissions == {contents: "read"}) and + (($job["continue-on-error"] // false) == false) and + (["๐Ÿค Validate staged OpenBao standby repair", + "๐Ÿ”’ Validate Crossplane provider egress policy", + "โœ‰๏ธ Prove native mail accounts and recipient restrictions", + "๐Ÿ” Validate Kyverno review admission boundaries", + "๐Ÿ—บ๏ธ Validate every unscanned namespace is reviewed", + "๐Ÿ” Validate the shared-publish-workflow pin guard", + "๐Ÿฉบ Validate declarative Coroot application health"] | all(.[]; + . as $name | + ([$job.steps[] | select(.name == $name and + .if == "needs.changes.outputs.k8s == '\''true'\''" and + ((.["continue-on-error"] // false) == false) and + (.run | type == "string" and length > 0))] | length) == 1 and + ([$workflow.jobs.validate.steps[]? | select(.name == $name)] | length) == 0)) and + (.jobs["ci-required-checks"].needs | index("validate-manifest-regressions") != null) +' "${work}/workflow.json" >/dev/null +regression_step="$(jq -c '[.steps[] | select(.name == "๐Ÿงช Require selected manifest regressions")]' <<<"${gate}")" +jq -e 'length == 1 and .[0].env == { + EVENT_NAME: "${{ github.event_name }}", + SELECTED: "${{ needs.changes.outputs.k8s }}", + REGRESSION_RESULT: "${{ needs.validate-manifest-regressions.result }}" +} and (.[0] | has("if") | not) + and ((.[0]["continue-on-error"] // false) == false) + and (.[0].run | type == "string" and length > 0)' <<<"${regression_step}" >/dev/null +jq -r '.[0].run' <<<"${regression_step}" >"${work}/regression-gate.sh" +for event in pull_request merge_group; do + for selected in true false unknown ''; do + for result in success failure cancelled skipped pending unknown ''; do + expected=1 actual=0 + case "$event:$selected:$result" in + pull_request:true:success | pull_request:false:skipped | merge_group:*:skipped) expected=0 ;; + esac + EVENT_NAME="$event" SELECTED="$selected" REGRESSION_RESULT="$result" \ + bash -e "${work}/regression-gate.sh" >"${work}/regression-output" 2>&1 || actual=$? + if [ "$actual" -ne "$expected" ]; then + printf 'FAIL: manifest regressions event=%s selected=%s result=%s expected=%s actual=%s\n' \ + "$event" "$selected" "$result" "$expected" "$actual" >&2 + exit 1 + fi + done + done +done +echo 'PASS: all seven manifest regressions run once and selected non-success outcomes are refused' diff --git a/scripts/validate-dr-signing/main_test.go b/scripts/validate-dr-signing/main_test.go index 2d9625b35b..76c2b693f6 100644 --- a/scripts/validate-dr-signing/main_test.go +++ b/scripts/validate-dr-signing/main_test.go @@ -1600,6 +1600,7 @@ func TestMergeQueueContractGateIsEnforced(t *testing.T) { [ changes, validate-repository-guards, + validate-publication-generator, validate-go-regressions, validate-webapp-compiler, validate-consumer-discovery, @@ -1617,6 +1618,7 @@ func TestMergeQueueContractGateIsEnforced(t *testing.T) { [ changes, validate-repository-guards, + validate-publication-generator, validate-go-regressions, validate-webapp-compiler, validate-consumer-discovery, diff --git a/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go b/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go index 4bafb1a9c0..e823dc67ee 100644 --- a/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go +++ b/scripts/validate-eks-ci-role-policy/canonical_source_workflow_test.go @@ -3,20 +3,21 @@ package main import ( "fmt" "os" + "slices" "strings" "testing" "gopkg.in/yaml.v3" ) -var canonicalCICommands = []string{ - "bash scripts/tests/test-canonical-source-bootstrap.sh", - "bash scripts/tests/test-canonical-constructor-attribution.sh", - "bash scripts/tests/test-canonical-generator-output.sh", - "bash scripts/tests/test-canonical-bootstrap-output.sh", - "bash scripts/tests/test-canonical-publisher-regeneration.sh", - "bash scripts/tests/test-tenant-source-discovery.sh", - "./scripts/guard-consumer-discovery-conservation.sh", +var canonicalCICommands = map[string]string{ + "bash scripts/tests/test-canonical-source-bootstrap.sh": "validate-publication-generator", + "bash scripts/tests/test-canonical-constructor-attribution.sh": "validate-publication-generator", + "bash scripts/tests/test-canonical-generator-output.sh": "validate-publication-generator", + "bash scripts/tests/test-canonical-bootstrap-output.sh": "validate-publication-generator", + "bash scripts/tests/test-canonical-publisher-regeneration.sh": "validate-publication-generator", + "bash scripts/tests/test-tenant-source-discovery.sh": "validate-repository-guards", + "./scripts/guard-consumer-discovery-conservation.sh": "validate-repository-guards", } var canonicalCIPaths = []string{ @@ -52,6 +53,7 @@ func TestCanonicalSourceCIRegistration(t *testing.T) { } } +// canonicalCIWorkflow loads the actual workflow so every mutation starts from shipping source. func canonicalCIWorkflow(t *testing.T) workflow { t.Helper() body, err := os.ReadFile("../../.github/workflows/ci.yaml") @@ -65,9 +67,11 @@ func canonicalCIWorkflow(t *testing.T) workflow { return wf } +// mutateCanonicalCommand changes exactly one registered command in its owning CI job. func mutateCanonicalCommand(t *testing.T, wf *workflow, command string, mutate func(*step)) { t.Helper() - j := wf.Jobs["validate-repository-guards"] + owner := canonicalCICommands[command] + j := wf.Jobs[owner] found := 0 for i := range j.Steps { if strings.Contains(j.Steps[i].Run, command) { @@ -78,9 +82,10 @@ func mutateCanonicalCommand(t *testing.T, wf *workflow, command string, mutate f if found != 1 { t.Fatalf("mutation requires exactly one native command block, got %d for %s", found, command) } - wf.Jobs["validate-repository-guards"] = j + wf.Jobs[owner] = j } +// mutateCanonicalFilter applies a control to the real source-selection configuration. func mutateCanonicalFilter(t *testing.T, wf *workflow, mutate func(map[string][]string)) { t.Helper() j := wf.Jobs["changes"] @@ -107,8 +112,9 @@ func mutateCanonicalFilter(t *testing.T, wf *workflow, mutate func(map[string][] wf.Jobs["changes"] = j } +// TestCanonicalSourceCIRejectsLostOrNeutralizedCommands rejects absent or masked source regressions. func TestCanonicalSourceCIRejectsLostOrNeutralizedCommands(t *testing.T) { - for _, command := range canonicalCICommands { + for command := range canonicalCICommands { for name, replacement := range map[string]string{ "removed": "", "comment": "# " + command, "echo": "echo " + command, "conditional": "if false; then " + command + "; fi", @@ -147,6 +153,7 @@ func TestCanonicalSourceCIRejectsLostOrNeutralizedCommands(t *testing.T) { } } +// TestCanonicalSourceCIRejectsLostSourceSelection protects every registered source trigger. func TestCanonicalSourceCIRejectsLostSourceSelection(t *testing.T) { for _, required := range canonicalCIPaths { for _, move := range []bool{false, true} { @@ -182,6 +189,7 @@ func TestCanonicalSourceCIRejectsLostSourceSelection(t *testing.T) { } } +// TestCanonicalSourceCIRejectsGateAndFilterBypasses checks scheduling, dependency and shell boundaries. func TestCanonicalSourceCIRejectsGateAndFilterBypasses(t *testing.T) { for name, mutate := range map[string]func(*workflow){ "missing guard job": func(w *workflow) { delete(w.Jobs, "validate-repository-guards") }, @@ -278,31 +286,94 @@ func TestCanonicalSourceCIRejectsGateAndFilterBypasses(t *testing.T) { for _, scope := range []string{"job", "step"} { wf = canonicalCIWorkflow(t) wf.Defaults.Run.Shell = "bash {0}" - j = wf.Jobs["validate-repository-guards"] - j.Defaults.Run.Shell = "bash" - if scope == "step" { - j.Defaults.Run.Shell = "bash {0}" - } - for i := range j.Steps { + for _, name := range []string{"validate-repository-guards", "validate-publication-generator"} { + j = wf.Jobs[name] + j.Defaults.Run.Shell = "bash" if scope == "step" { - j.Steps[i].Shell = "bash" - } else { - j.Steps[i].Shell = "" + j.Defaults.Run.Shell = "bash {0}" + } + for i := range j.Steps { + if scope == "step" { + j.Steps[i].Shell = "bash" + } else { + j.Steps[i].Shell = "" + } } + wf.Jobs[name] = j } - wf.Jobs["validate-repository-guards"] = j if err := canonicalSourceCIContract(wf); err != nil { t.Fatalf("rejected safe %s shell override: %v", scope, err) } } } +// TestCanonicalGeneratorCIRejectsLostRequiredJob requires generator success before deployment and CI. +func TestCanonicalGeneratorCIRejectsLostRequiredJob(t *testing.T) { + for _, mutation := range []string{"missing", "conditional", "tolerated", "unsafe-shell", "deploy-prod", "ci-required-checks"} { + t.Run(mutation, func(t *testing.T) { + wf := canonicalCIWorkflow(t) + if err := canonicalSourceCIContract(wf); err != nil { + t.Fatalf("positive control: %v", err) + } + name := "validate-publication-generator" + j := wf.Jobs[name] + want := "require unconditional " + name + switch mutation { + case "missing": + delete(wf.Jobs, name) + case "conditional": + j.If = "false" + wf.Jobs[name] = j + case "tolerated": + j.ContinueOnError = true + wf.Jobs[name] = j + case "unsafe-shell": + j.Defaults.Run.Shell = "bash {0}" + wf.Jobs[name] = j + want = "canonical source CI must enforce exactly one" + default: + consumer := wf.Jobs[mutation] + needs, valid := consumer.needsOf() + if !valid || !slices.Contains(needs, name) { + t.Fatal("mutation must remove an actual required dependency") + } + kept := make([]any, 0, len(needs)-1) + for _, dependency := range needs { + if dependency != name { + kept = append(kept, dependency) + } + } + consumer.Needs = kept + wf.Jobs[mutation] = consumer + want = mutation + " must require the publication generator" + } + err := canonicalSourceCIContract(wf) + if err == nil { + t.Fatal("accepted lost generator enforcement") + } + if !strings.Contains(err.Error(), want) { + t.Fatalf("wrong refusal: want %q in %v", want, err) + } + }) + } +} + +// canonicalSourceCIContract verifies source coverage and the required path to production authorization. func canonicalSourceCIContract(wf workflow) error { - guard, ok := wf.Jobs["validate-repository-guards"] - if !ok || guard.If != "" || errorIsTolerated(guard.ContinueOnError) { - return fmt.Errorf("canonical source contracts require the unconditional repository guard job") + for _, name := range []string{"validate-repository-guards", "validate-publication-generator"} { + guard, ok := wf.Jobs[name] + if !ok || guard.If != "" || errorIsTolerated(guard.ContinueOnError) { + return fmt.Errorf("canonical source contracts require unconditional %s", name) + } + } + for _, consumer := range []string{"deploy-prod", "ci-required-checks"} { + needs, valid := wf.Jobs[consumer].needsOf() + if !valid || !slices.Contains(needs, "validate-publication-generator") { + return fmt.Errorf("%s must require the publication generator", consumer) + } } - for _, command := range canonicalCICommands { + for command, owner := range canonicalCICommands { + guard := wf.Jobs[owner] count := 0 for _, s := range guard.Steps { shell := s.Shell diff --git a/scripts/validate-eks-ci-role-policy/coverage_test.go b/scripts/validate-eks-ci-role-policy/coverage_test.go index 7d652669ac..ead6216822 100644 --- a/scripts/validate-eks-ci-role-policy/coverage_test.go +++ b/scripts/validate-eks-ci-role-policy/coverage_test.go @@ -58,6 +58,11 @@ type job struct { ContinueOnError any `yaml:"continue-on-error"` Permissions any `yaml:"permissions"` Defaults runDefaults `yaml:"defaults"` + Strategy struct { + Matrix any `yaml:"matrix"` + FailFast any `yaml:"fail-fast"` + MaxParallel any `yaml:"max-parallel"` + } `yaml:"strategy"` } type runDefaults struct { diff --git a/scripts/validate-eks-ci-role-policy/tenant_bind_test.go b/scripts/validate-eks-ci-role-policy/tenant_bind_test.go index 0e73fac432..8f31627d11 100644 --- a/scripts/validate-eks-ci-role-policy/tenant_bind_test.go +++ b/scripts/validate-eks-ci-role-policy/tenant_bind_test.go @@ -19,6 +19,7 @@ type tenantBindRole struct { } `yaml:"rules"` } +// tenantPullSecretCIContract requires all fixed policy suites and one enforced pull-secret check. func tenantPullSecretCIContract(wf workflow) error { j, ok := wf.Jobs["validate-cluster-policies"] needs, valid := j.needsOf() @@ -26,6 +27,23 @@ func tenantPullSecretCIContract(wf workflow) error { strings.Join(strings.Fields(j.If), " ") != "github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true'" { return fmt.Errorf("lost selected policy job enforcement") } + matrix, valid := j.Strategy.Matrix.(map[string]any) + if !valid { + return fmt.Errorf("lost literal policy matrix routing") + } + suites, valid := matrix["suite"].([]any) + want := map[string]bool{"fixtures": true, "tenant": true, "services": true, "placement": true, "storage": true, "observability": true} + if !valid || len(matrix) != 1 || len(suites) != len(want) || + j.Strategy.FailFast != false || j.Strategy.MaxParallel != 3 { + return fmt.Errorf("lost complete policy matrix routing") + } + for _, item := range suites { + name, isString := item.(string) + if !isString || !want[name] { + return fmt.Errorf("unknown or duplicate policy suite") + } + delete(want, name) + } count := 0 for _, s := range j.Steps { shell := s.Shell @@ -35,7 +53,11 @@ func tenantPullSecretCIContract(wf workflow) error { if shell == "" { shell = wf.Defaults.Run.Shell } - if s.Uses == "" && s.enforces() && shellKeepsErrexit(shell) { + if s.Run != "" && (errorIsTolerated(s.ContinueOnError) || !shellKeepsErrexit(shell)) { + return fmt.Errorf("policy suite can hide a failed command") + } + if s.Uses == "" && s.If == "matrix.suite == 'observability'" && + !errorIsTolerated(s.ContinueOnError) && shellKeepsErrexit(shell) { count += canonicalCommandCount(s.Run, "bash scripts/tests/test-tenant-kro-pull-secret.sh") } } @@ -45,15 +67,23 @@ func tenantPullSecretCIContract(wf workflow) error { return nil } +// TestTenantPullSecretAdmissionRemainsInCI rejects mutations that omit, misroute or mask admission. func TestTenantPullSecretAdmissionRemainsInCI(t *testing.T) { wf := canonicalCIWorkflow(t) if err := tenantPullSecretCIContract(wf); err != nil { t.Fatal(err) } - for _, mutation := range []string{"missing", "skip", "tolerate", "comment", "mask", "unsafe-shell", "job-skip", "job-tolerate"} { + for _, mutation := range []string{ + "missing", "skip", "tolerate", "comment", "mask", "unsafe-shell", "job-skip", "job-tolerate", + "wrong-suite", "unconditional", "matrix-missing", "matrix-duplicate", "matrix-exclude", "matrix-unbounded", "matrix-cancel", "matrix-expression", "matrix-array", "matrix-null", + } { t.Run(mutation, func(t *testing.T) { w := canonicalCIWorkflow(t) j := w.Jobs["validate-cluster-policies"] + matrix, ok := j.Strategy.Matrix.(map[string]any) + if !ok { + t.Fatal("native policy matrix is not a literal mapping") + } found := 0 for i := range j.Steps { s := &j.Steps[i] @@ -78,6 +108,26 @@ func TestTenantPullSecretAdmissionRemainsInCI(t *testing.T) { j.If = "false" case "job-tolerate": j.ContinueOnError = true + case "wrong-suite": + s.If = "matrix.suite == 'unknown'" + case "unconditional": + s.If = "" + case "matrix-missing": + delete(matrix, "suite") + case "matrix-duplicate": + matrix["suite"] = []any{"fixtures", "tenant", "services", "placement", "storage", "storage"} + case "matrix-exclude": + matrix["exclude"] = []any{map[string]any{"suite": "observability"}} + case "matrix-unbounded": + j.Strategy.MaxParallel = 6 + case "matrix-cancel": + j.Strategy.FailFast = true + case "matrix-expression": + j.Strategy.Matrix = "${{ fromJSON(needs.prepare.outputs.matrix) }}" + case "matrix-array": + j.Strategy.Matrix = []any{"fixtures"} + case "matrix-null": + j.Strategy.Matrix = nil } } if found != 1 { @@ -91,6 +141,78 @@ func TestTenantPullSecretAdmissionRemainsInCI(t *testing.T) { } } +// TestPolicyMatrixRequiresSafeEffectiveShells checks failure propagation and shell-default precedence. +func TestPolicyMatrixRequiresSafeEffectiveShells(t *testing.T) { + for _, scope := range []string{"step", "job", "workflow"} { + t.Run(scope, func(t *testing.T) { + wf := canonicalCIWorkflow(t) + j := wf.Jobs["validate-cluster-policies"] + switch scope { + case "step": + found := false + for i := range j.Steps { + if j.Steps[i].Run != "" && j.Steps[i].If == "matrix.suite == 'fixtures'" { + j.Steps[i].Shell = "bash {0}" + found = true + } + } + if !found { + t.Fatal("no real fixture-suite step selected") + } + case "job": + j.Defaults.Run.Shell = "bash {0}" + case "workflow": + wf.Defaults.Run.Shell = "bash {0}" + } + wf.Jobs["validate-cluster-policies"] = j + if tenantPullSecretCIContract(wf) == nil { + t.Fatal("accepted a native suite that can hide an earlier failure") + } + }) + } + + // A narrower safe default or explicit step shell must override an unsafe + // inherited default. Check the same precedence the real runner resolves. + for _, scope := range []string{"job", "step"} { + t.Run("safe-override-"+scope, func(t *testing.T) { + wf := canonicalCIWorkflow(t) + wf.Defaults.Run.Shell = "bash {0}" + j := wf.Jobs["validate-cluster-policies"] + if scope == "job" { + j.Defaults.Run.Shell = "bash" + } else { + j.Defaults.Run.Shell = "bash {0}" + for i := range j.Steps { + if j.Steps[i].Run != "" { + j.Steps[i].Shell = "bash" + } + } + } + wf.Jobs["validate-cluster-policies"] = j + if err := tenantPullSecretCIContract(wf); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestWorkflowParserAcceptsUnrelatedExpressionMatrix keeps valid unrelated workflows parseable. +func TestWorkflowParserAcceptsUnrelatedExpressionMatrix(t *testing.T) { + const source = `on: pull_request +jobs: + unrelated: + strategy: + matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} + runs-on: ubuntu-latest + steps: + - run: echo fixture +` + var wf workflow + if err := yaml.Unmarshal([]byte(source), &wf); err != nil { + t.Fatalf("valid expression matrix disabled the shared workflow parser: %v", err) + } +} + func tenantBindContract(graphBody, roleBody []byte) error { var graph struct { Spec struct {