ci: parallelize and stabilize manifest validation

.github/workflows/ci.yaml

@@ -1690,6 +1690,46 @@ jobs:
           shellcheck scripts/admit-coroot-baseline-context.sh scripts/tests/test-admit-coroot-baseline-context.sh
           bash scripts/tests/test-admit-coroot-baseline-context.sh
 
+  validate-rgd-templates:
+    name: 🔬 Validate Nested RGD Templates
+    needs: [changes]
+    if: github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: 📑 Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: ⚙️ Setup Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: 📌 Install the reviewed YAML parser
+        env:
+          GOBIN: ${{ runner.temp }}/platform-validation-tools
+        run: |
+          go install github.com/mikefarah/yq/v4@v4.54.1
+          printf '%s\n' "$GOBIN" >> "$GITHUB_PATH"
+          PATH="$GOBIN:$PATH" yq --version
+
+      - name: ⚙️ Setup Trivy for nested RGD templates
+        uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1
+        with:
+          version: v0.74.0
+          cache: true
+
+      - name: 🔬 Validate nested RGD workload templates
+        # Extract every nested template, scan without the repository ignorefile,
+        # and retain the complete findings ratchet and behavioral controls.
+        run: |
+          shellcheck scripts/scan-rgd-templates.sh scripts/tests/test-rgd-template-static-scan.sh
+          bash scripts/tests/test-rgd-template-static-scan.sh
+
   validate:
     name: 🧪 Validate Manifests
     needs: [changes]
@@ -1874,23 +1914,6 @@ jobs:
           shellcheck scripts/tests/test-trivyignore-configmap-content-boundary.sh
           bash scripts/tests/test-trivyignore-configmap-content-boundary.sh
 
-      - name: ⚙️ Setup Trivy for nested RGD templates
-        if: needs.changes.outputs.k8s == 'true'
-        uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1
-        with:
-          version: v0.74.0
-          cache: true
-
-      - name: 🔬 Validate nested RGD workload templates
-        if: needs.changes.outputs.k8s == 'true'
-        # Trivy does not descend through a ResourceGraphDefinition custom resource. Extract every
-        # nested template, scan without the repository ignorefile, and compare complete findings,
-        # resource-graph controls, and committed substitution inputs to the explicit ratchet. The
-        # behavioral test also applies registry, placement, and policy-regression probes.
-        run: |
-          shellcheck scripts/scan-rgd-templates.sh scripts/tests/test-rgd-template-static-scan.sh
-          bash scripts/tests/test-rgd-template-static-scan.sh
-
       - name: 🔢 Validate scanner version-drift guard
         if: needs.changes.outputs.k8s == 'true'
         # Guards the guard, offline. The live comparison runs on main
@@ -4065,6 +4088,7 @@ jobs:
         validate-consumer-discovery,
         validate-publication-contract,
         validate-flux-bridge,
+        validate-rgd-templates,
         validate-matcher-efficacy,
         validate-eks-authorization,
         validate,
@@ -4082,6 +4106,28 @@ jobs:
     if: always()
     permissions: {}
     steps:
+      - name: 🔐 Require complete selected nested-template validation
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          K8S_CHANGED: ${{ needs.changes.outputs.k8s }}
+          RGD_RESULT: ${{ needs.validate-rgd-templates.result }}
+        run: |
+          case "$EVENT_NAME" in
+            pull_request)
+              case "$K8S_CHANGED" in
+                true) expected=success ;;
+                false) expected=skipped ;;
+                *) echo '::error::Cannot determine nested-template validation selection'; exit 1 ;;
+              esac
+              ;;
+            merge_group) expected=skipped ;;
+            *) echo '::error::Unknown nested-template validation event'; exit 1 ;;
+          esac
+          if [ "$RGD_RESULT" != "$expected" ]; then
+            echo "::error::Nested-template result was $RGD_RESULT; expected $expected"
+            exit 1
+          fi
+
       - name: 🔐 Require complete selected Flux bridge validation
         env:
           EVENT_NAME: ${{ github.event_name }}
@@ -4120,6 +4166,7 @@ jobs:
             ${{ needs.validate-consumer-discovery.result }}
             ${{ needs.validate-publication-contract.result }}
             ${{ needs.validate-flux-bridge.result }}
+            ${{ needs.validate-rgd-templates.result }}
             ${{ needs.validate-matcher-efficacy.result }}
             ${{ needs.validate-eks-authorization.result }}
             ${{ needs.validate.result }}

AGENTS.md

@@ -138,6 +138,13 @@ alongside manifest validation on pull requests selected by the existing `k8s` or
 and an intentional skip otherwise, rejecting missing selection evidence. The
 family is PR-only and is not a merge-group production prerequisite.
 
+The complete nested-RGD template regression runs alongside manifest validation
+in the read-only `validate-rgd-templates` PR job, selected by `k8s`. Required CI
+rejects failed, cancelled, missing or unexpectedly skipped selected scans. The
+scanner pin and full behavioral controls are unchanged; merge-group and recovery
+deployment retain their separate exact-checkout scans. See the
+[scheduling decision](docs/adr/ci-nested-template-parallelism.md).
+
 The scan is a **hard gate**: it fails the PR if the combined compliance score drops below the threshold, so new findings must be fixed or justified before merge. It evaluates **two** frameworks, NSA-CISA and MITRE ATT&CK, because NSA-CISA evaluates 20 controls in total but only 17 of the 76 named by the `ClusterSecurityException` CRs — the RBAC controls those CRs exist to govern were outside the gate entirely. Three non-obvious limits:
 
 - **ksail is Renovate-managed** (the Setup step, grouped `ksail` with the deploy pins). It was previously frozen at 7.65.0 because 7.66.x parallelised the in-process Helm render and made it racy — two distinct symptoms of the same regression: `ksail workload validate` non-deterministically corrupted the render with varying YAML parse errors ([devantler-tech/ksail#5362](https://github.com/devantler-tech/ksail/issues/5362), closed — contained since KSail v7.163.1 by the [ksail#5978](https://github.com/devantler-tech/ksail/issues/5978) stream-splitting fix, which is what let the temporary `--skip-helm-render` workaround be removed), and the scan's compliance score swung run-to-run ([devantler-tech/ksail#5371](https://github.com/devantler-tech/ksail/issues/5371), closed). Both are resolved upstream, so the pin is lifted back onto the latest release. Tripwire (kept in sync with the comments in `.github/workflows/ci.yaml`): if `validate` output or the `scan` score varies run-to-run again, re-add `--skip-helm-render` and reopen ksail#5362 (or re-pin to a known-good version, reopening #5371 if only the score swings).

docs/adr/ci-nested-template-parallelism.md

@@ -0,0 +1,34 @@
+# Parallel nested-template validation
+
+## Decision
+
+Run the complete nested ResourceGraphDefinition template regression in a
+read-only PR job alongside manifest validation, selected by the existing `k8s`
+filter. Keep its scanner version, action pin, finding baseline and behavioral
+controls unchanged. The required aggregate waits for this job and separately
+requires success whenever selected; missing evidence and unexpected skips fail.
+
+The PR-only job is not a production prerequisite. Merge-group validation and
+the current-main scan before recovery deployment retain their existing order.
+
+## Rationale
+
+Platform CI still exceeded fifteen minutes after earlier independent families
+were separated. The nested scan occupied 168–180 seconds inside the serial
+manifest job in three comparable successful runs. It needs only repository
+files and pinned offline tools, not a cluster, KSail or other validation results.
+Moving this work removes an unnecessary dependency without reducing coverage.
+
+Parallelism can shorten feedback while increasing setup and occupied runner
+time. It does not by itself prove that all checks finish within fifteen minutes.
+
+## Validation and recovery
+
+Workflow-graph mutation tests reject serialization, widened permissions and
+missing required dependencies. Tests execute the actual result guard against
+selected, unselected, failed, cancelled and unknown observations. The existing
+scanner regression and deployment-route wiring controls remain mandatory.
+
+Compare complete hosted run metadata before and after, reporting feedback and
+occupied runner time separately. Revert this scheduling change if the extra
+runner/setup cost outweighs its measured benefit; never remove the scan.

scripts/tests/openbao-health-native/listeners_test.go

@@ -0,0 +1,60 @@
+package native_test
+
+import (
+	"context"
+	"errors"
+	"net"
+	"os/exec"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// TestNativeListenerReservations proves initial listeners cannot reuse one another's addresses.
+func TestNativeListenerReservations(t *testing.T) {
+	for _, scenario := range []struct {
+		name  string
+		start bool
+	}{
+		{name: "before startup"},
+		{name: "after first startup", start: true},
+	} {
+		t.Run(scenario.name, func(t *testing.T) {
+			// A no-op executable exercises the real start path without binding
+			// sockets itself, making each reservation's release observable.
+			binary, err := exec.LookPath("true")
+			if err != nil {
+				t.Fatal(err)
+			}
+			c := newCluster(t, binary)
+			if scenario.start {
+				c.start(serverNames[0], true)
+			}
+			seen := map[string]bool{}
+			for _, name := range serverNames {
+				for _, addr := range []string{c.servers[name].apiAddr, c.servers[name].clusterAddr} {
+					if seen[addr] {
+						t.Fatal("native servers share a listener address")
+					}
+					seen[addr] = true
+					ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+					probe, err := (&net.ListenConfig{}).Listen(ctx, "tcp", addr)
+					cancel()
+					wantReserved := !scenario.start || name != serverNames[0]
+					if err == nil {
+						if closeErr := probe.Close(); closeErr != nil {
+							t.Fatal(closeErr)
+						}
+						if wantReserved {
+							t.Errorf("%s listener was released before its server started", name)
+						}
+					} else if !wantReserved {
+						t.Errorf("started server retained a reservation: %v", err)
+					} else if !errors.Is(err, syscall.EADDRINUSE) {
+						t.Errorf("listener was not proven reserved: %v", err)
+					}
+				}
+			}
+		})
+	}
+}

scripts/tests/openbao-health-native/native_test.go

@@ -43,14 +43,15 @@ const (
 var serverNames = []string{"openbao-0", "openbao-1", "openbao-2"}
 
 type server struct {
-	name        string
-	apiAddr     string
-	clusterAddr string
-	root        string
-	generation  int
-	command     *exec.Cmd
-	exited      chan struct{}
-	log         *os.File
+	name         string
+	apiAddr      string
+	clusterAddr  string
+	root         string
+	generation   int
+	command      *exec.Cmd
+	exited       chan struct{}
+	log          *os.File
+	reservations []net.Listener
 }
 
 type cluster struct {
@@ -195,6 +196,8 @@ func TestJudgeVerdictsAgainstNativeServers(t *testing.T) {
 	})
 }
 
+// newCluster prepares three disposable servers and reserves each initial API/peer
+// pair until its owning process starts.
 func newCluster(t *testing.T, binary string) *cluster {
 	t.Helper()
 	root := t.TempDir()
@@ -213,7 +216,9 @@ func newCluster(t *testing.T, binary string) *cluster {
 		client:   &http.Client{Timeout: 30 * time.Second},
 	}
 	for _, name := range serverNames {
-		c.servers[name] = &server{name: name, apiAddr: freeAddr(t), clusterAddr: freeAddr(t)}
+		api, peer := reserveAddr(t), reserveAddr(t)
+		c.servers[name] = &server{name: name, apiAddr: api.Addr().String(), clusterAddr: peer.Addr().String(),
+			reservations: []net.Listener{api, peer}}
 	}
 	t.Cleanup(func() {
 		for _, name := range serverNames {
@@ -249,17 +254,22 @@ func declaredRaftConfig(t *testing.T) string {
 	return document.Spec.Values.Server.HA.Raft.Config
 }
 
-func freeAddr(t *testing.T) string {
+// reserveAddr keeps an initial endpoint unavailable until its server starts.
+// Cleanup also releases listeners when setup fails before a server can start.
+func reserveAddr(t *testing.T) net.Listener {
 	t.Helper()
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatal(err)
 	}
-	addr := listener.Addr().String()
-	if err := listener.Close(); err != nil {
-		t.Fatal(err)
-	}
-	return addr
+	t.Cleanup(func() {
+		if err := listener.Close(); err != nil && !errors.Is(err, net.ErrClosed) {
+			t.Errorf("closing native listener reservation: %v", err)
+		}
+	})
+	return listener
 }
 
 var retryJoin = regexp.MustCompile(`(?m)^[ \t]*retry_join \{[^}\n]*\}[ \t]*\n`)
@@ -299,6 +309,8 @@ func (c *cluster) config(s *server, join bool) string {
 	return rendered + fmt.Sprintf("\napi_addr = %q\ncluster_addr = %q\n", "http://"+s.apiAddr, "https://"+s.clusterAddr)
 }
 
+// start releases only this server's initial reservations before launching its
+// native process. Restarts retain the server's assigned addresses.
 func (c *cluster) start(name string, join bool) {
 	c.t.Helper()
 	s := c.servers[name]
@@ -313,6 +325,14 @@ func (c *cluster) start(name string, join bool) {
 	if err := os.WriteFile(path, []byte(c.config(s, join)), 0o600); err != nil {
 		c.t.Fatal(err)
 	}
+	// Release only this server's pair. Every not-yet-started peer keeps its
+	// addresses reserved, preventing API and Raft listeners from sharing one.
+	for _, reservation := range s.reservations {
+		if err := reservation.Close(); err != nil {
+			c.t.Fatal(err)
+		}
+	}
+	s.reservations = nil
 	log, err := os.OpenFile(filepath.Join(s.root, "server.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
 	if err != nil {
 		c.t.Fatal(err)

scripts/tests/test-rgd-template-static-scan-wiring.sh

@@ -57,18 +57,28 @@ jq -e --argjson required_inputs '[
   fail "ci.yaml does not route every RGD gate input through k8s validation"
 
 jq -e --arg setup "$SETUP_TRIVY" --arg behavior_run "$BEHAVIORAL_RUN" '
-  .jobs.validate as $gate
+  .jobs["validate-rgd-templates"] as $gate
   | (($gate.if // "") ==
-      "github.event_name == '\''pull_request'\'' && (needs.changes.outputs.k8s == '\''true'\'' || needs.changes.outputs.bridge_validation == '\''true'\'')")
+      "github.event_name == '\''pull_request'\'' && needs.changes.outputs.k8s == '\''true'\''")
   and (($gate["continue-on-error"] // false) == false)
+  and ($gate.needs == ["changes"])
+  and ($gate.permissions == {"contents": "read"})
   and any($gate.steps[];
-      (.uses // "") == $setup and .with.version == "v0.74.0")
+      (.uses // "") == $setup and .with.version == "v0.74.0"
+      and .with.cache == true
+      and ((.["continue-on-error"] // false) == false)
+      and (has("if") | not))
   and any($gate.steps[];
       (.run // "") == $behavior_run
       and ((.["continue-on-error"] // false) == false)
-      and ((.if // "") == "needs.changes.outputs.k8s == '\''true'\''"))
+      and (has("if") | not))
+  and (.jobs.validate.needs | index("validate-rgd-templates") == null)
+  and (.jobs["deploy-prod"].needs | index("validate-rgd-templates") == null)
+  and (.jobs["ci-required-checks"].needs | index("validate-rgd-templates") != null)
+  and any(.jobs["ci-required-checks"].steps[];
+    (.with["job-results"] // "") | contains("${{ needs.validate-rgd-templates.result }}"))
 ' <<<"$ci_workflow_json" >/dev/null ||
-  fail "the pull-request validate job does not run the pinned RGD behavioral gate"
+  fail "the independent pull-request job does not retain the required pinned RGD behavioral gate"
 
 jq -e --arg setup "$SETUP_TRIVY" --arg behavior_run "$BEHAVIORAL_RUN" '
   .jobs["validate-rgd-templates-merge-group"] as $gate
@@ -188,6 +198,28 @@ if [ "${RGD_WIRING_SKIP_ABLATIONS:-false}" != "true" ]; then
   ablation_work="$(mktemp -d)"
   trap 'rm -rf "$ablation_work"' EXIT
 
+  # Prove the independent PR route and required aggregate cannot lose the scan.
+  pr_ablation_index=0
+  # shellcheck disable=SC2016 # GitHub expressions are literal workflow fixture data.
+  for mutation in \
+    'del(.jobs."validate-rgd-templates")' \
+    '.jobs."validate-rgd-templates".needs += ["validate"]' \
+    '.jobs."validate-rgd-templates".permissions.contents = "write"' \
+    '.jobs."validate-rgd-templates".if = false' \
+    '(.jobs."validate-rgd-templates".steps[] | select(.uses == "aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567")).if = false' \
+    '(.jobs."validate-rgd-templates".steps[] | select((.run // "") | contains("bash scripts/tests/test-rgd-template-static-scan.sh"))).continue-on-error = true' \
+    '.jobs."ci-required-checks".needs |= map(select(. != "validate-rgd-templates"))' \
+    '(.jobs."ci-required-checks".steps[] | select(has("with"))).with."job-results" = "${{ needs.validate.result }}"'; do
+    pr_ablation_index=$((pr_ablation_index + 1))
+    pr_workflow="${ablation_work}/ci-pr-${pr_ablation_index}.yaml"
+    cp "$CI_WORKFLOW" "$pr_workflow"
+    yq -i "$mutation" "$pr_workflow"
+    if RGD_WIRING_CI_WORKFLOW="$pr_workflow" \
+      RGD_WIRING_SKIP_ABLATIONS=true "$0" >"${ablation_work}/pr-${pr_ablation_index}.log" 2>&1; then
+      fail "the wiring validator accepted unsafe PR scan mutation ${pr_ablation_index}"
+    fi
+  done
+
   suppressed_wiring_step_workflow="${ablation_work}/ci-wiring-step-continue-on-error.yaml"
   cp "$CI_WORKFLOW" "$suppressed_wiring_step_workflow"
   yq -i '(.jobs."validate-repository-guards".steps[] |