Repository navigation
ci: run complete Go regressions independently #5052
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
12 commits
Select commit
Hold shift + click to select a range
37f8fda
ci: run complete Go regressions independently
devantler 2a90b90
test(ci): retain workflow contracts after job separation
devantler ff3ea22
fix(kro): declare namespaced schema scope on the tenant and webapp RGDs
devantler befe5d8
fix(kro): approve the scoped tenant RGD in the authorization surface
devantler 81fb21e
ci(policy): run complete policy validation in parallel
devantler d410169
ci: integrate schema-validation prerequisite
devantler aa1f138
ci: integrate schema-validation prerequisite
devantler acf88e6
test(ci): identify required authorization aggregator by action
devantler 9c1bdc7
test(ci): track policy result regression as executable
devantler f5a08a9
test(ci): guard complete-suite working directories
devantler d20ccc9
ci: integrate independent policy validation with full Go tests
devantler 314b364
ci: integrate merged policy validation with complete Go gate
devantler File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
137 changes: 137 additions & 0 deletions
137
scripts/validate-merge-group-heal/go_regressions_test.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "os" | ||
| "os/exec" | ||
| "slices" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "gopkg.in/yaml.v3" | ||
| ) | ||
|
|
||
| type regressionRunDefaults struct { | ||
| Run struct { | ||
| WorkingDirectory string `yaml:"working-directory"` | ||
| } `yaml:"run"` | ||
| } | ||
|
|
||
| type regressionWorkflow struct { | ||
| Defaults regressionRunDefaults `yaml:"defaults"` | ||
| Jobs map[string]struct { | ||
| Defaults regressionRunDefaults `yaml:"defaults"` | ||
| If string `yaml:"if"` | ||
| Needs []string `yaml:"needs"` | ||
| Continue any `yaml:"continue-on-error"` | ||
| Permissions map[string]string `yaml:"permissions"` | ||
| Steps []struct { | ||
| Name string `yaml:"name"` | ||
| If string `yaml:"if"` | ||
| Run string `yaml:"run"` | ||
| Continue any `yaml:"continue-on-error"` | ||
| Env map[string]string `yaml:"env"` | ||
| WorkingDirectory string `yaml:"working-directory"` | ||
| } `yaml:"steps"` | ||
| } `yaml:"jobs"` | ||
| } | ||
|
|
||
| // readRegressionWorkflow loads the actual CI workflow rather than a separate fixture. | ||
| func readRegressionWorkflow(t *testing.T) regressionWorkflow { | ||
| t.Helper() | ||
| data, err := os.ReadFile("../../.github/workflows/ci.yaml") | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| var workflow regressionWorkflow | ||
| if err := yaml.Unmarshal(data, &workflow); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| return workflow | ||
| } | ||
|
|
||
| // Full-suite work is independent of path classification. Serializing it ahead | ||
| // of changes delays every selected validator without adding a safety barrier. | ||
| func TestFullGoRegressionsRunIndependently(t *testing.T) { | ||
| workflow := readRegressionWorkflow(t) | ||
| job, found := workflow.Jobs["validate-go-regressions"] | ||
| if !found { | ||
| t.Fatal("full Go regressions need an independent validation job") | ||
| } | ||
| if len(job.Needs) != 0 || job.If != "" || job.Continue != nil { | ||
| t.Fatal("full regressions must start independently and run unconditionally") | ||
| } | ||
| if len(job.Permissions) != 1 || job.Permissions["contents"] != "read" { | ||
| t.Fatal("the regression runner needs only read-only checkout permission") | ||
| } | ||
| for _, command := range []string{ | ||
| "go test ./...", | ||
| "go -C scripts/retire-backstage-stateless/poller-native test -race -mod=readonly -count=1 ./...", | ||
| } { | ||
| found := false | ||
| for name, candidate := range workflow.Jobs { | ||
| for _, step := range candidate.Steps { | ||
| if strings.TrimSpace(step.Run) != command { | ||
| continue | ||
| } | ||
| if name != "validate-go-regressions" || step.If != "" || step.Continue != nil { | ||
| t.Fatalf("complete suite must run once, unconditionally in its independent job: %s", command) | ||
| } | ||
| for _, directory := range []string{ | ||
| workflow.Defaults.Run.WorkingDirectory, | ||
| candidate.Defaults.Run.WorkingDirectory, | ||
| step.WorkingDirectory, | ||
| } { | ||
| if directory != "" && directory != "." && directory != "./" { | ||
| t.Fatalf("complete suite must run from repository root, not %q: %s", directory, command) | ||
| } | ||
| } | ||
| if found { | ||
| t.Fatalf("duplicate complete-suite invocation: %s", command) | ||
| } | ||
| found = true | ||
| } | ||
| } | ||
| if !found { | ||
| t.Fatalf("complete suite was removed or narrowed: %s", command) | ||
| } | ||
| } | ||
| for _, name := range []string{"deploy-prod", "ci-required-checks"} { | ||
| if !slices.Contains(workflow.Jobs[name].Needs, "validate-go-regressions") { | ||
| t.Fatalf("%s must wait for full Go regressions", name) | ||
| } | ||
| } | ||
| deploy := workflow.Jobs["deploy-prod"] | ||
| if deploy.Continue != nil || strings.Contains(deploy.If, "always(") || strings.Contains(deploy.If, "cancelled(") || strings.Contains(deploy.If, "failure(") { | ||
| t.Fatal("deployment must not bypass failed or skipped dependencies") | ||
| } | ||
| } | ||
|
|
||
| // Execute the actual aggregate guard: GitHub considers some skipped checks | ||
| // successful, but an unconditional full suite has no legitimate skipped result. | ||
| func TestFullGoRegressionsRejectEveryNonSuccessResult(t *testing.T) { | ||
| workflow := readRegressionWorkflow(t) | ||
| gate := workflow.Jobs["ci-required-checks"] | ||
| if gate.If != "always()" { | ||
| t.Fatal("the aggregate must still run after a failed dependency") | ||
| } | ||
| for _, step := range gate.Steps { | ||
| if step.Name != "🧪 Require complete Go regressions" { | ||
| continue | ||
| } | ||
| if step.If != "" || step.Continue != nil || step.Env["REGRESSION_RESULT"] != "${{ needs.validate-go-regressions.result }}" { | ||
| t.Fatal("full-suite result enforcement must be unconditional and bind the real job") | ||
| } | ||
| for _, result := range []string{"success", "failure", "cancelled", "skipped", "", "unknown"} { | ||
| t.Run(result, func(t *testing.T) { | ||
| command := exec.Command("bash", "-e", "-o", "pipefail", "-c", step.Run) | ||
| command.Env = append(os.Environ(), "REGRESSION_RESULT="+result) | ||
| output, err := command.CombinedOutput() | ||
| if (err == nil) != (result == "success") { | ||
| t.Fatalf("result %q: error=%v, output=%s", result, err, output) | ||
| } | ||
| }) | ||
| } | ||
| return | ||
| } | ||
| t.Fatal("required aggregate has no full-suite result enforcement") | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.