From dfcf2c334851b1ef751dff90d1877128bd1dc9f7 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Thu, 1 Oct 2026 14:45:03 +0200 Subject: [PATCH] fix(release): run catalogue self-tests without production credentials --- .github/tests/create-release-fixture.sh | 88 +++++++++++++++ .../test-create-release-fixture-controls.sh | 42 +++++++ .github/tests/test-create-release-offline.sh | 106 ++++++++++++++++++ .github/workflows/ci.yaml | 24 +++- .github/workflows/create-release.yaml | 54 ++++++++- README.md | 5 +- 6 files changed, 313 insertions(+), 6 deletions(-) create mode 100644 .github/tests/create-release-fixture.sh create mode 100644 .github/tests/test-create-release-fixture-controls.sh create mode 100644 .github/tests/test-create-release-offline.sh diff --git a/.github/tests/create-release-fixture.sh b/.github/tests/create-release-fixture.sh new file mode 100644 index 00000000..66596334 --- /dev/null +++ b/.github/tests/create-release-fixture.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Execute the shipped release command/config against local Git remotes, without auth. +set -euo pipefail +root="$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +workflow="${1:-$root/.github/workflows/create-release.yaml}" +config="${2:-$root/.releaserc}" +disabled="${DISABLE_ISSUE_SIDE_EFFECTS:-false}" +[[ "$disabled" == true || "$disabled" == false ]] || { + echo 'invalid hook setting' >&2 + exit 1 +} +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +run="$(yq -r '.jobs.release.steps[] | select(.name == "๐ŸŽ‰ Release") | .run' "$workflow")" +hooks="\${{ inputs.disable-issue-side-effects && '--success false --fail false' || '' }}" +dry="\${{ inputs.dry-run && '--dry-run' || '' }}" +[[ "$run" == "npx semantic-release@25.0.3 $hooks $dry" ]] || { + echo 'FAIL: unsupported shipped release command' >&2 + exit 1 +} +flags='' +[[ "$disabled" == false ]] || flags='--success false --fail false' +run="${run//"$hooks"/$flags}" +run="${run//"$dry"/--dry-run}" +read -r -a command_parts <<<"$run" +cache="$(npm config get cache)" +: >"$work/npm-user.conf" +: >"$work/npm-global.conf" + +# Fixture setup and assertions must ignore ambient Git paths, config, templates and hooks too. +fixture_git() { + env -i PATH="$PATH" GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_COUNT=0 \ + GIT_ALLOW_PROTOCOL=file git -c core.hooksPath=/dev/null -c init.templateDir= "$@" +} +cases=0 +while IFS=$'\t' read -r name message expected; do + repo="$work/$name" + remote="$work/$name.git" + fixture_git -c init.defaultBranch=main init --quiet --bare "$remote" + fixture_git -c init.defaultBranch=main init --quiet "$repo" + fixture_git -C "$repo" config user.name 'Release fixture' + fixture_git -C "$repo" config user.email 'fixture@example.invalid' + cp "$config" "$repo/.releaserc" + fixture_git -C "$repo" add .releaserc + fixture_git -C "$repo" -c commit.gpgsign=false commit --quiet -m 'chore: baseline' + fixture_git -C "$repo" -c tag.gpgsign=false tag v1.2.3 + fixture_git -C "$repo" -c commit.gpgsign=false commit --quiet --allow-empty -m "$message" + fixture_git -C "$repo" remote add origin "file://$remote" + fixture_git -C "$repo" push --quiet origin main --tags + before="$(fixture_git -C "$repo" show-ref)" + remote_before="$(fixture_git --git-dir="$remote" show-ref)" + if ! (cd "$repo" && env -i PATH="$PATH" npm_config_cache="$cache" npm_config_offline=true npm_config_yes=true \ + npm_config_userconfig="$work/npm-user.conf" npm_config_globalconfig="$work/npm-global.conf" \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_COUNT=0 GIT_ALLOW_PROTOCOL=file \ + "${command_parts[@]}" --no-ci --repository-url "file://$remote") >"$work/output" 2>&1; then + cat "$work/output" >&2 + echo "FAIL: $name release command failed" >&2 + exit 1 + fi + if [[ "$expected" == none ]]; then + grep -qF 'no new version is released' "$work/output" || { + cat "$work/output" >&2 + echo "FAIL: $name expected no release" >&2 + exit 1 + } + else + grep -qF "The next release version is $expected" "$work/output" || { + cat "$work/output" >&2 + echo "FAIL: $name expected $expected" >&2 + exit 1 + } + fi + [[ "$(fixture_git -C "$repo" show-ref)" == "$before" && "$(fixture_git --git-dir="$remote" show-ref)" == "$remote_before" ]] || { + echo "FAIL: $name dry-run changed refs" >&2 + exit 1 + } + cases=$((cases + 1)) + echo "PASS: $name -> $expected; local and bare refs unchanged" +done <<'CASES' +fix fix: repair workflow 1.2.4 +perf perf: accelerate workflow 1.2.4 +revert revert: remove faulty change 1.2.4 +feature feat: add workflow 1.3.0 +breaking feat!: require new configuration 2.0.0 +scoped-breaking fix(release)!: require new configuration 2.0.0 +docs docs: clarify usage none +CASES +echo "PASS: $cases actual release decisions; issue-hook suppression=$disabled; no credentials" diff --git a/.github/tests/test-create-release-fixture-controls.sh b/.github/tests/test-create-release-fixture-controls.sh new file mode 100644 index 00000000..59f59291 --- /dev/null +++ b/.github/tests/test-create-release-fixture-controls.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Reject configuration/command regressions using the actual offline release suite. +set -euo pipefail +root="$(git rev-parse --show-toplevel)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +fixture="$root/.github/tests/create-release-fixture.sh" +workflow="$root/.github/workflows/create-release.yaml" +config="$root/.releaserc" +mkdir -p "$work/template/hooks" +printf '#!/bin/sh\nexit 99\n' >"$work/template/hooks/pre-commit" +chmod +x "$work/template/hooks/pre-commit" +if ! env GIT_DIR="$work/foreign-git" GIT_WORK_TREE="$work/foreign-tree" GIT_INDEX_FILE="$work/foreign-index" \ + GIT_TEMPLATE_DIR="$work/template" GIT_CONFIG_PARAMETERS="'core.hooksPath'='$work/template/hooks'" \ + bash "$fixture" >"$work/output" 2>&1; then + cat "$work/output" >&2 + echo 'FAIL: ambient Git overrides redirected fixture operations' >&2 + exit 1 +fi +[[ ! -e "$work/foreign-git" && ! -e "$work/foreign-tree" && ! -e "$work/foreign-index" ]] +echo 'PASS: ambient Git paths, templates and hooks cannot redirect fixture operations' +reject() { + local label="$1" workflow_path="$2" config_path="$3" diagnostic="$4" + if bash "$fixture" "$workflow_path" "$config_path" >"$work/output" 2>&1; then + echo "FAIL: $label was accepted" >&2 + exit 1 + fi + grep -qF "$diagnostic" "$work/output" || { + cat "$work/output" >&2 + exit 1 + } + echo "PASS: rejected $label ($diagnostic)" +} +jq 'del(.plugins[0][1].parserOpts.breakingHeaderPattern)' "$config" >"$work/no-parser.json" +reject 'missing breaking parser' "$workflow" "$work/no-parser.json" 'FAIL: breaking expected 2.0.0' +jq '(.plugins[0][1].releaseRules[] | select(.breaking == true) | .release) = false' "$config" >"$work/no-major.json" +reject 'disabled major rule' "$workflow" "$work/no-major.json" 'FAIL: breaking expected 2.0.0' +jq '.plugins[0][1].releaseRules |= map(select(.type != "revert"))' "$config" >"$work/no-revert.json" +reject 'missing revert rule' "$workflow" "$work/no-revert.json" 'FAIL: revert expected 1.2.4' +yq '(.jobs.release.steps[] | select(.name == "๐ŸŽ‰ Release") | .run) = "echo PASS"' "$workflow" >"$work/bypass.yaml" +reject 'release command bypass' "$work/bypass.yaml" "$config" 'unsupported shipped release command' +echo 'PASS: four independent behavioral regressions rejected' diff --git a/.github/tests/test-create-release-offline.sh b/.github/tests/test-create-release-offline.sh new file mode 100644 index 00000000..e7476eb9 --- /dev/null +++ b/.github/tests/test-create-release-offline.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# #348: catalogue release tests must execute without production write credentials. +set -euo pipefail +root="$(git rev-parse --show-toplevel)" +workflow="${1:-$root/.github/workflows/create-release.yaml}" +ci="${2:-$root/.github/workflows/ci.yaml}" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +yq -o=json '.' "$workflow" >"$work/workflow.json" +yq -o=json '.' "$ci" >"$work/ci.json" +jq -n --slurpfile w "$work/workflow.json" --slurpfile c "$work/ci.json" \ + '{workflow:$w[0],ci:$c[0]}' >"$work/bundle.json" +condition="\${{ github.event_name != 'merge_group' && !startsWith(github.event.head_commit.message, 'chore(main): release ') }}" +guard() { + jq -e --arg condition "$condition" ' + def runnable: .if == null and (.["continue-on-error"] // false) == false and + (.shell == null or .shell == "bash") and .["working-directory"] == null; + .workflow as $w | .ci as $c | $w.jobs["offline-test"] as $j | + if $w.on.workflow_call.inputs["offline-test"].type != "boolean" or + $w.on.workflow_call.inputs["offline-test"].default != false or + $w.on.workflow_call.inputs["dry-run"].default != false or + $w.on.workflow_call.secrets.APP_PRIVATE_KEY.required != false + then error("offline-test must be default-off and secret-free") + elif $w.jobs.release.if != "!inputs.offline-test" + then error("offline-test must exclude the production release job") + elif $j.if != "inputs.offline-test" or $j.needs != null or + ($j["continue-on-error"] // false) != false + then error("offline-test must execute independently") + elif $j.permissions != {contents:"read"} or + ([$j,$w.env // {}] | tostring | test("\\bsecrets\\b|create-github-app-token|github[.]token|GH_TOKEN|GITHUB_TOKEN|PRIVATE_KEY";"i")) + then error("offline-test must have read permission and no credentials") + elif [$j.steps[] | select(.uses != null)] | length != 3 + then error("offline-test must use only hardening, checkout and Node setup") + elif [$j.steps[] | select(.with.repository == "${{ job.workflow_repository }}" and + .with.ref == "${{ job.workflow_sha }}" and .with.path == ".devantler-tech-actions" and + .with["persist-credentials"] == false and .if == null)] | length != 1 + then error("offline-test must check out its immutable workflow commit") + elif ([$j.steps[] | select(.name == "๐Ÿ”Ž Validate offline mode") | + select(runnable and .env.DRY_RUN == "${{ inputs.dry-run }}" and + (.run | contains("[[ \"$DRY_RUN\" == true ]]")) and (.run | contains("exit 1")))] | length) != 1 + then error("offline-test must require dry-run before executing") + elif $j.defaults.run != {shell:"bash","working-directory":".devantler-tech-actions"} or + ([$j.steps[] | select(.run == "bash .github/tests/create-release-fixture.sh") | + select(runnable and .env.DISABLE_ISSUE_SIDE_EFFECTS == "${{ inputs.disable-issue-side-effects }}")] | length) != 1 + then error("offline release decisions must execute with the caller hook setting") + elif (["test-create-release","test-create-release-no-issue-side-effects"] | all(. as $name | + $c.jobs[$name].uses == "./.github/workflows/create-release.yaml" and + $c.jobs[$name].with["offline-test"] == true and $c.jobs[$name].with["dry-run"] == true and + $c.jobs[$name].secrets == null and $c.jobs[$name].permissions == {contents:"read"} and + $c.jobs[$name].if == $condition and $c.jobs[$name].needs == null and + ($c.jobs[$name]["continue-on-error"] // false) == false and + ($c.jobs["ci-required-checks"].needs | index($name)) != null and + any($c.jobs["ci-required-checks"].steps[]; (.env.JOB_RESULTS // "" | + contains("needs."+$name+".result")) and (.run // "" | contains("$JOB_RESULTS"))))) | not + then error("secret-free release calls must gate required CI") + elif (["test-create-release-offline.sh","test-create-release-fixture-controls.sh"] | all(. as $script | + [$c.jobs["test-create-release-config"].steps[] | select(.run == "bash .github/tests/"+$script)] | + length == 1 and all(runnable))) | not + then error("offline boundary regressions must execute in CI") + else true end' "$1" +} +guard "$work/bundle.json" >/dev/null +[[ "${3:-}" != --guard-only ]] || exit 0 +count=0 +while IFS=$'\t' read -r label mutation diagnostic; do + jq "$mutation" "$work/bundle.json" >"$work/mutated.json" + if guard "$work/mutated.json" >"$work/result" 2>&1; then + echo "FAIL: $label was accepted" >&2 + exit 1 + fi + grep -qF "$diagnostic" "$work/result" || { + cat "$work/result" >&2 + exit 1 + } + count=$((count + 1)) +done <<'CASES' +required secret .workflow.on.workflow_call.secrets.APP_PRIVATE_KEY.required=true secret-free +changed default .workflow.on.workflow_call.inputs["offline-test"].default=true default-off +consumer preview changed .workflow.on.workflow_call.inputs["dry-run"].default=true default-off +production execution .workflow.jobs.release.if=null exclude the production +skipped fixture .workflow.jobs["offline-test"].if="false" execute independently +fixture prerequisite .workflow.jobs["offline-test"].needs="release" execute independently +ignored failure .workflow.jobs["offline-test"]["continue-on-error"]=true execute independently +write permission .workflow.jobs["offline-test"].permissions.contents="write" read permission +inherited permission .workflow.jobs["offline-test"].permissions=null read permission +App key .workflow.jobs["offline-test"].env.KEY="${{ secrets.APP_PRIVATE_KEY }}" no credentials +bracket secret .workflow.jobs["offline-test"].env.KEY="${{ secrets['APP_PRIVATE_KEY'] }}" no credentials +uppercase secret .workflow.jobs["offline-test"].env.KEY="${{ SECRETS.APP_PRIVATE_KEY }}" no credentials +serialized secrets .workflow.jobs["offline-test"].env.KEY="${{ toJSON(secrets) }}" no credentials +App mint .workflow.jobs["offline-test"].steps += [{uses:"actions/create-github-app-token@sha"}] no credentials +token forwarding .workflow.jobs["offline-test"].env.GH_TOKEN="${{ github.token }}" no credentials +mutable checkout .workflow.jobs["offline-test"].steps |= map(if .with.path then .with.ref="main" else . end) immutable workflow commit +persisted token .workflow.jobs["offline-test"].steps |= map(if .with.path then .with["persist-credentials"]=true else . end) immutable workflow commit +fixture bypass .workflow.jobs["offline-test"].steps |= map(if .run == "bash .github/tests/create-release-fixture.sh" then .run="echo PASS" else . end) decisions must execute +hook setting lost .workflow.jobs["offline-test"].steps |= map(if .run then .env.DISABLE_ISSUE_SIDE_EFFECTS="true" else . end) caller hook setting +default App key .ci.jobs["test-create-release"].secrets.APP_PRIVATE_KEY="key" secret-free release calls +inherited secrets .ci.jobs["test-create-release-no-issue-side-effects"].secrets="inherit" secret-free release calls +live mode .ci.jobs["test-create-release"].with["offline-test"]=false secret-free release calls +write caller .ci.jobs["test-create-release"].permissions.contents="write" secret-free release calls +lost aggregation .ci.jobs["ci-required-checks"].needs |= map(select(. != "test-create-release")) gate required CI +lost result .ci.jobs["ci-required-checks"].steps |= map(if .env.JOB_RESULTS then .env.JOB_RESULTS |= gsub("needs.test-create-release.result";"needs.other.result") else . end) gate required CI +boundary skipped .ci.jobs["test-create-release-config"].steps |= map(if .run == "bash .github/tests/test-create-release-offline.sh" then .if="false" else . end) regressions must execute +invalid offline mode .workflow.jobs["offline-test"].steps |= map(select(.name != "๐Ÿ”Ž Validate offline mode")) require dry-run +behavior controls skipped .ci.jobs["test-create-release-config"].steps |= map(select(.run != "bash .github/tests/test-create-release-fixture-controls.sh")) regressions must execute +CASES +echo "PASS: release offline boundary rejects $count independent regressions" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a32d4cf5..8346643d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3641,21 +3641,23 @@ jobs: if: "${{ github.event_name != 'merge_group' && !startsWith(github.event.head_commit.message, 'chore(main): release ') }}" name: "[Test] Create Release - Dry Run" uses: ./.github/workflows/create-release.yaml + permissions: + contents: read with: dry-run: true - secrets: - APP_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }} + offline-test: true test-create-release-no-issue-side-effects: if: "${{ github.event_name != 'merge_group' && !startsWith(github.event.head_commit.message, 'chore(main): release ') }}" name: "[Test] Create Release - No Issue Side Effects" uses: ./.github/workflows/create-release.yaml + permissions: + contents: read with: disable-issue-side-effects: true warn-missing-breaking-bang: true dry-run: true - secrets: - APP_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }} + offline-test: true test-create-release-config: if: "${{ github.event_name != 'merge_group' && !startsWith(github.event.head_commit.message, 'chore(main): release ') }}" @@ -3683,6 +3685,20 @@ jobs: - name: ๐Ÿงช Verify breaking-change warning and silent cases run: bash .github/tests/test-create-release-breaking-bang.sh + - name: ๐Ÿงช Verify offline release credential boundary + run: bash .github/tests/test-create-release-offline.sh + + - name: ๐Ÿ“ฆ Setup Node.js for release-decision controls + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + + - name: ๐Ÿ“ฆ Acquire pinned release dependency for controls + run: npx --yes semantic-release@25.0.3 --version + + - name: ๐Ÿงช Verify release-decision regressions fail + run: bash .github/tests/test-create-release-fixture-controls.sh + test-release-please-semantics: if: "${{ github.event_name != 'merge_group' && !startsWith(github.event.head_commit.message, 'chore(main): release ') }}" name: "[Test] Release PR Lifecycle Guard" diff --git a/.github/workflows/create-release.yaml b/.github/workflows/create-release.yaml index 23ad2dda..cbc01027 100644 --- a/.github/workflows/create-release.yaml +++ b/.github/workflows/create-release.yaml @@ -17,14 +17,20 @@ on: required: false default: false type: boolean + offline-test: + description: "Run catalogue release-decision fixtures without production credentials (requires dry-run)" + required: false + default: false + type: boolean secrets: APP_PRIVATE_KEY: - required: true + required: false permissions: {} jobs: release: + if: "!inputs.offline-test" name: Release runs-on: ubuntu-latest # One release at a time per repository and ref, so overlapping merges cannot compute the same @@ -126,3 +132,49 @@ jobs: ${{ inputs.dry-run && '--dry-run' || '' }} env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} + + offline-test: + name: Offline release decisions + if: inputs.offline-test + runs-on: ubuntu-latest + permissions: + contents: read + defaults: + run: + shell: bash + working-directory: .devantler-tech-actions + steps: + - name: ๐Ÿ›ก๏ธ Harden runner + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: ๐Ÿ“‘ Checkout catalogue fixtures + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: .devantler-tech-actions + persist-credentials: false + + - name: ๐Ÿ“ฆ Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + + - name: ๐Ÿ”Ž Validate offline mode + env: + DRY_RUN: ${{ inputs.dry-run }} + run: | + [[ "$DRY_RUN" == true ]] || { + echo '::error::offline-test requires dry-run: true' >&2 + exit 1 + } + + - name: ๐Ÿ“ฆ Acquire pinned release dependency + run: npx --yes semantic-release@25.0.3 --version + + - name: ๐Ÿงช Evaluate release decisions without credentials + env: + DISABLE_ISSUE_SIDE_EFFECTS: ${{ inputs.disable-issue-side-effects }} + run: bash .github/tests/create-release-fixture.sh diff --git a/README.md b/README.md index 61cc02c6..ffbcd1d5 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,8 @@ This is a narrow configuration check, not proof that a commit will produce a maj Consumer rollout and the decision on removing this temporary flag are tracked in [devantler-tech/actions#1347](https://github.com/devantler-tech/actions/issues/1347). +For catalogue self-tests, set `offline-test: true` together with `dry-run: true` and omit the App secret. This independent job holds only contents-read permission and evaluates the pinned release command and catalogue configuration against disposable local repositories. It verifies patch, minor, major, and no-release decisions without changing refs. Both issue-hook settings are exercised; this does not test live notification delivery. Normal releases and consumer dry-run previews continue to use the consumer's own configuration and App credentials. + #### Usage ```yaml @@ -197,10 +199,11 @@ jobs: | Key | Type | Default | Required | Description | |------------------------------|-----------------|---------|----------|---------------------------------------------------------------------------| | `APP_CLIENT_ID` | Variable | - | Yes | GitHub App client ID used to mint the release token | -| `APP_PRIVATE_KEY` | Secret | - | Yes | GitHub App private key (paired with the `APP_CLIENT_ID` variable) | +| `APP_PRIVATE_KEY` | Secret | - | No | GitHub App private key; required for consumer releases and previews, omitted for offline tests | | `disable-issue-side-effects` | Input (boolean) | `false` | No | Disable success/fail hooks and omit issue/pull-request token permissions | | `warn-missing-breaking-bang` | Input (boolean) | `false` | No | Warn about missing explicit breaking-header handling in supported JSON configurations | | `dry-run` | Input (boolean) | `false` | No | Run semantic-release in dry-run mode (no tags or publishes) | +| `offline-test` | Input (boolean) | `false` | No | Run secret-free catalogue release-decision fixtures; requires dry-run |