diff --git a/.buildkite/README.md b/.buildkite/README.md new file mode 100644 index 0000000..0de7628 --- /dev/null +++ b/.buildkite/README.md @@ -0,0 +1,198 @@ +# api-spec on Buildkite (shadow mode) + +This directory ports api-spec's two **required** GitHub status checks to Buildkite. Each step runs +on a self-hosted agent, in its own disposable, isolated guest, on the `fpc-isolated` queue. + +**The GH workflows stay untouched.** Nothing under `.github/` changes in this port, and branch +protection's required contexts are **not** touched. The GH checks remain the required checks until +an operator decides otherwise -- see "Operator steps needed" below. Buildkite statuses are +informational until that decision is made. + +## Layout + +| path | role | +|---|---| +| `pipeline.yml` | Two parallel command steps. Uploaded by the UI bootstrap step, which runs caps-lint first. | +| `scripts/.sh` | One checked-in script per step. `command:` is only ever a script path. | +| `scripts/lib/common.sh` | Log groups, the gate runner, temp dirs cleaned on exit, an env-credential fence and a fork-PR fence. Trimmed subset of the org's internal Buildkite template (see the script's own header for what was dropped and why -- this pipeline needs no Node and no install credential). | + +## What is ported + +| step key | script | shadows (GH workflow / job) | required context | needs secret | timeout | +|---|---|---|---|---|---| +| `gate-checks` | `gate-checks.sh` | `.github/workflows/_checks.yml` job `checks` (called by `foundation-gate.yml` with `max_lines: 800`) | `gate / checks` | none | 10 | +| `secrets-content-policy` | `secrets-content-policy.sh` | `.github/workflows/public-repo-guard.yml` job `guard` | `Secrets + content policy` | `GUARD_PRIVATE_REPOS` | 10 | + +Each `command:` is a single checked-in script, so the pipeline is compatible with agent +`no-command-eval`. Both required contexts resolve on Buildkite as +`buildkite//gate-checks` and `buildkite//secrets-content-policy` +(pipeline-slug is assigned when the pipeline is created) -- **but only once the GitHub provider +settings below are enabled; see "Operator steps needed"**. + +### `gate-checks` (shadows `gate / checks`) + +Two gates, copied byte-faithful from `_checks.yml`'s `checks` job: the secret-regex grep (with its +`.github/.secret-allowlist` and inline `# pragma: allowlist secret`-shaped exemptions) and the +file-size gate (`.ts`/`.tsx`/`.js`/`.py`, 800-line ceiling, `.github/.filesize-allowlist` exemptions, +`.types.ts`/`.d.ts` excluded). `_checks.yml`'s other two jobs in this repo's call +(`skill-validate`, `verify-routes`) are not part of the `gate / checks` required context and are not +ported. + +**Difference from GH, on purpose:** `_checks.yml`'s secret-scan step echoes the full matched line +(`path:line:match`) into the GH Actions log. This step's `gate_secret_scan` prints only `path:line`, +with the matched text replaced by the same `«match redacted — open this location to view»` marker +`content-policy.sh` already uses -- a matched credential must never be echoed into a build log. The +detection regex, the `--exclude-dir` list and the allowlist files are unchanged. Proven locally: a +planted `AKIA`-prefixed test value fails the gate, and the captured stdout+stderr of the run was +grepped for the literal value afterward -- absent. + +### `secrets-content-policy` (shadows `Secrets + content policy`) + +The `guard` job's five steps, byte-faithful: gitleaks 8.30.1 installed with the **same pinned +sha256** as the GH workflow, `gitleaks detect --no-git --source . --config .gitleaks.toml --redact +--no-banner --exit-code 1`, the same ripgrep-with-PCRE2 fallback install (pinned sha256), then +`bash scripts/public-repo-guard/content-policy.sh .` and the gate's own fixture self-test +(`scripts/public-repo-guard/tests/body-policy.test.sh`). The workflow's other job, `body-guard`, +scans PR/issue/comment text out of the GitHub event payload; Buildkite has no equivalent trigger for +that, it is not a required context, and it is not ported. + +**`GUARD_PRIVATE_REPOS`**, handled exactly as the GH workflow handles it: GH sources it from an +org-level Actions *variable* (`env: GUARD_PRIVATE_REPOS: ${{ vars.GUARD_PRIVATE_REPOS }}`, not a +secret) and `content-policy.sh` treats an unset value as "this one rule is skipped" (its own +documented behavior, unchanged here). This pipeline does the same: the script reads +`GUARD_PRIVATE_REPOS` from its own environment and passes it through unmodified -- it never sets, +defaults, or widens it. On Buildkite, the `secrets-content-policy` step's +`secrets: [GUARD_PRIVATE_REPOS]` attribute in `pipeline.yml` exports the Buildkite cluster secret of +that name (org `wave`, cluster "WAVE self-hosted CI") as the `GUARD_PRIVATE_REPOS` env var for that +step only, and Buildkite redacts the value from build logs if it is ever printed. For parity with GH, +an operator must create that cluster secret with the same value as the GH org variable (see "Operator +steps needed"). + +**The cluster secret must exist, not merely be unset.** The Buildkite agent fetches every key named +in a step's `secrets:` list before the step's command runs at all; a key that does not exist in the +cluster fails that fetch and the job never starts -- `secrets-content-policy.sh` never runs, so its +own "unset -> skip the rule" fallback never gets a chance to apply. That differs from the GH Actions +variable, which really can be left undefined. To get the GH-equivalent behavior on Buildkite ("rule +skipped"), the cluster secret must exist with an **empty string** value, not be absent. + +**Same-repo PR builds receive this secret.** This pipeline builds push and pull_request events with +"Build PRs from forks" off (see "Pipeline settings" below); fork PRs never get a Buildkite agent at +all, so only contributors who can push a branch in `wave-av/api-spec` can trigger a build that sees +`GUARD_PRIVATE_REPOS`. That is the same trust boundary every other step on this queue already +operates under -- this secret does not widen it. It is also a low-sensitivity value: GitHub itself +stores the equivalent as a plain, unmasked Actions **variable**, not a secret, so step-scoped +injection plus automatic log redaction here is already stricter than the GH baseline. Restricting the +secret to protected-branch-only builds was considered and rejected: this step's job is to scan each +PR's own content for policy violations, including a PR that edits the step itself -- running it only +from a trusted ref would defeat that purpose. + +### Known pre-existing gap (not introduced here, not fixed here) + +`scripts/public-repo-guard/content-policy.sh`'s private-repo rule is entirely configuration-driven: +it only ever checks the names listed in `GUARD_PRIVATE_REPOS`. The GH org variable that name +currently holds lists nine product repos and does not include every org-private repository. This +repo's own CI config (workflow comments, `AGENTS.md`) plaintext-references at least two private +repos and one private npm-scoped package by name in a handful of lines -- none of which the +currently-configured allowlist catches. This is a **pre-existing content-policy gap in this repo**, +not introduced by this PR; full file:line detail and the local proof are in the PR description (not +duplicated here, on purpose, so this file does not itself add a fresh hit once the allowlist is +corrected). **Do not fix it in this PR** -- correcting the allowlist, or the flagged references, is +separate, tightly-scoped follow-up work. This step reproduces `content-policy.sh` exactly as +configured; it does not widen, narrow, or otherwise change what the script checks. + +GH's own required-check history for this workflow is not a reliable signal right now: the org's +GitHub Actions billing lock (documented in the org's other shadow-port PRs referenced in this PR's +description) causes most recent runs to complete in ~2 seconds with no steps executed, which is why +this port exists in the first place. + +## Guest-image prerequisites + +- **linux x86_64.** The pinned gitleaks and ripgrep releases are amd64. +- `bash`, `git`, `grep`, `wc` (gate-checks); `curl`, `tar`, `sha256sum`, `install` (secrets-content-policy, to fetch and verify the pinned tool releases). +- **Egress:** `github.com` releases for the pinned gitleaks/ripgrep downloads. `secrets-content-policy` + always re-downloads gitleaks -- there is no pre-baked-tool check for it -- and skips only the ripgrep + download, when an `rg` with PCRE2 support is already on `PATH`. `gate-checks` needs no network at all. +- No Node, no npm, no sudo, no services. Both steps run over the checked-out working tree only. + `secrets-content-policy` is the one exception to "no secrets": it declares + `secrets: [GUARD_PRIVATE_REPOS]` in `pipeline.yml` (see "Known pre-existing gap" above), scoped to + that step alone and redacted from build logs; `gate-checks` still needs none. + +## Pipeline settings (Buildkite UI, not YAML) + +- **Visibility: private.** The Buildkite *pipeline* stays private even though the GitHub repo is + public -- the pipeline is operational tooling, not the product. +- **Steps:** only the caps-lint bootstrap, matching the org's standard bootstrap step (lint, then + `&&`-upload from the host-provisioned, read-only caps-linter install, schema sha256 pinned in the + UI step), `timeout_in_minutes: 5`, `retry.manual: false`, `agents.queue: fpc-isolated`. +- **GitHub:** trigger on push and pull_request. **Build PRs from forks: off** (the second fence is + `bk_refuse_fork_build` in `lib/common.sh`, keyed on `BK_REPO_SLUG`). +- **Per-step GitHub commit statuses -- required for the two contexts in "What is ported" to exist + at all.** Left at Buildkite's defaults, GitHub sees one pipeline-level status, not one per step, + and neither `buildkite//gate-checks` nor + `buildkite//secrets-content-policy` is ever posted. Per Buildkite's GitHub pipeline + provider settings ( / + the REST API's `provider_settings`), enable all three, in order: + - `publish_commit_status` (UI: "Update commit statuses") -- posts any commit status at all. + - `publish_commit_status_per_step` (UI: "Create a status for each job") -- one status per job + instead of one for the whole build. + - `use_step_key_as_commit_status` -- uses each job's `key` (`gate-checks`, + `secrets-content-policy`) as the GitHub context instead of its emoji label. Requires the two + settings above to both be `true` first. +- **Cancel intermediate builds** and **skip intermediate builds** on `!main`. +- **`GUARD_PRIVATE_REPOS`** Buildkite cluster secret (org `wave`, cluster "WAVE self-hosted CI"), + matching the GH org Actions variable of the same name (see above), exported to the + `secrets-content-policy` step via its `secrets:` attribute in `pipeline.yml`. + +## Caps (checked by caps-lint before upload) + +- Both steps set `timeout_in_minutes: 10`. The org maximum is 60. +- `retry.automatic` only covers `exit_status: -1` (agent lost) and `255` (guest died), `limit: 1` + each. A real gate failure is never auto-retried. +- `agents.queue: fpc-isolated` is set at the pipeline level. +- No plugins, no cache plugin, no secret values in `env`, no `cancel_on_build_failing`, no + `bk pipeline convert`. + +## Run locally + +From the repo root, on linux x86_64 with `bash`, `git`, `curl`, `tar`, `sha256sum` on PATH: + +```sh +.buildkite/scripts/gate-checks.sh +.buildkite/scripts/secrets-content-policy.sh +``` + +`gate-checks.sh` needs nothing else, runs on any platform, and passes clean on `origin/main`. + +`secrets-content-policy.sh`'s gitleaks install step is **not** portable: it always fetches the pinned +linux-x64 tarball with no `command -v gitleaks` bypass, and `TOOLS_DIR` is placed first on `PATH`, so +even a native `gitleaks` already on `PATH` is shadowed once the download completes. Running the full +script therefore requires linux x86_64 (or an emulation layer that can execute that binary) -- it is +not a byte-faithful local repro on macOS or other non-x86_64-linux workstations. The ripgrep install +step, by contrast, does check `command -v rg` first: a native `rg` 14.1.1 (PCRE2 build) already on +PATH is used as-is and its download is skipped. Running `secrets-content-policy.sh` directly (outside +Buildkite) will not have `GUARD_PRIVATE_REPOS` injected by the step's `secrets:` attribute, so export it +by hand to match the GH org variable for parity (unset runs the same as GH's own "variable not +configured" case: that one rule is skipped, loudly, by `content-policy.sh` itself). + +## Operator steps needed + +This PR is code only. It does not, and cannot from inside a PR, do any of the following -- each is a +separate, operator-gated action: + +1. Create the Buildkite pipeline, pointed at this repo, with the settings above. The `fpc-isolated` + agents must run Buildkite agent v3.106.0 or later, the minimum version that supports the + pipeline-YAML `secrets:` attribute `secrets-content-policy` uses + (). +2. Enable `publish_commit_status`, `publish_commit_status_per_step` and + `use_step_key_as_commit_status` in the pipeline's GitHub provider settings -- without all three, + GitHub never sees `gate / checks` or `Secrets + content policy`'s Buildkite counterparts at all, + only one undifferentiated pipeline-level status. +3. Create the `GUARD_PRIVATE_REPOS` Buildkite cluster secret (org `wave`, cluster "WAVE self-hosted + CI") to match GH's org Actions variable. The `secrets-content-policy` step already declares + `secrets: [GUARD_PRIVATE_REPOS]` in `pipeline.yml`, so once the cluster secret exists it is + exported to that step automatically -- no further pipeline config needed. +4. Watch both steps run green (or identify and fix real failures -- `secrets-content-policy` is + expected to be red on `main` today; see "Known pre-existing gap" above) for a soak period before + considering any change to branch protection's required contexts. +5. Any change to `main`'s required status checks, or to `.github/workflows/*`, stays a distinct, + explicitly operator-approved change. This PR makes neither. diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..2277460 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,52 @@ +# api-spec required checks on Buildkite (SHADOW mode). See .buildkite/README.md. +# +# Uploaded only by the UI bootstrap step, which runs caps-lint first and uploads with `&&`. +# Every GH workflow stays untouched and remains the gate until this pipeline is green for 7 days. +# Nothing under .github/ changes, and branch protection's required contexts are not touched by this +# change -- both are separate, operator-gated decisions (see the README's "Operator steps needed"). +# +# Rules for every step here (enforced by caps-lint): +# - command is one checked-in script (compatible with agent no-command-eval) +# - explicit timeout_in_minutes (<= 30 here; org cap 60) +# - automatic retry only for infra loss (-1 agent lost, 255 guest died), limit 1 each; a real gate +# failure is never retried automatically, so a real red stays visible +# - no plugins, no secret values in env +# Steps have no depends_on: each runs in its own disposable guest, in parallel. + +agents: + queue: fpc-isolated + +env: + BK_REPO_SLUG: "wave-av/api-spec" + +steps: + - label: ":closed_lock_with_key: gate / checks (secret scan + file-size gate)" + key: gate-checks + command: .buildkite/scripts/gate-checks.sh + timeout_in_minutes: 10 + retry: + automatic: + - exit_status: -1 + limit: 1 + - exit_status: 255 + limit: 1 + manual: + allowed: true + permit_on_passed: false + + - label: ":lock: secrets + content policy (gitleaks + trade-secret gate)" + key: secrets-content-policy + command: .buildkite/scripts/secrets-content-policy.sh + secrets: + - GUARD_PRIVATE_REPOS + # gitleaks/ripgrep pinned-download install adds ~15-30s to the scan itself on a cold guest. + timeout_in_minutes: 10 + retry: + automatic: + - exit_status: -1 + limit: 1 + - exit_status: 255 + limit: 1 + manual: + allowed: true + permit_on_passed: false diff --git a/.buildkite/scripts/gate-checks.sh b/.buildkite/scripts/gate-checks.sh new file mode 100755 index 0000000..aa3cae1 --- /dev/null +++ b/.buildkite/scripts/gate-checks.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Step `gate-checks`: shadow port of the required GH check "gate / checks" -- the `checks` job in +# .github/workflows/_checks.yml, called by foundation-gate.yml as `jobs: gate: uses: ./.github/workflows/_checks.yml` +# with `max_lines: 800`. +# +# Only the `checks` job is ported here (the required context is "gate / checks", i.e. outer job +# "gate" + inner job "checks"). foundation-gate.yml's other jobs (skill-validate, verify-routes, +# spec-lint, sdk-types, breaking-change, skills-index-coverage) are separate, non-required jobs and +# are out of scope for this port. +# +# The two gates below are copied BYTE-FAITHFUL (same regex, same commands, same allowlist files) from +# _checks.yml's `checks` job. They are wrapped in shell functions instead of inline `run:` blocks so +# bk_gate can run both and report every failure (see lib/common.sh: DIFFERENCE FROM GH, on purpose -- +# a GH Actions job stops at its first failing step; every gate here still runs). +# +# ANOTHER DIFFERENCE FROM GH, on purpose: _checks.yml's secret-scan step echoes the full matched +# line (path:line:match) into the GH Actions log. This step prints only path:line, the same +# redaction convention content-policy.sh already uses ("match redacted — open this location to +# view") -- a matched credential must never be echoed into a Buildkite build log. The detection +# regex and allowlist logic are unchanged; only what gets printed on a hit differs. +set -euo pipefail +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/common.sh +source "${here}/lib/common.sh" + +cd "$BK_REPO_ROOT" +bk_preflight + +# Mirrors _checks.yml's `inputs.max_lines` (default 800), which foundation-gate.yml pins to 800. +MAX="${MAX:-800}" + +# --- Secret scan (fail-closed, allowlist-aware) ------------------------------------------------- +# Detection logic is a byte-faithful copy of _checks.yml's "Secret scan (fail-closed, +# allowlist-aware)" step (same regex, same --exclude-dir list, same allowlist files). What gets +# PRINTED on a hit is not byte-faithful, deliberately: see the file header ("ANOTHER DIFFERENCE +# FROM GH, on purpose"). $hits is grep -n output ("path:line:matched-text"); only "path:line" is +# ever echoed, never the matched text. +gate_secret_scan() { + local hits + hits=$(grep -rIEn '(sk-[A-Za-z0-9]{20}|sk_(live|test)_[A-Za-z0-9]{20}|npm_[A-Za-z0-9]{30}|sbp_[a-f0-9]{40}|github_pat_[A-Za-z0-9_]{40}|AKIA[0-9A-Z]{16}|ghp_[A-Za-z0-9]{30}|AIzaSy[A-Za-z0-9_-]{20}|xai-[A-Za-z0-9]{40}|xoxb-[A-Za-z0-9-]+|-----BEGIN [A-Z ]*PRIVATE KEY)' \ + --exclude-dir=.git --exclude-dir=node_modules --exclude-dir=dist . | grep -v 'allowlist secret' \ + | { [ -f .github/.secret-allowlist ] && grep -vFf .github/.secret-allowlist || cat; } || true) + if [ -n "$hits" ]; then + echo "::error::secret-like pattern found — do not commit credentials" + printf '%s\n' "$hits" | sed -E 's/^([^:]+:[0-9]+):.*/\1: «match redacted — open this location to view»/' + return 1 + fi + echo "secret-scan clean" +} + +# --- File-size gate ------------------------------------------------------------------------------ +# Byte-faithful copy of _checks.yml's "File-size gate" step. MAX takes the place of `${{ inputs.max_lines }}`. +gate_file_size() { + local fail=0 f n + while IFS= read -r f; do + grep -qxF "$f" .github/.filesize-allowlist 2>/dev/null && continue # justified exception + n=$(wc -l < "$f") + if [ "$n" -gt "$MAX" ]; then echo "::error::$f has $n lines (> $MAX)"; fail=1; fi + done < <(git ls-files '*.ts' '*.tsx' '*.js' '*.py' | grep -vE '\.(types|d)\.ts$') + if [ "$fail" = 0 ]; then echo "file-size gate passed (all <= $MAX lines)"; fi + return "$fail" +} + +bk_gate "secret scan (fail-closed, allowlist-aware)" gate_secret_scan +bk_gate "file-size gate (max ${MAX} lines)" gate_file_size + +bk_gates_summary diff --git a/.buildkite/scripts/lib/common.sh b/.buildkite/scripts/lib/common.sh new file mode 100755 index 0000000..550cf7d --- /dev/null +++ b/.buildkite/scripts/lib/common.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# api-spec Buildkite shadow port: shared step helpers. SOURCED by every .buildkite/scripts/.sh, +# never run directly. +# +# VENDORED, trimmed subset of the org's private platform-foundation repo's Buildkite template v0 +# `lib/common.sh` (queue fpc-isolated). api-spec is PUBLIC and that template lives in a PRIVATE repo, +# so GitHub Actions' "public repo cannot consume a private reusable workflow" rule applies to any +# cross-repo reference too -- the helpers are copied in, not referenced, same reasoning +# .github/workflows/foundation-gate.yml already documents for _checks.yml. +# +# Trimmed vs. the upstream template: this pipeline's two steps need no Node, no npm/pnpm install and +# no GitHub Packages credential (gate-checks is grep/wc; secrets-content-policy is gitleaks/ripgrep +# over the working tree), so bk_assert_node_major, the npm/pnpm install helpers and the NODE_AUTH_TOKEN +# handling were dropped. bk_refuse_env_names, bk_refuse_fork_build, bk_gate/bk_gates_summary, +# bk_section/bk_err/bk_mktemp_dir/bk_require_tools are kept, unchanged in behavior. +# +# Contract every step script inherits from here: +# - `set -euo pipefail`, and never `set -x` (xtrace would print expanded values into the build log). +# - Never print an environment variable's VALUE. Errors name the variable, never its contents. +# - Temp dirs are job-local and removed on exit, so a disposable guest has nothing left to leak. +# +# Per-repo parameters (plain, non-secret values in the pipeline's top-level `env:`): +# BK_REPO_SLUG "wave-av/api-spec". REQUIRED by bk_refuse_fork_build. +# BK_EXTRA_DENY_ENV_NAMES optional, space-separated names added to the bk_refuse_env_names list. +set -euo pipefail + +_bk_lib_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -z "${BK_REPO_ROOT:-}" ]]; then + if ! BK_REPO_ROOT="$(git -C "$_bk_lib_dir" rev-parse --show-toplevel 2>/dev/null)"; then + printf 'error: %s\n' "cannot resolve the repo root from ${_bk_lib_dir} (not a git checkout); set BK_REPO_ROOT" >&2 + exit 1 + fi +fi +export BK_REPO_ROOT + +_bk_cleanup_paths=() +_bk_failed_gates=() + +# Credentials no CI step on a shared queue may hold, even though neither step in this pipeline needs +# one -- this is defence in depth against a misconfigured agent environment hook. +BK_DEFAULT_DENY_ENV_NAMES=( + DOPPLER_TOKEN DOPPLER_SERVICE_TOKEN + CLOUDFLARE_API_TOKEN CLOUDFLARE_ACCOUNT_ID CLOUDFLARE_API_KEY CF_API_TOKEN CF_ACCOUNT_ID CF_API_KEY + WAVE_GATEWAY_SECRET + GH_TOKEN GITHUB_TOKEN GH_ENTERPRISE_TOKEN NPM_TOKEN NODE_AUTH_TOKEN + AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + SUPABASE_ACCESS_TOKEN SUPABASE_SERVICE_ROLE_KEY SUPABASE_DB_PASSWORD + ANTHROPIC_API_KEY OPENAI_API_KEY + BUILDKITE_AGENT_TOKEN BUILDKITE_API_TOKEN +) + +bk_cleanup_on_exit() { + local p + for p in "${_bk_cleanup_paths[@]+"${_bk_cleanup_paths[@]}"}"; do + rm -rf -- "$p" + done +} +trap bk_cleanup_on_exit EXIT + +# Buildkite log group. `---` is collapsed; bk_gate expands the group again when its gate fails. +bk_section() { + printf -- '--- %s\n' "$*" +} + +bk_err() { + printf 'error: %s\n' "$*" >&2 +} + +# bk_mktemp_dir : create a job-local temp dir, register it for cleanup and store its path in +# . It sets a variable instead of printing the path, because a $(...) subshell would lose the +# cleanup registration. +bk_mktemp_dir() { + local __dir + __dir="$(mktemp -d "${TMPDIR:-/tmp}/bk.XXXXXX")" + _bk_cleanup_paths+=("$__dir") + printf -v "$1" '%s' "$__dir" +} + +bk_require_tools() { + local tool missing=0 + for tool in "$@"; do + if ! command -v "$tool" >/dev/null 2>&1; then + bk_err "required tool '$tool' is not on PATH (guest-image prerequisite, see .buildkite/README.md)" + missing=1 + fi + done + return "$missing" +} + +# bk_require_param []: a per-repo parameter must be set, non-empty and (optionally) match +# . Parameters are plain values, so a mismatch may show the value; secrets are never parameters. +bk_require_param() { + local name="$1" re="${2:-}" + if [[ -z "${!name:-}" ]]; then + bk_err "$name is not set. It is a required per-repo parameter of this pipeline; set it in .buildkite/pipeline.yml's top-level env" + return 1 + fi + if [[ -n "$re" && ! "${!name}" =~ $re ]]; then + bk_err "$name='${!name}' does not match ${re}" + return 1 + fi +} + +# bk_refuse_env_names [NAME...]: fail when any denied credential name is present in the environment, +# even if empty. The pipeline never sets one; this refuses to run if the agent environment leaked one +# in. The list is the default set plus BK_EXTRA_DENY_ENV_NAMES plus the arguments; it only grows. +# shellcheck disable=SC2120 # the arguments are optional extra names +bk_refuse_env_names() { + local name leaked=0 + local -a names=("${BK_DEFAULT_DENY_ENV_NAMES[@]}") + if [[ -n "${BK_EXTRA_DENY_ENV_NAMES:-}" ]]; then + local -a extra + read -r -a extra <<<"$BK_EXTRA_DENY_ENV_NAMES" + names+=("${extra[@]}") + fi + names+=("$@") + for name in "${names[@]}"; do + if [[ ! "$name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + bk_err "deny list entry is not a valid environment variable name; refusing to run" + return 1 + fi + if [[ -n "${!name+x}" ]]; then + bk_err "$name is set in this job's environment; CI steps must not hold this credential. Remove it from the agent environment hook." + leaked=1 + fi + done + return "$leaked" +} + +# bk_refuse_fork_build: on a PR build, the head repo must be BK_REPO_SLUG. The primary fence is the +# pipeline setting "Build PRs from forks: off"; this catches a misconfiguration. It cannot stop a +# hostile fork on its own, because a fork PR controls its own copy of this file. +bk_refuse_fork_build() { + bk_require_param BK_REPO_SLUG '^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$' || return 1 + if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then + return 0 + fi + local slug_re="${BK_REPO_SLUG//./\\.}" + local re="^(https://|git://|ssh://git@|git@)github\\.com[:/]${slug_re}(\\.git)?$" + if [[ ! "${BUILDKITE_PULL_REQUEST_REPO:-}" =~ $re ]]; then + bk_err "BUILDKITE_PULL_REQUEST_REPO is not ${BK_REPO_SLUG}; refusing to build a PR from another repository" + return 1 + fi +} + +# bk_preflight: the checks every step runs first, in one call. +bk_preflight() { + local rc=0 + # shellcheck disable=SC2119 # the default list plus BK_EXTRA_DENY_ENV_NAMES; no extra arguments here + bk_refuse_env_names || rc=1 + bk_refuse_fork_build || rc=1 + return "$rc" +} + +# bk_gate : run one gate in its own log group and record a failure without +# stopping, so a single build reports every failing gate. Each gate is one external command (or shell +# function), because errexit is suspended inside `||` and a multi-command body would hide an early +# failure. +# +# DIFFERENCE FROM GH, on purpose: a GH Actions job stops at its first failing step. Here every gate +# still runs, and bk_gates_summary lists every failure at once -- the same convention the fleet's other +# shadow ports use (it surfaces more in one build instead of a red/fix/red/fix loop). +bk_gate() { + local name="$1" + shift + bk_section "$name" + local rc=0 + "$@" || rc=$? + if ((rc != 0)); then + printf '^^^ +++\n' + bk_err "gate failed: ${name} (exit ${rc})" + _bk_failed_gates+=("${name} (exit ${rc})") + fi +} + +# Last call of a gate-running script: exits non-zero when any bk_gate failed, listing each one. +bk_gates_summary() { + if ((${#_bk_failed_gates[@]} > 0)); then + printf '+++ %s gate(s) failed\n' "${#_bk_failed_gates[@]}" + printf ' - %s\n' "${_bk_failed_gates[@]}" + return 1 + fi + printf '+++ all gates passed\n' +} diff --git a/.buildkite/scripts/secrets-content-policy.sh b/.buildkite/scripts/secrets-content-policy.sh new file mode 100755 index 0000000..6ba7dbb --- /dev/null +++ b/.buildkite/scripts/secrets-content-policy.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Step `secrets-content-policy`: shadow port of the required GH check "Secrets + content policy" -- +# the `guard` job in .github/workflows/public-repo-guard.yml. (The workflow's other job, `body-guard`, +# scans PR/issue/comment BODIES via the GitHub event payload, which Buildkite has no equivalent +# trigger for; it is not required and is out of scope for this port.) +# +# Every sub-check below is copied BYTE-FAITHFUL from the `guard` job: the same gitleaks version and +# sha256, the same ripgrep fallback-install logic, and the same script invocations. Wrapped in +# bk_gate so every failure is reported in one build (see lib/common.sh: DIFFERENCE FROM GH, on +# purpose -- a GH Actions job stops at its first failing step). +# +# GUARD_PRIVATE_REPOS is handled exactly as the GH workflow handles it: read from the environment +# (GH: `env: GUARD_PRIVATE_REPOS: ${{ vars.GUARD_PRIVATE_REPOS }}`, an org Actions *variable*, not a +# secret) and passed through to content-policy.sh unmodified. This script does not set, default, or +# widen it. An operator must configure a Buildkite pipeline (or org) environment variable of the same +# name, with the same value as the GH org variable, for this rule to run with parity. Left unset, +# content-policy.sh's own documented behavior applies: the private-repo rule is skipped (see +# scripts/public-repo-guard/content-policy.sh, "Unset locally -> this check is skipped"). +set -euo pipefail +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/common.sh +source "${here}/lib/common.sh" + +cd "$BK_REPO_ROOT" +bk_preflight +bk_require_tools curl tar sha256sum + +TOOLS_DIR="" +bk_mktemp_dir TOOLS_DIR +export PATH="${TOOLS_DIR}:${PATH}" + +# --- Install gitleaks (pinned + checksum-verified) ----------------------------------------------- +# Byte-faithful copy of public-repo-guard.yml's "Install gitleaks (pinned + checksum-verified)" step. +gate_install_gitleaks() { + local GITLEAKS_VERSION="8.30.1" + local GITLEAKS_SHA256="551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb" + local workdir="" + bk_mktemp_dir workdir + ( + cd "$workdir" + curl -fsSL --proto '=https' --tlsv1.2 -o gitleaks.tar.gz \ + "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + echo "${GITLEAKS_SHA256} gitleaks.tar.gz" | sha256sum -c - + tar -xzf gitleaks.tar.gz gitleaks + install -m 0755 gitleaks "${TOOLS_DIR}/gitleaks" + ) + gitleaks version +} + +# --- gitleaks (secret scan — published tree) ------------------------------------------------------ +# Byte-faithful copy of public-repo-guard.yml's "gitleaks (secret scan — published tree)" step. +gate_gitleaks_scan() { + gitleaks detect --no-git --source . --config .gitleaks.toml --redact --no-banner --exit-code 1 +} + +# --- Install ripgrep (PCRE2 build required) --------------------------------------------------- +# Byte-faithful copy of public-repo-guard.yml's "Install ripgrep (PCRE2 build required)" step +# (identical in both the `guard` and `body-guard` jobs). +gate_install_ripgrep() { + local RIPGREP_VERSION="14.1.1" + local RIPGREP_SHA256="4cf9f2741e6c465ffdb7c26f38056a59e2a2544b51f7cc128ef28337eeae4d8e" + if ! command -v rg >/dev/null || ! rg --pcre2-version >/dev/null 2>&1; then + local workdir="" + bk_mktemp_dir workdir + ( + cd "$workdir" + curl -fsSL --proto '=https' --tlsv1.2 -o ripgrep.tar.gz \ + "https://github.com/BurntSushi/ripgrep/releases/download/${RIPGREP_VERSION}/ripgrep-${RIPGREP_VERSION}-x86_64-unknown-linux-musl.tar.gz" + echo "${RIPGREP_SHA256} ripgrep.tar.gz" | sha256sum -c - + tar -xzf ripgrep.tar.gz --strip-components=1 "ripgrep-${RIPGREP_VERSION}-x86_64-unknown-linux-musl/rg" + install -m 0755 rg "${TOOLS_DIR}/rg" + ) + hash -r + fi + rg --pcre2-version +} + +# --- content policy (WAVE trade-secret / internal-leak gate) ------------------------------------- +# Byte-faithful invocation from public-repo-guard.yml's "content policy" step. GUARD_PRIVATE_REPOS +# flows through from this job's own environment (see header comment above); never set here. +gate_content_policy() { + GUARD_PRIVATE_REPOS="${GUARD_PRIVATE_REPOS:-}" bash scripts/public-repo-guard/content-policy.sh . +} + +# --- body policy self-test (fixtures) ------------------------------------------------------------- +# Byte-faithful invocation from public-repo-guard.yml's "body policy self-test (fixtures)" step. +# This runs the gate's own fixture suite, not a scan of this build's PR/issue body (Buildkite has no +# equivalent event payload for that -- see header comment above). +gate_body_policy_selftest() { + bash scripts/public-repo-guard/tests/body-policy.test.sh +} + +bk_gate "install gitleaks 8.30.1 (pinned + checksum-verified)" gate_install_gitleaks +bk_gate "gitleaks (secret scan — published tree)" gate_gitleaks_scan +bk_gate "install ripgrep (PCRE2 build required)" gate_install_ripgrep +bk_gate "content policy (WAVE trade-secret / internal-leak gate)" gate_content_policy +bk_gate "body policy self-test (fixtures)" gate_body_policy_selftest + +bk_gates_summary