diff --git a/.github/workflows/notebook-tests.yml b/.github/workflows/notebook-tests.yml new file mode 100644 index 0000000000..a33cd836f8 --- /dev/null +++ b/.github/workflows/notebook-tests.yml @@ -0,0 +1,151 @@ +name: Notebook Tests (v3 examples) + +# --------------------------------------------------------------------------- +# Runs the SageMaker v3 example notebooks on real SageMaker infrastructure when a +# PR touches v3-examples/**. Flow: +# 1. Collaborator check -> collaborators auto-approve; fork / non-collaborator +# PRs are gated behind the manual-approval environment. +# 2. Assume an AWS role via GitHub OIDC (no stored AWS keys) and start the +# `notebook-test-engine` CodeBuild project against this PR's notebooks. +# 3. Poll the build to completion; this job's pass/fail IS the PR check, and a +# failing notebook is attached as a downloadable `failing-notebooks` artifact. +# --------------------------------------------------------------------------- + +on: + pull_request_target: + branches: + - master + paths: + - 'v3-examples/**' + +# Only the newest commit of a PR runs; a new push cancels the older run. +concurrency: + group: notebook-tests-${{ github.event.pull_request.number || github.head_ref }} + cancel-in-progress: true + +permissions: + id-token: write # required to mint the GitHub OIDC token for AWS + contents: read + +jobs: + # 1) Collaborator? -> auto-approve. Otherwise -> manual-approval (gated below). + collab-check: + runs-on: ubuntu-latest + outputs: + approval-env: ${{ steps.collab-check.outputs.result }} + steps: + - name: Collaborator check + uses: actions/github-script@v7 + id: collab-check + with: + github-token: ${{ secrets.COLLAB_CHECK_TOKEN }} + result-encoding: string + script: | + try { + const res = await github.rest.repos.checkCollaborator({ + owner: context.repo.owner, + repo: context.repo.repo, + username: "${{ github.event.pull_request.user.login }}", + }); + return res.status == 204 ? "auto-approve" : "manual-approval"; + } catch (error) { + return "manual-approval"; + } + + # 2) Non-collaborators block on the "manual-approval" environment's required + # reviewers. Collaborators pass straight through the "auto-approve" env. + wait-for-approval: + runs-on: ubuntu-latest + needs: [collab-check] + environment: ${{ needs.collab-check.outputs.approval-env }} + steps: + - run: echo "Approved — starting notebook tests." + + # 3) Assume the engine role via OIDC, start the build against the PR's notebooks, + # and poll to completion. This job's result is the PR check. + notebook-tests: + runs-on: ubuntu-latest + needs: [wait-for-approval] + # GitHub hard-caps hosted jobs at 360 min; the CodeBuild project times out at + # 300 min, so 350 leaves margin for start + poll overhead. + timeout-minutes: 350 + steps: + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.NOTEBOOK_TEST_ENGINE_ROLE_ARN }} + aws-region: us-west-2 + # 6h session so the poll below outlasts the longest possible (5h) build. + role-duration-seconds: 21600 + + - name: Start CodeBuild and wait + id: build + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_SHA: ${{ github.event.pull_request.head.sha }} + PR_BRANCH: ${{ github.event.pull_request.head.ref }} + PR_REPO: ${{ github.event.pull_request.head.repo.full_name }} + PR_REPO_FULL: ${{ github.event.pull_request.base.repo.full_name }} + PR_ARTIFACTS_BUCKET: notebook-test-engine-pr-674622101542-us-west-2 + run: | + set -euo pipefail + PROJECT=notebook-test-engine + + # Check out the PR head of the 'sdk' secondary source. refs/pull//head + # resolves in the BASE repo even for fork PRs; ^{} pins the exact commit. + SDK_VERSION="refs/pull/${PR_NUMBER}/head^{${PR_SHA}}" + + BUILD_ID=$(aws codebuild start-build \ + --project-name "$PROJECT" \ + --secondary-sources-version-override "[{\"sourceIdentifier\":\"sdk\",\"sourceVersion\":\"${SDK_VERSION}\"}]" \ + --environment-variables-override \ + "name=TEST_MODE,value=pr_check,type=PLAINTEXT" \ + "name=PR_NUMBER,value=${PR_NUMBER},type=PLAINTEXT" \ + "name=PR_BRANCH,value=${PR_BRANCH},type=PLAINTEXT" \ + "name=PR_REPO,value=${PR_REPO},type=PLAINTEXT" \ + "name=PR_SHA,value=${PR_SHA},type=PLAINTEXT" \ + "name=PR_REPO_FULL,value=${PR_REPO_FULL},type=PLAINTEXT" \ + --query 'build.id' --output text) + + echo "Started build: $BUILD_ID" + echo "Console: https://us-west-2.console.aws.amazon.com/codesuite/codebuild/projects/${PROJECT}/build/${BUILD_ID//:/%3A}/log" + + # Poll until the build leaves IN_PROGRESS. + STATUS=IN_PROGRESS + while [ "$STATUS" = "IN_PROGRESS" ]; do + sleep 300 + STATUS=$(aws codebuild batch-get-builds --ids "$BUILD_ID" \ + --query 'builds[0].buildStatus' --output text 2>/dev/null || echo IN_PROGRESS) + echo "Build status: $STATUS" + done + + echo "Final build status: $STATUS" + echo "status=$STATUS" >> "$GITHUB_OUTPUT" + + # On any non-success, pull the engine-rendered failing-notebook HTML/ipynb + # (written under pr///) so the next step can attach it to this run. + if [ "$STATUS" != "SUCCEEDED" ]; then + echo "Build did not succeed; fetching this run's failing-notebook artifacts..." + # Only the NEWEST run-timestamp folder -- not every historical render for + # this PR -- so the artifact holds just this run's notebook(s). + LATEST=$(aws s3 ls "s3://${PR_ARTIFACTS_BUCKET}/pr/${PR_NUMBER}/" 2>/dev/null | awk '{print $NF}' | grep '/$' | sort | tail -1 || true) + if [ -n "$LATEST" ]; then + aws s3 cp --recursive "s3://${PR_ARTIFACTS_BUCKET}/pr/${PR_NUMBER}/${LATEST}" pr-artifacts/ || true + fi + fi + + - name: Upload failing notebook(s) + if: steps.build.outputs.status != 'SUCCEEDED' + uses: actions/upload-artifact@v4 + with: + name: failing-notebooks + path: pr-artifacts/ + if-no-files-found: ignore + + - name: Reflect build result as the check + if: always() + run: | + echo "CodeBuild final status: ${{ steps.build.outputs.status }}" + # Anything other than SUCCEEDED (FAILED / FAULT / TIMED_OUT / STOPPED) + # fails this job -> red PR check. + [ "${{ steps.build.outputs.status }}" = "SUCCEEDED" ]