feat(tenants): prepare verified source upgrades

.github/workflows/ci.yaml

@@ -563,6 +563,7 @@ jobs:
               - 'scripts/canonical-tenant-source.lib.sh'
               - 'scripts/canonical-tenant-source-bindings.json'
               - 'scripts/tests/test-canonical-source-bootstrap.sh'
+              - 'scripts/tests/test-canonical-source-upgrade.sh'
               - 'scripts/tests/test-canonical-constructor-attribution.sh'
               - 'scripts/tests/test-canonical-generator-output.sh'
               - 'scripts/tests/test-canonical-bootstrap-output.sh'
@@ -1402,6 +1403,8 @@ jobs:
           bash scripts/tests/test-bootstrap-tenant-source.sh
           shellcheck -x scripts/canonical-tenant-source.lib.sh scripts/tests/test-canonical-source-bootstrap.sh scripts/tests/test-canonical-bootstrap-output.sh
           bash scripts/tests/test-canonical-source-bootstrap.sh
+          shellcheck -x scripts/tests/test-canonical-source-upgrade.sh
+          bash scripts/tests/test-canonical-source-upgrade.sh
           shellcheck -x scripts/tests/test-canonical-constructor-attribution.sh
           bash scripts/tests/test-canonical-constructor-attribution.sh
           shellcheck -x scripts/tests/test-canonical-generator-output.sh

docs/TENANTS.md

@@ -196,6 +196,24 @@ and inventory, and writes the candidate only after all checks pass. The existing
 `--consumer business-site --verify-only` mode remains evidence-only. Review the
 candidate and registration together; generating a file is not deployed acceptance.
 
+For a later source-only upgrade, use
+`scripts/bootstrap-tenant-source.sh --prepare-canonical-upgrade --tag <tag> --source-sha <sha> --digest <digest> --output /absolute/new-candidate-directory`
+with that same read-only context. The existing source must already be Ready and
+SourceVerified with its admitted digest and exact publisher constraint. The tool
+verifies the proposed publication, checks the source again, and generates two
+reviewable files in a new private directory. It never updates the checkout or
+cluster. Copy those generated files to their matching repository paths for review;
+do not hand-edit the generated admission or change its disabled application flags.
+
+The `published-upgrade` record retains the previous binding and a hashed source
+observation separately from the proposed publication. Discovery projects the new
+desired digest; regeneration preserves the last actual observation while the
+previous source remains verified. It records the replacement only after Flux has
+verified the new digest at its current generation, then removes the transitional
+record. A proposed version is never an applied version. Missing, ambiguous,
+recreated, stale or weakened previous sources fail preparation or regeneration;
+an upgrade never restores first-publication authority or enables an application.
+
 Before first verification, complete inventory may show the exact source absent
 or pending without an artifact. Regeneration emits no applied row for that state.
 Once the uniquely identified source is Ready and SourceVerified at its current

docs/adr/canonical-only-source-bootstrap.md

@@ -46,6 +46,30 @@ It never erases the observation or restores bootstrap authority. Conflicting
 record/observation states fail closed. Historical observation is not current
 readiness, and a failed read is not evidence of absence.
 
+## Source-only upgrades
+
+An existing source cannot use first-admission bootstrap again. An explicitly
+prepared `published-upgrade` binds a separately verified replacement publication
+and retains the previous binding plus a hashed, current source observation.
+Preparation validates complete discovery and the exact source verification
+constraint, observes the source twice, and rechecks the declaration, constructor
+and admission before creating a new private output directory. It never changes
+the checkout or cluster and never expands the approved publisher set.
+
+Raw and rendered discovery project the proposed digest. Until it is applied,
+regeneration accepts only the previously verified source with its recorded UID
+hash and generation, and preserves the previous observation row or its absence.
+It never labels the proposal as applied. Current-generation Ready/SourceVerified
+evidence for the proposed digest permits ordinary regeneration to replace the
+transitional record with `published-binding` and its actual observation row.
+Missing or inconsistent evidence never restores bootstrap authority. Both
+application flags, external DNS and SOPS remain disabled throughout this path.
+
+The maintained preparation command generates the admission and Tenant changes
+together for normal review. A failed output reservation cannot overwrite an
+existing directory or final symlink. Interrupted partial output is not a valid
+preparation result; only a successful complete invocation may be copied for review.
+
 ## Delivery boundary
 
 Source registration uses the normal reviewed production pipeline and fixed

k8s/providers/hetzner/apps/business-site/tenant.yaml

@@ -11,6 +11,6 @@ spec:
   sops: false
   webApp: false
   source:
-    digest: sha256:fa22e5796a39872feb8c75b5d56e2c6d68ff289230e2c5762f0bbfa0b30d6744
+    digest: sha256:2f43bf60c87e373ab99f9fc08f187287d9526b3f1b3212b6234e2ae80b9c5e82
     publisherRevision: 467f07582436a26b62cf4aa756423512f85ba0c0
     reconcileApplication: false

scripts/bootstrap-tenant-source.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
-# Verify one first publication without claiming that Flux has applied it.
-# Approval bootstrap is scoped to Data Products; business-site has evidence-only mode.
+# Verify publications and prepare bounded source-only admission/upgrade candidates.
+# Generation never claims that Flux has applied the proposed publication.
 set -euo pipefail
 BOOTSTRAP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=scripts/generate-publish-workflow-approved-revisions.sh
@@ -94,7 +94,93 @@ canonical_bootstrap_admission() (
   echo 'canonical source admission candidate generated; review required; application reconciliation remains disabled'
 )
 
+canonical_upgrade_preparation() (
+  set -euo pipefail
+  trap - EXIT
+  local upgrade_root="$REPO_ROOT" upgrade_output='' tag='' source_sha='' digest='' seen=':' option
+  local original record receipt first_snapshot second_snapshot today uid_hash snapshot_hash prepared_root prior_node
+  while [ "$#" -gt 0 ]; do
+    option="$1"
+    case "$seen" in *":$option:"*) return 2 ;; esac
+    seen="$seen$option:"
+    [ "$#" -ge 2 ] || return 2
+    case "$option" in
+      --source-root) upgrade_root="$2" ;;
+      --output) upgrade_output="$2" ;;
+      --tag) tag="$2" ;;
+      --source-sha) source_sha="$2" ;;
+      --digest) digest="$2" ;;
+      *) return 2 ;;
+    esac
+    shift 2
+  done
+  [[ "$tag" =~ ^v[0-9]+[.][0-9]+[.][0-9]+$ ]] && is_sha "$source_sha" && is_digest "$digest" || return 2
+  [ -n "${PUBLISH_KUBE_CONTEXT:-}" ] && [[ "$upgrade_output" = /* ]] || return 2
+  local output_parent output_name upgrade_tenant graph_path
+  output_parent="$(dirname "$upgrade_output")"; output_name="$(basename "$upgrade_output")"
+  [[ "$output_name" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]] || return 2
+  output_parent="$(cd "$output_parent" && pwd -P)" || return 2
+  upgrade_output="$output_parent/$output_name"
+  [ ! -e "$upgrade_output" ] && [ ! -L "$upgrade_output" ] || return 1
+  upgrade_root="$(cd "$upgrade_root" && pwd -P)" || return 2
+  upgrade_tenant=k8s/providers/hetzner/apps/business-site/tenant.yaml
+  graph_path=k8s/bases/infrastructure/resource-graph-definitions/tenant/resource-graph-definition.yaml
+  [ -z "${CANONICAL_TENANT_BINDINGS_FILE:-}" ] || return 2
+  original="$(canonical_tenant_bindings "$upgrade_root")" || return 1
+  record="$(printf '%s\n' "$original" | jq -cer '.bindings|select(length==1)|.[0]|select(.kind!="published-upgrade")')" || return 1
+  canonical_observation_consistent "$record" "${APPROVED_IDENTITIES_FILE:-$upgrade_root/scripts/publish-workflow-approved-revisions.identities.tsv}" || return 1
+  prior_node="$(yq -o=json -I=0 '.' "$upgrade_root/$upgrade_tenant" | jq -ces 'select(length==1)|.[0]')" || return 1
+  canonical_tenant_projection "$upgrade_root" "$prior_node" >/dev/null || return 1
+  discover_consumers "$upgrade_root" >/dev/null || return 1
+  prepared_root="$(mktemp -d "$output_parent/.canonical-upgrade.XXXXXX")" || return 1
+  trap 'rm -rf "$prepared_root"' EXIT
+  kubectl --context "$PUBLISH_KUBE_CONTEXT" get ocirepositories.source.toolkit.fluxcd.io -A -o json >"$prepared_root/first-inventory.json" || return 1
+  first_snapshot="$(canonical_source_observation "$record" "$prepared_root/first-inventory.json")" || return 1
+  receipt="$(business_site_manifest_evidence --consumer business-site --verify-only --tag "$tag" --source-sha "$source_sha" --digest "$digest")" || return 1
+  today="${APPROVED_REVISIONS_OBSERVED_ON:-$(date -u +%Y-%m-%d)}"
+  is_date "$today" || return 1
+  uid_hash="$(printf '%s' "$(printf '%s\n' "$first_snapshot" | jq -r .metadata.uid)" | shasum -a 256 | awk '{print $1}')" || return 1
+  snapshot_hash="$(printf '%s\n' "$first_snapshot" | shasum -a 256 | awk '{print $1}')" || return 1
+  mkdir -p "$prepared_root/scripts" "$prepared_root/$(dirname "$upgrade_tenant")" "$prepared_root/$(dirname "$graph_path")"
+  printf '%s\n' "$receipt" | jq -es --argjson old "$record" --arg tag "$tag" --arg source "$source_sha" \
+    --arg digest "$digest" --arg date "$today" --arg uid "$uid_hash" --arg snapshot "$snapshot_hash" --argjson observed "$first_snapshot" '
+    select(length==1)|.[0]|select(.schemaVersion==1 and .kind=="manifest-publication-evidence" and
+      .consumer=="business-site" and .deploymentAuthorized==false and .gitTag==$tag and .sourceSHA==$source and .digest==$digest) |
+    . as $evidence | {schemaVersion:1,bindings:[($old|.kind="published-upgrade"|
+      .previousBinding=$old|.previousObservation={uidSha256:$uid,generation:$observed.metadata.generation,snapshotSha256:$snapshot,observedOn:$date}|
+      .sourceIdentity.digest=$digest|.publication={gitTag:$tag,sourceSHA:$source,publisherIdentity:$evidence.publisherIdentity,
+        callerWorkflow:$evidence.callerWorkflow,event:$evidence.event,workflowRunId:$evidence.workflowRunId,
+        workflowRunAttempt:$evidence.workflowRunAttempt,verifiedOn:$date})]}' >"$prepared_root/scripts/canonical-tenant-source-bindings.json" || return 1
+  cp "$upgrade_root/$graph_path" "$prepared_root/$graph_path"
+  cp "$upgrade_root/scripts/publish-workflow-canonical-approvals.tsv" "$prepared_root/scripts/"
+  cp "${APPROVED_IDENTITIES_FILE:-$upgrade_root/scripts/publish-workflow-approved-revisions.identities.tsv}" "$prepared_root/scripts/"
+  UPGRADE_PREPARED_DIGEST="$digest" yq '.spec.source.digest=strenv(UPGRADE_PREPARED_DIGEST)' "$upgrade_root/$upgrade_tenant" >"$prepared_root/$upgrade_tenant" || return 1
+  # The same strict reader and raw discovery used by CI must accept generated output.
+  APPROVED_IDENTITIES_FILE="$prepared_root/scripts/publish-workflow-approved-revisions.identities.tsv" \
+    CANONICAL_APPROVALS_FILE="$prepared_root/scripts/publish-workflow-canonical-approvals.tsv" \
+    discover_consumers "$prepared_root" >/dev/null || return 1
+  kubectl --context "$PUBLISH_KUBE_CONTEXT" get ocirepositories.source.toolkit.fluxcd.io -A -o json >"$prepared_root/second-inventory.json" || return 1
+  second_snapshot="$(canonical_source_observation "$record" "$prepared_root/second-inventory.json")" || return 1
+  [ "$first_snapshot" = "$second_snapshot" ] || return 1
+  [ "$(canonical_tenant_bindings "$upgrade_root")" = "$original" ] || return 1
+  [ "$(yq -o=json -I=0 '.' "$upgrade_root/$upgrade_tenant" | jq -ces 'select(length==1)|.[0]')" = "$prior_node" ] || return 1
+  canonical_tenant_projection "$upgrade_root" "$prior_node" >/dev/null || return 1
+  discover_consumers "$upgrade_root" >/dev/null || return 1
+  # Reserve a new private output directory only after every input has been validated.
+  # Exclusive links cannot overwrite a conflicting file or follow its final symlink.
+  (umask 077; mkdir "$upgrade_output") || return 1
+  mkdir -p "$upgrade_output/scripts" "$upgrade_output/$(dirname "$upgrade_tenant")" || return 1
+  link "$prepared_root/scripts/canonical-tenant-source-bindings.json" "$upgrade_output/scripts/canonical-tenant-source-bindings.json" || return 1
+  link "$prepared_root/$upgrade_tenant" "$upgrade_output/$upgrade_tenant" || return 1
+  printf 'Canonical source upgrade prepared for review; existing observations and disabled application flags preserved.\n'
+)
+
 bootstrap_tenant_source() (
+  if [ "${1:-}" = --prepare-canonical-upgrade ]; then
+    shift
+    canonical_upgrade_preparation "$@"
+    return
+  fi
   if [ "${1:-}" = --admit-canonical-source ]; then
     shift
     canonical_bootstrap_admission "$@"

scripts/canonical-tenant-source-bindings.json

@@ -5,7 +5,7 @@
       "consumer": "business-site",
       "consumerRepository": "devantler-tech/business-site",
       "workflow": "publish-app",
-      "kind": "published-binding",
+      "kind": "published-upgrade",
       "sourcePath": "k8s/providers/hetzner/apps/business-site/tenant.yaml",
       "constructorSpecSha256": "16939512acb4d4dccb1ebe11caa3f48fe49503c8a9a8a4c44aafd328a5a45fdb",
       "tenantIdentity": {
@@ -20,17 +20,55 @@
         "namespace": "business-site",
         "name": "business-site",
         "artifactURL": "oci://ghcr.io/devantler-tech/business-site/manifests",
-        "digest": "sha256:fa22e5796a39872feb8c75b5d56e2c6d68ff289230e2c5762f0bbfa0b30d6744"
+        "digest": "sha256:2f43bf60c87e373ab99f9fc08f187287d9526b3f1b3212b6234e2ae80b9c5e82"
       },
       "publication": {
-        "gitTag": "v1.0.3",
-        "sourceSHA": "498248e86e49024b72f3249b518e4dd1cef0104b",
+        "gitTag": "v2.0.0",
+        "sourceSHA": "2eca5f823676adde374556848cf8bf86eb139804",
         "publisherIdentity": "devantler-tech/.github/.github/workflows/publish-app.yaml@467f07582436a26b62cf4aa756423512f85ba0c0",
         "callerWorkflow": ".github/workflows/publish-runtime.yaml",
         "event": "release",
-        "workflowRunId": 39921694859,
+        "workflowRunId": 40247127226,
         "workflowRunAttempt": 1,
-        "verifiedOn": "2026-10-25"
+        "verifiedOn": "2026-10-26"
+      },
+      "previousBinding": {
+        "consumer": "business-site",
+        "consumerRepository": "devantler-tech/business-site",
+        "workflow": "publish-app",
+        "kind": "published-binding",
+        "sourcePath": "k8s/providers/hetzner/apps/business-site/tenant.yaml",
+        "constructorSpecSha256": "16939512acb4d4dccb1ebe11caa3f48fe49503c8a9a8a4c44aafd328a5a45fdb",
+        "tenantIdentity": {
+          "apiVersion": "kro.run/v1alpha1",
+          "kind": "Tenant",
+          "namespace": "flux-system",
+          "name": "business-site"
+        },
+        "sourceIdentity": {
+          "apiVersion": "source.toolkit.fluxcd.io/v1",
+          "kind": "OCIRepository",
+          "namespace": "business-site",
+          "name": "business-site",
+          "artifactURL": "oci://ghcr.io/devantler-tech/business-site/manifests",
+          "digest": "sha256:fa22e5796a39872feb8c75b5d56e2c6d68ff289230e2c5762f0bbfa0b30d6744"
+        },
+        "publication": {
+          "gitTag": "v1.0.3",
+          "sourceSHA": "498248e86e49024b72f3249b518e4dd1cef0104b",
+          "publisherIdentity": "devantler-tech/.github/.github/workflows/publish-app.yaml@467f07582436a26b62cf4aa756423512f85ba0c0",
+          "callerWorkflow": ".github/workflows/publish-runtime.yaml",
+          "event": "release",
+          "workflowRunId": 39921694859,
+          "workflowRunAttempt": 1,
+          "verifiedOn": "2026-10-25"
+        }
+      },
+      "previousObservation": {
+        "uidSha256": "38aff5c8aafbd89fc51dbbe131d27c9a3fb0e78108a20779d0e7a5680a6cc2d2",
+        "generation": 1,
+        "snapshotSha256": "610500b54e61b2d3e33cc2db548971e04816e05231e2df50dd6f255ddd92600e",
+        "observedOn": "2026-10-26"
       }
     }
   ]