Skip to content

fix(wedding-db): correct pre-loss recovery witness #3742

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
devantler merged 1 commit into from
Sep 12, 2026
Merged

fix(wedding-db): correct pre-loss recovery witness #3742

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions scripts/recover-wedding-db-incident.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,30 @@ const PRELOSS_BACKUP_UID='549fe940-b119-4010-bdc8-fa8e6ebc93ae';
const PRELOSS_CLUSTER_UID='6b6d4879-e437-4ea5-a0cb-257de8edad00';
const PRELOSS_TARGET_TIME='2026-09-09T01:57:41Z';
const PRELOSS_TARGET_TIMELINE='162';
const PRELOSS_RECONCILIATION_DIGEST='sha256:022128434868723705c489546f68ba344e9cbe9e5c2b930a404d8aa2122ad9c7';
const PRELOSS_RECONCILIATION_DIGEST='sha256:022128434868723705c489546f68ba344a9cbe9e5c2b930a404d8aa2122ad9c7';
const RECOVERY_MODE='point in time 2026-09-09T01:57:41Z from backup 20260908T030001 on timeline 162';
const PROOF='wedding-db-data-recovery-proof';
const RECOVERY_OWNER_ANNOTATION='devantler.tech/wedding-db-recovery-owner';
const RECOVERY_RECONCILE_ANNOTATION='kustomize.toolkit.fluxcd.io/reconcile';
const RECOVERY_OWNER_PATH='/metadata/annotations/devantler.tech~1wedding-db-recovery-owner';
const RECOVERY_RECONCILE_PATH='/metadata/annotations/kustomize.toolkit.fluxcd.io~1reconcile';
const RECOVERY_REFUSAL_PHASES=new Set(['source-state','before-backup','restore-and-merge','after-backup','proof','cleanup']);
let recoveryInvocationVerified=false;
let recoveryPhase='invocation';
const check=value=>{if(!value)throw Error('refused');return value;};
const integer=value=>{check(typeof value==='string'&&/^[1-9][0-9]*$/.test(value));return value;};
const uid=value=>{check(typeof value==='string'&&/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/.test(value));return value;};
const timestamp=value=>{check(typeof value==='string'&&!Number.isNaN(Date.parse(value)));return value;};
const condition=(value,type)=>value.status?.conditions?.some(item=>item.type===type&&item.status==='True');

export function recoveryRefusalMessage({verified=false,phase='invocation'}={}){
return verified&&RECOVERY_REFUSAL_PHASES.has(phase)?`Wedding database incident recovery refused (phase: ${phase}).\n`:'Wedding database incident recovery refused.\n';
}

export function hasPrelossWitness(kustomization){
return kustomization.status?.history?.some(item=>item.lastReconciled===PRELOSS_TARGET_TIME&&item.lastReconciledStatus==='ReconciliationSucceeded'&&item.digest===PRELOSS_RECONCILIATION_DIGEST&&item.metadata?.originRevision==='v1.15.11@sha1:5f0f5be0a228ee189ea3d10e4bd1b61ef0a8efe9')===true;
}

function exactObject(value,{apiVersion,kind,name}){
check(value?.apiVersion===apiVersion&&value.kind===kind);
check(value.metadata?.name===name&&value.metadata.namespace===NAMESPACE&&!value.metadata.deletionTimestamp);
Expand Down Expand Up @@ -306,7 +317,7 @@ function sourceState(){
const kustomization=object('kustomizations.kustomize.toolkit.fluxcd.io',LIVE_KUSTOMIZATION);
check(kustomization.metadata.uid===LIVE_KUSTOMIZATION_UID&&kustomization.metadata.creationTimestamp==='2026-05-23T01:52:44Z');
check(kustomization.spec?.force===false&&kustomization.spec.suspend!==true&&condition(kustomization,'Ready'));
check(kustomization.status?.history?.some(item=>item.lastReconciled===PRELOSS_TARGET_TIME&&item.lastReconciledStatus==='ReconciliationSucceeded'&&item.digest===PRELOSS_RECONCILIATION_DIGEST&&item.metadata?.originRevision==='v1.15.11@sha1:5f0f5be0a228ee189ea3d10e4bd1b61ef0a8efe9'));
check(hasPrelossWitness(kustomization));
const appPolicy=object('ciliumnetworkpolicies.cilium.io','app');
check(appPolicy.metadata.uid===LIVE_APP_POLICY_UID&&appPolicy.metadata.creationTimestamp==='2026-06-16T18:14:37Z');
check(appPolicy.spec?.endpointSelector&&Object.keys(appPolicy.spec.endpointSelector).length===0);
Expand Down Expand Up @@ -582,8 +593,10 @@ function recordProof({config,source,recovery,before,after,recovered,liveBefore,m
}

function run(){
const config=verifyCandidate(),source=sourceState();
const config=verifyCandidate();recoveryInvocationVerified=true;recoveryPhase='source-state';
const source=sourceState();recoveryPhase='before-backup';
const before=backup(config,'before',source);
recoveryPhase='restore-and-merge';
let recovery,recoveredInventory,recovered,liveBefore,merge,mergePrimary,error,schemaCleanupError,resumeError,cleanupError;
try{
recovery=createRecovery(config,source);
Expand All @@ -608,13 +621,15 @@ function run(){
try{resumeApplication(config);}catch(candidate){resumeError=candidate;}
try{cleanupRecovery(config,recovery);}catch(candidate){cleanupError=candidate;}
if(error||schemaCleanupError||resumeError||cleanupError)throw Error('refused');
recoveryPhase='after-backup';
const after=backup(config,'after',source);
recoveryPhase='proof';
recordProof({config,source,recovery,before,after,recovered,liveBefore,merge});
return {restored:true,currentClusterUid:source.currentClusterUid,prelossBackupUid:source.prelossBackupUid,recoveryClusterUid:recovery.uid,beforeBackupUid:before.uid,afterBackupUid:after.uid,recovered,liveBefore,merge,cleanup:true};
}

function runCleanup(){
const config=verifyCandidate();
const config=verifyCandidate();recoveryInvocationVerified=true;recoveryPhase='cleanup';
let schemaCleanupError,resumeError,cleanupError,resumed=false,schemaCleaned=false;
const kustomization=object('kustomizations.kustomize.toolkit.fluxcd.io',LIVE_KUSTOMIZATION);
const owner=kustomization.metadata?.annotations?.[RECOVERY_OWNER_ANNOTATION];
Expand Down Expand Up @@ -642,5 +657,5 @@ if(process.argv[1]===fileURLToPath(import.meta.url)){
check(process.argv.length===2||(process.argv.length===3&&['--cleanup','--verify-source'].includes(process.argv[2])));
const result=process.argv[2]==='--verify-source'?(verifyCandidate(),{sourceVerified:true}):process.argv[2]==='--cleanup'?runCleanup():run();
process.stdout.write(JSON.stringify(result)+'\n');
}catch{process.stderr.write('Wedding database incident recovery refused.\n');process.exitCode=2;}
}catch{process.stderr.write(recoveryRefusalMessage({verified:recoveryInvocationVerified,phase:recoveryPhase}));process.exitCode=2;}
}
20 changes: 20 additions & 0 deletions tests/wedding-db-incident-recovery/recovery.test.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import {
buildResumePatch,
buildSchemaCleanupSQL,
buildSuspendPatch,
hasPrelossWitness,
recoveryRefusalMessage,
recoveryOwner,
recoveryOwnerAttempt,
recoverySource,
Expand All @@ -20,6 +22,24 @@ const SOURCE_UID='9cd2ba9b-c7bf-43e2-bd2d-a5c7c139fafb';
const BACKUP_UID='549fe940-b119-4010-bdc8-fa8e6ebc93ae';
const CONTROLLER_UID='48c6521a-483a-4de9-895a-bae1a61ea25e';

test('pre-loss reconciliation witness matches the live Flux history exactly',()=>{
const history=[{
lastReconciled:'2026-09-09T01:57:41Z',
lastReconciledStatus:'ReconciliationSucceeded',
digest:'sha256:022128434868723705c489546f68ba344a9cbe9e5c2b930a404d8aa2122ad9c7',
metadata:{originRevision:'v1.15.11@sha1:5f0f5be0a228ee189ea3d10e4bd1b61ef0a8efe9'},
}];
assert.equal(hasPrelossWitness({status:{history}}),true);
history[0].digest='sha256:022128434868723705c489546f68ba344e9cbe9e5c2b930a404d8aa2122ad9c7';
assert.equal(hasPrelossWitness({status:{history}}),false);
});

test('verified recovery failures disclose only an allow-listed phase',()=>{
assert.equal(recoveryRefusalMessage(), 'Wedding database incident recovery refused.\n');
assert.equal(recoveryRefusalMessage({verified:true,phase:'source-state'}), 'Wedding database incident recovery refused (phase: source-state).\n');
assert.equal(recoveryRefusalMessage({verified:true,phase:'subprocess stderr'}), 'Wedding database incident recovery refused.\n');
});

function fixture(){
return {
cluster:{apiVersion:'postgresql.cnpg.io/v1',kind:'Cluster',metadata:{name:'wedding-db',namespace:'wedding-app',uid:CURRENT_UID,creationTimestamp:'2026-09-09T01:58:14Z',generation:6},spec:{
Expand Down
Loading