From be5348585dfa4e0961a675afd2af46787c86c0e0 Mon Sep 17 00:00:00 2001 From: yirenz Date: Wed, 7 Oct 2026 16:10:49 +0000 Subject: [PATCH] Guard FlatKV rollback floor (#4496) Guard FlatKV's retained-history floor before composite rollback mutates any backend, and treat stale idle FlatKV directories as disposable Auto-mode seeds instead of reconcilable state. - `sei-db/state_db/sc/flatkv/snapshot.go`: adds a single read-only reachability check for snapshots plus WAL. Rollback still uses the same check before any mutation. Only "no snapshot at or below the target" is tagged `ErrVersionUnreachable`; filesystem errors from the snapshot scan pass through untagged. - `sei-db/state_db/sc/flatkv/store.go`: rejects unreachable historical read-only loads before creating a `readonly-*` working directory. - `sei-db/state_db/giga/types/live_state_store.go`: exposes `CheckVersionReachable` so composite rollback can preflight the FlatKV backend, and owns its error contract, `ErrVersionUnreachable` (the only non-fatal error of the interface). FlatKV keeps `flatkv.ErrVersionUnreachable` as an alias. - `sei-db/state_db/sc/composite/store.go`: refuses rollback targets FlatKV cannot reach before memIAVL moves, and points operators to state sync. Before seeding or reconciliation, it removes an idle Auto-mode FlatKV dir whose seed is above memIAVL or more than one block below it. An idle seed exactly one block behind is kept for `reconcileVersions`, because that is what a crash between the memIAVL and FlatKV commits of the kickoff block leaves. - `sei-db/state_db/sc/composite/store_test.go`: updates the test fake for the new interface. - `sei-db/state_db/sc/flatkv/store_test.go`: updates the beyond-WAL read-only test for the new early reachability failure. - `sei-db/state_db/sc/composite/rollback_floor_test.go`: covers rollback to K-2 refusal without mutation, historical read/export below the floor returning errors without panic, rollback to K-1 followed by re-kickoff, delayed kickoff after a stale idle FlatKV dir, a torn kickoff commit (memIAVL at K, FlatKV seed at K-1) that must roll memIAVL back and replay K to the canonical AppHash, and state sync from a pre-kickoff snapshot with replay through migration. - `sei-db/state_db/sc/flatkv/snapshot_test.go`: an empty snapshot dir is `ErrVersionUnreachable`; a snapshot-scan read failure is not. - `sei-db/state_db/sc/flatkv/store_replay_test.go`: covers snapshot-only reachability, snapshot-plus-WAL reachability, below-history errors, and failing read-only loads before creating a read-only work dir. - `go test ./sei-db/state_db/sc/flatkv ./sei-db/state_db/sc/composite` - `make fmtcheck` - `make dblint` - `go test -race ./sei-db/state_db/sc/composite/... ./sei-db/state_db/sc/flatkv/... ./sei-db/state_db/giga/... ./sei-db/bootstrap/...` - `scripts/ramtest.sh ./sei-db/...` failed once in unrelated `TestRapidFlushes` with "Expected at most 25 flushes, got 32"; rerunning `scripts/ramtest.sh ./sei-db/db_engine/litt/disktable` passed. --------- Co-authored-by: Cursor (cherry picked from commit 8b5e550e9c5de26a0e0907f9bcc422e446cdc3a3) --- .../sc/composite/rollback_floor_test.go | 323 ++++++++++++++++++ sei-db/state_db/sc/composite/store.go | 68 +++- sei-db/state_db/sc/composite/store_test.go | 1 + sei-db/state_db/sc/flatkv/api.go | 9 + sei-db/state_db/sc/flatkv/snapshot.go | 51 +-- sei-db/state_db/sc/flatkv/snapshot_test.go | 14 +- sei-db/state_db/sc/flatkv/store.go | 5 + .../state_db/sc/flatkv/store_replay_test.go | 47 +++ sei-db/state_db/sc/flatkv/store_test.go | 4 +- 9 files changed, 493 insertions(+), 29 deletions(-) create mode 100644 sei-db/state_db/sc/composite/rollback_floor_test.go diff --git a/sei-db/state_db/sc/composite/rollback_floor_test.go b/sei-db/state_db/sc/composite/rollback_floor_test.go new file mode 100644 index 0000000000..91337859d1 --- /dev/null +++ b/sei-db/state_db/sc/composite/rollback_floor_test.go @@ -0,0 +1,323 @@ +package composite + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" +) + +const ( + rollbackFloorPreBlocks = 8 + rollbackFloorKickoff = rollbackFloorPreBlocks + 1 + rollbackFloorBatch = 4 + + // noKickoff is a kickoff height past every block a test commits, so those blocks stay memiavl-only. + noKickoff = rollbackFloorKickoff + 100 +) + +func openRollbackFloorStore( + t *testing.T, dir string, mode types.WriteMode, batch int, +) *CompositeCommitStore { + t.Helper() + return openCompositeForRollback(t, dir, mode, batch, rollbackSnapSettings{ + memiavlInterval: 2, + flatkvInterval: 2, + }) +} + +func buildFixedModeMigrationStore(t *testing.T, dir string) *CompositeCommitStore { + t.Helper() + workload := newMigrationWorkload(0x5702) + + cs := openRollbackFloorStore(t, dir, types.MemiavlOnly, 0) + for i := 0; i < rollbackFloorPreBlocks; i++ { + require.NoError(t, cs.ApplyChangeSets(workload.generateBlock(30, 0, 0, 5, 0))) + _, err := cs.Commit() + require.NoError(t, err) + } + require.NoError(t, cs.Close()) + + cs = openRollbackFloorStore(t, dir, types.MigrateEVM, rollbackFloorBatch) + for i := 0; i < 6; i++ { + require.NoError(t, cs.ApplyChangeSets(workload.generateBlock(8, 4, 1, 2, 1))) + _, err := cs.Commit() + require.NoError(t, err) + } + return cs +} + +func TestCompositeRollbackBelowFlatKVFloorIsRefusedBeforeMutation(t *testing.T) { + dir := t.TempDir() + cs := buildFixedModeMigrationStore(t, dir) + + target := int64(rollbackFloorKickoff - 2) + latest := cs.Version() + require.Equal(t, latest, cs.memIAVL.Version()) + require.Equal(t, latest, cs.loadFlatKV().Version()) + + err := cs.Rollback(target) + require.Error(t, err) + require.ErrorIs(t, err, flatkv.ErrVersionUnreachable) + require.Contains(t, err.Error(), "use state sync") + require.Equal(t, latest, cs.memIAVL.Version()) + require.Equal(t, latest, cs.loadFlatKV().Version()) + require.NoError(t, cs.Close()) + + reopened := openRollbackFloorStore(t, dir, types.MigrateEVM, rollbackFloorBatch) + require.Equal(t, latest, reopened.Version()) + require.Equal(t, latest, reopened.memIAVL.Version()) + require.Equal(t, latest, reopened.loadFlatKV().Version()) + require.NoError(t, reopened.Close()) +} + +func TestCompositeHistoricalReadBelowFlatKVFloorErrorsWithoutPanic(t *testing.T) { + dir := t.TempDir() + cs := buildFixedModeMigrationStore(t, dir) + defer func() { require.NoError(t, cs.Close()) }() + + target := int64(rollbackFloorKickoff - 2) + _, err := cs.LoadVersionReadOnly(target) + require.Error(t, err) + require.ErrorIs(t, err, flatkv.ErrVersionUnreachable) + + _, err = cs.Exporter(target) + require.Error(t, err) + require.ErrorIs(t, err, flatkv.ErrVersionUnreachable) + + for _, reachable := range []int64{rollbackFloorKickoff - 1, rollbackFloorKickoff, cs.Version()} { + ro, err := cs.LoadVersionReadOnly(reachable) + require.NoError(t, err, "version %d must stay reachable", reachable) + require.NoError(t, ro.Close()) + } +} + +func rollbackFloorBlocks(seed int64, last int64) map[int64][]*proto.NamedChangeSet { + w := newMigrationWorkload(seed) + blocks := make(map[int64][]*proto.NamedChangeSet, last) + for h := int64(1); h <= last; h++ { + blocks[h] = w.generateBlock(5, 5, 1, 2, 2) + } + return blocks +} + +func beginRollbackFloorBlock(t *testing.T, cs *CompositeCommitStore, height int64, kickoff int64) { + t.Helper() + batch := 0 + if height >= kickoff { + batch = rollbackFloorBatch + } + require.NoError(t, cs.SetMigrationBatchSize(batch)) + if batch > 0 && cs.loadWriteMode() == types.MemiavlOnly { + require.NoError(t, cs.SetWriteMode(types.MigrateEVM)) + } +} + +func commitRollbackFloorBlock( + t *testing.T, + cs *CompositeCommitStore, + blocks map[int64][]*proto.NamedChangeSet, + height int64, + kickoff int64, +) { + t.Helper() + require.Equal(t, height-1, cs.Version()) + beginRollbackFloorBlock(t, cs, height, kickoff) + require.NoError(t, cs.ApplyChangeSets(cloneChangeSets(t, blocks[height]))) + v, err := cs.Commit() + require.NoError(t, err) + require.Equal(t, height, v) +} + +func runAutoMigration( + t *testing.T, + dir string, + cfg config.StateCommitConfig, + blocks map[int64][]*proto.NamedChangeSet, + last int64, + kickoff int64, +) (map[int64]*proto.CommitInfo, *CompositeCommitStore) { + t.Helper() + cs := openAutoStoreWithConfig(t, dir, cfg, 0) + canonical := make(map[int64]*proto.CommitInfo, last) + for h := int64(1); h <= last; h++ { + commitRollbackFloorBlock(t, cs, blocks, h, kickoff) + canonical[h] = cloneCommitInfo(cs.LastCommitInfo()) + } + return canonical, cs +} + +func TestCompositeAutoRollbackToSeededFloorCanKickoffAgain(t *testing.T) { + const last = int64(14) + blocks := rollbackFloorBlocks(0x5703, last) + cfg := autoExportConfig() + cfg.FlatKVConfig.SnapshotKeepRecent = 100 + + canonical, src := runAutoMigration(t, t.TempDir(), cfg, blocks, last, rollbackFloorKickoff) + require.NoError(t, src.Close()) + + dir := t.TempDir() + _, cs := runAutoMigration(t, dir, cfg, blocks, last, rollbackFloorKickoff) + require.NoError(t, cs.Rollback(rollbackFloorKickoff-1)) + require.Equal(t, int64(rollbackFloorKickoff-1), cs.Version()) + + for h := int64(rollbackFloorKickoff); h <= last; h++ { + commitRollbackFloorBlock(t, cs, blocks, h, rollbackFloorKickoff) + requireCommitInfoEqual(t, canonical[h], cs.LastCommitInfo(), fmt.Sprintf("replayed height %d", h)) + } + require.NoError(t, cs.Close()) +} + +func TestCompositeAutoDiscardStaleIdleFlatKV(t *testing.T) { + dir := t.TempDir() + cfg := autoExportConfig() + cfg.FlatKVConfig.SnapshotKeepRecent = 100 + blocks := rollbackFloorBlocks(0x5704, 13) + + cs := openAutoStoreWithConfig(t, dir, cfg, 0) + for h := int64(1); h < rollbackFloorKickoff; h++ { + commitRollbackFloorBlock(t, cs, blocks, h, noKickoff) + } + + beginRollbackFloorBlock(t, cs, rollbackFloorKickoff, rollbackFloorKickoff) + require.NotNil(t, cs.loadFlatKV()) + require.NoError(t, cs.ApplyChangeSets(cloneChangeSets(t, blocks[rollbackFloorKickoff]))) + require.NoError(t, cs.Close()) + + cs = openAutoStoreWithConfig(t, dir, cfg, 0) + require.Equal(t, int64(rollbackFloorKickoff-1), cs.Version()) + require.Nil(t, cs.loadFlatKV()) + for h := int64(rollbackFloorKickoff); h <= rollbackFloorKickoff+3; h++ { + commitRollbackFloorBlock(t, cs, blocks, h, noKickoff) + } + require.NoError(t, cs.Close()) + + cs = openAutoStoreWithConfig(t, dir, cfg, 0) + require.Equal(t, int64(rollbackFloorKickoff+3), cs.Version()) + require.Nil(t, cs.loadFlatKV()) + require.False(t, utils.DirExists(utils.GetFlatKVPath(dir))) + + commitRollbackFloorBlock(t, cs, blocks, rollbackFloorKickoff+4, rollbackFloorKickoff+4) + require.NotNil(t, cs.loadFlatKV()) + require.Equal(t, int64(rollbackFloorKickoff+4), cs.Version()) + require.NoError(t, cs.Close()) +} + +// A crash between the memIAVL and flatkv commits of the kickoff block leaves memIAVL at K and an idle +// flatkv at its seed K-1. That is a torn commit, not a stale seed: the restart must keep flatkv, roll +// memIAVL back to K-1, and replay K to the canonical AppHash. +func TestCompositeAutoTornKickoffCommitReplaysKickoffBlock(t *testing.T) { + const last = int64(rollbackFloorKickoff + 3) + blocks := rollbackFloorBlocks(0x5705, last) + cfg := autoExportConfig() + cfg.FlatKVConfig.SnapshotKeepRecent = 100 + + canonical, src := runAutoMigration(t, t.TempDir(), cfg, blocks, last, rollbackFloorKickoff) + require.NoError(t, src.Close()) + require.True(t, hasLattice(canonical[rollbackFloorKickoff])) + + dir := t.TempDir() + cs := openAutoStoreWithConfig(t, dir, cfg, 0) + for h := int64(1); h < rollbackFloorKickoff; h++ { + commitRollbackFloorBlock(t, cs, blocks, h, rollbackFloorKickoff) + } + beginRollbackFloorBlock(t, cs, rollbackFloorKickoff, rollbackFloorKickoff) + require.NoError(t, cs.ApplyChangeSets(cloneChangeSets(t, blocks[rollbackFloorKickoff]))) + committed, err := cs.memIAVL.Commit() + require.NoError(t, err) + require.Equal(t, int64(rollbackFloorKickoff), committed) + require.Equal(t, int64(rollbackFloorKickoff-1), cs.loadFlatKV().Version()) + require.NoError(t, cs.Close()) + + cs = openAutoStoreWithConfig(t, dir, cfg, 0) + require.Equal(t, int64(rollbackFloorKickoff-1), cs.Version(), "memIAVL must be rolled back to the seed") + require.True(t, utils.DirExists(utils.GetFlatKVPath(dir)), "a torn kickoff commit must keep flatkv") + + for h := int64(rollbackFloorKickoff); h <= last; h++ { + commitRollbackFloorBlock(t, cs, blocks, h, rollbackFloorKickoff) + requireCommitInfoEqual(t, canonical[h], cs.LastCommitInfo(), fmt.Sprintf("replayed height %d", h)) + } + require.NoError(t, cs.Close()) +} + +func TestIsStaleSeed(t *testing.T) { + require.False(t, isStaleSeed(9, 9), "matching versions") + require.False(t, isStaleSeed(10, 9), "one block behind is a torn kickoff commit") + require.True(t, isStaleSeed(11, 9), "more than one block behind") + require.True(t, isStaleSeed(5, 9), "seed above memIAVL") +} + +func TestCompositeAutoPreKickoffSnapshotReplaysThroughMigration(t *testing.T) { + const ( + snapshotHeight = int64(5) + kickoffHeight = int64(10) + lastHeight = int64(20) + ) + blocks := rollbackFloorBlocks(0xC0FFEE, lastHeight) + cfg := autoExportConfig() + cfg.FlatKVConfig.SnapshotKeepRecent = 100 + + src := openAutoStoreWithConfig(t, t.TempDir(), cfg, 0) + canonical := make(map[int64]*proto.CommitInfo, lastHeight) + var items []exportedItem + for h := int64(1); h <= lastHeight; h++ { + commitRollbackFloorBlock(t, src, blocks, h, kickoffHeight) + canonical[h] = cloneCommitInfo(src.LastCommitInfo()) + if h == snapshotHeight { + exp, err := src.Exporter(h) + require.NoError(t, err) + items = drainCompositeExporter(t, exp) + require.NoError(t, exp.Close()) + } + } + require.NotContains(t, moduleNamesOf(items), keys.FlatKVStoreKey) + require.False(t, hasLattice(canonical[kickoffHeight-1])) + require.True(t, hasLattice(canonical[kickoffHeight])) + require.NoError(t, src.Close()) + + cases := []struct { + name string + crashInKickoffBlock bool + }{ + {name: "clean", crashInKickoffBlock: false}, + {name: "crash_in_kickoff_block", crashInKickoffBlock: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + dst := openAutoStoreWithConfig(t, dir, cfg, 0) + require.NoError(t, dst.Close()) + imp, err := dst.Importer(snapshotHeight) + require.NoError(t, err) + replayImport(t, imp, items) + require.NoError(t, imp.Close()) + require.Nil(t, dst.loadFlatKV()) + + dst = openAutoStoreWithConfig(t, dir, cfg, 0) + require.Equal(t, snapshotHeight, dst.Version()) + require.Equal(t, types.MemiavlOnly, dst.loadWriteMode()) + + for h := snapshotHeight + 1; h <= lastHeight; h++ { + if tc.crashInKickoffBlock && h == kickoffHeight { + beginRollbackFloorBlock(t, dst, h, kickoffHeight) + require.NotNil(t, dst.loadFlatKV()) + require.NoError(t, dst.ApplyChangeSets(cloneChangeSets(t, blocks[h]))) + require.NoError(t, dst.Close()) + dst = openAutoStoreWithConfig(t, dir, cfg, 0) + require.Equal(t, h-1, dst.Version()) + require.Nil(t, dst.loadFlatKV()) + } + commitRollbackFloorBlock(t, dst, blocks, h, kickoffHeight) + requireCommitInfoEqual(t, canonical[h], dst.LastCommitInfo(), fmt.Sprintf("height %d", h)) + } + require.NoError(t, dst.Close()) + }) + } +} diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index 66393e6d8a..83527885d0 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "math" + "os" "sync/atomic" ics23 "github.com/confio/ics23/go" @@ -340,6 +341,9 @@ func (cs *CompositeCommitStore) LoadLatest() error { return fmt.Errorf("failed to load FlatKV: %w", err) } } + if err := cs.discardStaleIdleFlatKV(); err != nil { + return err + } if cs.memIAVL != nil && cs.loadFlatKV() != nil { // Migration-entry seeding: turning on a non-MemiavlOnly mode on a chain that has been running on @@ -365,6 +369,47 @@ func (cs *CompositeCommitStore) LoadLatest() error { return cs.adoptPersistedWriteMode() } +// discardStaleIdleFlatKV removes an Auto-mode FlatKV directory that has no migration state and whose seed +// no longer matches memIAVL: it is above memIAVL, or more than one block below it. Such a directory holds +// only a seed, so the next migration kickoff recreates it exactly. +func (cs *CompositeCommitStore) discardStaleIdleFlatKV() error { + if cs.config.WriteMode != types.Auto || cs.memIAVL == nil || cs.loadFlatKV() == nil { + return nil + } + derived, err := migration.DeriveWriteMode(cs.loadFlatKV()) + if err != nil { + return fmt.Errorf("failed to derive write mode before idle flatkv cleanup: %w", err) + } + if derived != types.MemiavlOnly || !isStaleSeed(cs.memIAVL.Version(), cs.loadFlatKV().Version()) { + return nil + } + + flatKVDir := utils.GetFlatKVPath(cs.homeDir) + logger.Warn("discarding stale idle flatkv directory", + "memiavlVersion", cs.memIAVL.Version(), + "flatkvVersion", cs.loadFlatKV().Version(), + "flatkvDir", flatKVDir) + if err := cs.loadFlatKV().Close(); err != nil { + return fmt.Errorf("failed to close stale idle flatkv: %w", err) + } + cs.storeFlatKV(nil) + if err := os.RemoveAll(flatKVDir); err != nil { + return fmt.Errorf("failed to remove stale idle flatkv directory %q: %w", flatKVDir, err) + } + return nil +} + +// isStaleSeed reports whether an idle flatkv seeded at flatKVVersion is stale against memIAVL at +// memIAVLVersion. +// +// Exactly one block behind is not stale. It is what a crash between the memIAVL and flatkv commits of +// the kickoff block leaves: memIAVL holds block K and flatkv still holds only its seed at K-1, because +// the migration boundary is first written by flatkv's K commit. Discarding flatkv there would lose +// block K's flatkv half, so reconcileVersions must roll memIAVL back to the seed and let K replay. +func isStaleSeed(memIAVLVersion, flatKVVersion int64) bool { + return flatKVVersion > memIAVLVersion || memIAVLVersion-flatKVVersion > 1 +} + // LoadVersionReadOnly returns an isolated read-only composite view at targetVersion (0 = latest). This store // is left untouched and must stay open while the view is in use; the caller owns the view and must Close it. func (cs *CompositeCommitStore) LoadVersionReadOnly(targetVersion int64) (_ types.Committer, retErr error) { @@ -883,12 +928,14 @@ func (cs *CompositeCommitStore) reconcileVersions() error { if cosmosVer > minVer { if err := cs.memIAVL.Rollback(minVer); err != nil { - return fmt.Errorf("failed to rollback cosmos to reconciled version %d: %w", minVer, err) + return fmt.Errorf("failed to rollback cosmos to reconciled version %d: %w; use state sync instead", + minVer, err) } } if evmVer > minVer { if err := cs.loadFlatKV().Rollback(minVer); err != nil { - return fmt.Errorf("failed to rollback EVM to reconciled version %d: %w", minVer, err) + return fmt.Errorf("failed to rollback EVM to reconciled version %d: %w; use state sync instead", + minVer, err) } } @@ -1237,6 +1284,10 @@ func (cs *CompositeCommitStore) ReleaseSnapshotRefs() error { // Rollback rolls back to the specified version func (cs *CompositeCommitStore) Rollback(targetVersion int64) error { + if err := cs.requireRollbackReachable(targetVersion); err != nil { + return err + } + if cs.memIAVL != nil { if err := cs.memIAVL.Rollback(targetVersion); err != nil { return fmt.Errorf("failed to rollback cosmos commit store: %w", err) @@ -1280,6 +1331,19 @@ func (cs *CompositeCommitStore) Rollback(targetVersion int64) error { return nil } +// requireRollbackReachable refuses a rollback before any backend moves when FlatKV cannot reconstruct +// the target version from its retained history. +func (cs *CompositeCommitStore) requireRollbackReachable(targetVersion int64) error { + if cs.loadFlatKV() == nil { + return nil + } + if err := cs.loadFlatKV().CheckVersionReachable(targetVersion); err != nil { + return fmt.Errorf("flatkv cannot roll back to version %d: %w; use state sync instead", + targetVersion, err) + } + return nil +} + // exportNeedsMetadataGating reports whether the configured mode allows // backend presence to diverge from AppHash participation at some version, // in which case the exporter must consult migration metadata as-of the diff --git a/sei-db/state_db/sc/composite/store_test.go b/sei-db/state_db/sc/composite/store_test.go index dc7f1f9d59..a5cb2c8282 100644 --- a/sei-db/state_db/sc/composite/store_test.go +++ b/sei-db/state_db/sc/composite/store_test.go @@ -33,6 +33,7 @@ func (f *failingEVMStore) LoadLatest() error { return fmt.Errorf("flatkv unavail func (f *failingEVMStore) LoadVersionReadOnly(int64) (flatkv.Store, error) { return nil, fmt.Errorf("flatkv unavailable") } +func (f *failingEVMStore) CheckVersionReachable(int64) error { return nil } func (f *failingEVMStore) ApplyChangeSets(int64, []*proto.NamedChangeSet) error { return nil } diff --git a/sei-db/state_db/sc/flatkv/api.go b/sei-db/state_db/sc/flatkv/api.go index cc7f342ed9..9329bd0bd3 100644 --- a/sei-db/state_db/sc/flatkv/api.go +++ b/sei-db/state_db/sc/flatkv/api.go @@ -1,6 +1,7 @@ package flatkv import ( + "errors" "io" dbm "github.com/tendermint/tm-db" @@ -17,6 +18,9 @@ type Options struct { Dir string } +// ErrVersionUnreachable reports that a store cannot reconstruct a version from its retained history. +var ErrVersionUnreachable = errors.New("version unreachable") + // Store provides EVM state storage with LtHash integrity. // // Lifecycle: NewCommitStore (create) → LoadLatest (open) → ApplyChangeSets/Commit → Close. @@ -36,6 +40,11 @@ type Store interface { // so a version whose history has been pruned fails rather than being served approximately. LoadVersionReadOnly(targetVersion int64) (Store, error) + // CheckVersionReachable reports whether targetVersion can be reconstructed from retained history. It + // does not modify the store. An unreachable version returns an error wrapping ErrVersionUnreachable, + // and the store stays usable. + CheckVersionReachable(targetVersion int64) error + // ApplyChangeSets buffers changesets at the given version, to be // persisted by the next Commit. // diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index ad99d77308..2840b01f27 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -105,7 +105,7 @@ func seekSnapshot(root string, targetVersion int64) (int64, error) { return 0, err } if !ok { - return 0, fmt.Errorf("no snapshot found for target version %d", targetVersion) + return 0, fmt.Errorf("%w: no snapshot found for target version %d", ErrVersionUnreachable, targetVersion) } return found, nil } @@ -492,24 +492,19 @@ func (s *CommitStore) pruneSnapshotsByCount(dir string, currentVersion int64) in return pruned } -// rollbackBaseVersion returns the snapshot version Rollback should rewind to for targetVersion, and reports -// an error if the target cannot be reached from it. A target is reachable when a snapshot at or below it -// exists and either sits exactly on it, or the WAL still holds every block between that snapshot and the -// target. With a nil WAL there is no replay, so only a snapshot sitting exactly on the target qualifies. -// When the only snapshot behind the target is the initial one, the store's history starts at the WAL's first -// block rather than at block 1, so a target is reachable from there iff the WAL holds it. -// -// It reads only: no snapshot, symlink, or WAL state is modified, so Rollback can consult it before touching -// anything and refuse an impossible target outright. -func (s *CommitStore) rollbackBaseVersion(dir string, targetVersion int64) (int64, error) { +// reachableBaseVersion returns the newest snapshot at or below targetVersion from which targetVersion can be +// reconstructed, or an error wrapping ErrVersionUnreachable when retained history cannot reach it. A target +// is reachable when that snapshot sits exactly on it, or the WAL holds every block between the two. With a +// nil WAL, only a snapshot exactly on the target qualifies. It modifies no snapshot, symlink, or WAL state. +func (s *CommitStore) reachableBaseVersion(dir string, targetVersion int64) (int64, error) { if targetVersion < 1 { - return 0, fmt.Errorf("rollback target %d is invalid: version 0 means no state, so there is nothing "+ - "to roll back to", targetVersion) + return 0, fmt.Errorf("%w: target version %d is invalid: version 0 means no state", + ErrVersionUnreachable, targetVersion) } baseVersion, err := seekSnapshot(dir, targetVersion) if err != nil { - return 0, fmt.Errorf("seek snapshot for rollback: %w", err) + return 0, fmt.Errorf("seek snapshot for version %d: %w", targetVersion, err) } if baseVersion == targetVersion { return baseVersion, nil @@ -517,27 +512,35 @@ func (s *CommitStore) rollbackBaseVersion(dir string, targetVersion int64) (int6 // The snapshot lands below the target, so the WAL has to supply the blocks in between. if s.wal == nil { - return 0, fmt.Errorf("cannot roll back to version %d: nearest snapshot is %d and this store has no "+ - "WAL to replay the difference", targetVersion, baseVersion) + return 0, fmt.Errorf("%w: cannot reach version %d: nearest snapshot is %d and this store has no "+ + "WAL to replay the difference", ErrVersionUnreachable, targetVersion, baseVersion) } ok, first, last, err := s.wal.GetStoredRange() if err != nil { - return 0, fmt.Errorf("read WAL range for rollback: %w", err) + return 0, fmt.Errorf("read WAL range for version %d: %w", targetVersion, err) } if !ok { - return 0, fmt.Errorf("cannot roll back to version %d: nearest snapshot is %d and the WAL is empty, "+ - "so blocks %d-%d are unavailable", targetVersion, baseVersion, baseVersion+1, targetVersion) + return 0, fmt.Errorf("%w: cannot reach version %d: nearest snapshot is %d and the WAL is empty, "+ + "so blocks %d-%d are outside the retained FlatKV history", + ErrVersionUnreachable, targetVersion, baseVersion, baseVersion+1, targetVersion) } needFrom := uint64(baseVersion) + 1 //nolint:gosec // baseVersion >= 0 needTo := uint64(targetVersion) //nolint:gosec // targetVersion >= 1 checked above if first > needFrom || last < needTo { - return 0, fmt.Errorf("cannot roll back to version %d: nearest snapshot is %d, so blocks %d-%d are "+ - "needed, but the WAL only holds %d-%d", - targetVersion, baseVersion, needFrom, needTo, first, last) + return 0, fmt.Errorf("%w: cannot reach version %d: nearest snapshot is %d, so blocks %d-%d are "+ + "needed and are missing; the WAL only holds %d-%d", + ErrVersionUnreachable, targetVersion, baseVersion, needFrom, needTo, first, last) } return baseVersion, nil } +// CheckVersionReachable reports whether targetVersion can be reconstructed from this store's retained +// snapshots and WAL. It does not modify the store. +func (s *CommitStore) CheckVersionReachable(targetVersion int64) error { + _, err := s.reachableBaseVersion(s.flatkvDir(), targetVersion) + return err +} + // Rollback restores state to targetVersion by rewinding to the highest // snapshot <= targetVersion, replaying WAL to reach the target, and // truncating all WAL entries and snapshots beyond that point. @@ -578,7 +581,7 @@ func (s *CommitStore) Rollback(targetVersion int64) (err error) { // Establish reachability first: everything below this point mutates the store irreversibly, and closing // the DBs would leave it unusable on an early return. - baseVersion, err := s.rollbackBaseVersion(dir, targetVersion) + baseVersion, err := s.reachableBaseVersion(dir, targetVersion) if err != nil { return err } @@ -609,7 +612,7 @@ func (s *CommitStore) Rollback(targetVersion int64) (err error) { // can't replay past target and the write head resumes at targetVersion+1. Rollback is a startup/offline // operation (no concurrent commits), so rather than mutating a live instance we close the WAL, prune it // offline, and reopen it. The store owns the injected WAL and its location (see NewCommitStore), so the - // reopen goes through the same config. The prune only ever runs for a target rollbackBaseVersion already + // reopen goes through the same config. The prune only ever runs for a target reachableBaseVersion already // established is reachable, including the case where the target predates every retained block and the // prune empties the WAL. Skipped when the WAL is nil — the outer context owns it. if s.wal != nil { diff --git a/sei-db/state_db/sc/flatkv/snapshot_test.go b/sei-db/state_db/sc/flatkv/snapshot_test.go index 890122cea8..3f43db0c1f 100644 --- a/sei-db/state_db/sc/flatkv/snapshot_test.go +++ b/sei-db/state_db/sc/flatkv/snapshot_test.go @@ -680,7 +680,7 @@ func TestRollbackRejectsTargetTheWALNoLongerCovers(t *testing.T) { // TestRollbackRejectsVersionZero verifies version 0 is refused: it means no state, so there is nothing to roll // back to, and it is the one target that would reach PruneAfter's retains-block-zero boundary. func TestRollbackRejectsVersionZero(t *testing.T) { - requireRollbackRejected(t, rollbackFixture(t), 0, "nothing to roll back to") + requireRollbackRejected(t, rollbackFixture(t), 0, "version 0 means no state") } // rollbackFixtureMidChainWALStart returns a store seeded to begin at block 10, so its snapshot sits at 9 and @@ -1264,6 +1264,18 @@ func TestSeekSnapshotEmptyDir(t *testing.T) { dir := t.TempDir() _, err := seekSnapshot(dir, 10) require.Error(t, err, "empty dir should not find any snapshot") + require.ErrorIs(t, err, ErrVersionUnreachable) +} + +// A filesystem fault while listing snapshots says nothing about which versions the store holds, so it +// must not be reported as an unreachable version. +func TestSeekSnapshotReadFailureIsNotUnreachable(t *testing.T) { + root := filepath.Join(t.TempDir(), "not-a-dir") + require.NoError(t, os.WriteFile(root, nil, 0600)) + + _, err := seekSnapshot(root, 10) + require.Error(t, err) + require.NotErrorIs(t, err, ErrVersionUnreachable) } func TestSeekSnapshotExact(t *testing.T) { diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index c8c3768066..15779b6ac7 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -380,6 +380,11 @@ func (s *CommitStore) LoadVersionReadOnly(targetVersion int64) (opened Store, re if s.readOnly { return nil, errReadOnly } + if targetVersion != 0 { + if err := s.CheckVersionReachable(targetVersion); err != nil { + return nil, err + } + } lazyLock := s.fileLock == nil if lazyLock { diff --git a/sei-db/state_db/sc/flatkv/store_replay_test.go b/sei-db/state_db/sc/flatkv/store_replay_test.go index 09c1e57209..7cf419d75d 100644 --- a/sei-db/state_db/sc/flatkv/store_replay_test.go +++ b/sei-db/state_db/sc/flatkv/store_replay_test.go @@ -209,6 +209,53 @@ func TestReadOnlySurfacesReplayGap(t *testing.T) { require.Equal(t, int64(6), s.Version()) } +func TestCheckVersionReachableCoversSeededFloorAndWAL(t *testing.T) { + cfg := config.DefaultTestConfig(t) + cfg.DataDir = filepath.Join(t.TempDir(), flatkvRootDir) + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s.LoadLatest()) + defer func() { require.NoError(t, s.Close()) }() + + require.NoError(t, s.SetInitialVersion(10)) + require.NoError(t, s.CheckVersionReachable(9), "seeded snapshot should be reachable without WAL replay") + + err = s.CheckVersionReachable(8) + require.Error(t, err) + require.ErrorIs(t, err, ErrVersionUnreachable) + + commitStorageEntry(t, s, ktype.Address{0xAA}, ktype.Slot{0x01}, []byte{0x10}) + require.NoError(t, s.CheckVersionReachable(10), "WAL should bridge from the seed to the first block") + + err = s.CheckVersionReachable(11) + require.Error(t, err) + require.ErrorIs(t, err, ErrVersionUnreachable) +} + +func TestLoadVersionReadOnlyBelowHistoryFailsBeforeCreatingWorkDir(t *testing.T) { + cfg := config.DefaultTestConfig(t) + cfg.DataDir = filepath.Join(t.TempDir(), flatkvRootDir) + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s.LoadLatest()) + defer func() { require.NoError(t, s.Close()) }() + + require.NoError(t, s.SetInitialVersion(10)) + require.Empty(t, readonlyWorkDirs(t, cfg.DataDir)) + + _, err = s.LoadVersionReadOnly(8) + require.Error(t, err) + require.ErrorIs(t, err, ErrVersionUnreachable) + require.Empty(t, readonlyWorkDirs(t, cfg.DataDir)) +} + +func readonlyWorkDirs(t *testing.T, dir string) []string { + t.Helper() + matches, err := filepath.Glob(filepath.Join(dir, readOnlyDirPrefix+"*")) + require.NoError(t, err) + return matches +} + // TestResolveReplayRangeBounds exercises resolveReplayRange directly. It is the one piece of arithmetic both // replay destinations share, so a mistake here is a mistake in crash recovery and in read-only export at once. // The gap case matters most: silently shortening the range instead of erroring would commit a state whose diff --git a/sei-db/state_db/sc/flatkv/store_test.go b/sei-db/state_db/sc/flatkv/store_test.go index 10fae609fb..0c7322eccf 100644 --- a/sei-db/state_db/sc/flatkv/store_test.go +++ b/sei-db/state_db/sc/flatkv/store_test.go @@ -482,8 +482,8 @@ func TestReadOnlyAtBeyondWALFails(t *testing.T) { require.NoError(t, s2.LoadLatest()) defer func() { require.NoError(t, s2.Close()) }() _, err = s2.LoadVersionReadOnly(100) - require.ErrorContains(t, err, "readonly version mismatch", - "a view beyond the WAL should fail") + require.Error(t, err, "a view beyond the WAL should fail") + require.ErrorIs(t, err, ErrVersionUnreachable) } // =============================================================================