From e09411d99d7a0c264531b0afe364e10b836ec4c0 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Thu, 24 Sep 2026 13:55:32 -0500 Subject: [PATCH 1/5] refactor giga state store for hashes --- cmd/seid/cmd/legacy_config_fuzz_test.go | 59 +-- config/tendermintbase/tendermintbase.go | 12 +- giga/evmonly/giga_store.go | 18 + giga/evmonly/giga_store_test.go | 24 + giga/evmonly/memory_store.go | 20 + sei-db/bench/cryptosim/block_hashes.go | 12 +- sei-db/bench/cryptosim/block_hashes_test.go | 6 +- sei-db/bench/cryptosim/cryptosim.go | 6 +- sei-db/bench/cryptosim/database.go | 4 + sei-db/bench/gigasim/block_hashes.go | 12 +- sei-db/bench/gigasim/execution_state.go | 5 + sei-db/bootstrap/recovery.go | 33 +- sei-db/bootstrap/recovery_test.go | 6 +- sei-db/bootstrap/storage_manager.go | 2 +- sei-db/bootstrap/storage_manager_test.go | 10 +- sei-db/common/utils/path.go | 5 + sei-db/config/giga_config.go | 10 + sei-db/config/hashvault_config.go | 44 ++ sei-db/state_db/giga/state_db.go | 341 ++++++-------- .../giga/state_db_hash_listener_test.go | 8 +- sei-db/state_db/giga/state_db_hash_vault.go | 52 +++ .../state_db/giga/state_db_hash_vault_test.go | 320 +++++++++++++ sei-db/state_db/giga/state_db_recovery.go | 263 +++++++++++ .../state_db/giga/state_db_recovery_test.go | 172 +++++++ sei-db/state_db/giga/state_db_replay.go | 211 ++++----- sei-db/state_db/giga/state_db_replay_test.go | 84 +--- sei-db/state_db/giga/state_db_test.go | 2 +- sei-db/state_db/giga/types/state_db.go | 30 +- sei-db/state_db/sc/composite/hashlog.go | 3 +- sei-db/state_db/sc/composite/store.go | 4 +- .../sc/flatkv/finalization_manager.go | 2 +- .../state_db/sc/flatkv/hash_listeners_test.go | 28 +- .../state_db/sc/flatkv/lthash/hash_engine.go | 2 +- .../sc/flatkv/lthash/hash_engine_messages.go | 4 +- .../sc/flatkv/lthash/hash_engine_test.go | 8 +- .../state_db/sc/flatkv/lthash/hash_types.go | 2 +- sei-db/state_db/sc/flatkv/snapshot.go | 12 + sei-db/state_db/sc/flatkv/store.go | 2 +- sei-db/state_db/sc/flatkv/store_meta.go | 2 +- sei-db/state_db/sc/flatkv/store_write.go | 2 +- sei-db/state_db/sc/flatkv/store_write_test.go | 12 +- .../sc/hashlog/flatkv_listener_test.go | 2 +- sei-db/state_db/sc/hashlog/hash_logger.go | 2 +- .../state_db/sc/hashlog/hash_logger_impl.go | 4 +- .../state_db/sc/hashlog/noop_hash_logger.go | 2 +- sei-db/state_db/sc/hashvault/hashvault.go | 414 +++++++++++++--- .../state_db/sc/hashvault/hashvault_config.go | 41 -- .../state_db/sc/hashvault/hashvault_test.go | 441 ++++++++++-------- .../state_db/sc/hashvault/noop_hashvault.go | 30 -- sei-db/state_db/sc/hashvault/offline.go | 119 +++++ sei-db/state_db/sc/hashvault/offline_test.go | 77 +++ .../state_db/sc/hashvault/pebble_hashvault.go | 253 ---------- .../sc/hashvault/pebble_hashvault_codec.go | 160 ------- .../sc/hashvault/pebble_hashvault_rollback.go | 111 ----- .../pebble_hashvault_rollback_test.go | 141 ------ .../sc/hashvault/pebble_hashvault_test.go | 338 -------------- sei-db/state_db/sc/hashvault/prune.go | 75 +++ sei-db/state_db/sc/hashvault/record.go | 55 +++ sei-db/state_db/sc/memiavl/hashlog_test.go | 2 +- sei-tendermint/config/autobahn_toml_test.go | 18 +- sei-tendermint/config/config.go | 36 +- sei-tendermint/config/config_fuzz_test.go | 20 +- sei-tendermint/config/toml.go | 21 +- sei-tendermint/internal/p2p/giga_router.go | 7 +- .../internal/p2p/giga_router_common.go | 103 +--- .../internal/p2p/giga_router_common_test.go | 49 -- sei-tendermint/node/public.go | 2 +- sei-tendermint/node/setup.go | 14 +- 68 files changed, 2365 insertions(+), 2026 deletions(-) create mode 100644 sei-db/config/hashvault_config.go create mode 100644 sei-db/state_db/giga/state_db_hash_vault.go create mode 100644 sei-db/state_db/giga/state_db_hash_vault_test.go create mode 100644 sei-db/state_db/giga/state_db_recovery.go create mode 100644 sei-db/state_db/giga/state_db_recovery_test.go delete mode 100644 sei-db/state_db/sc/hashvault/hashvault_config.go delete mode 100644 sei-db/state_db/sc/hashvault/noop_hashvault.go create mode 100644 sei-db/state_db/sc/hashvault/offline.go create mode 100644 sei-db/state_db/sc/hashvault/offline_test.go delete mode 100644 sei-db/state_db/sc/hashvault/pebble_hashvault.go delete mode 100644 sei-db/state_db/sc/hashvault/pebble_hashvault_codec.go delete mode 100644 sei-db/state_db/sc/hashvault/pebble_hashvault_rollback.go delete mode 100644 sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go delete mode 100644 sei-db/state_db/sc/hashvault/pebble_hashvault_test.go create mode 100644 sei-db/state_db/sc/hashvault/prune.go create mode 100644 sei-db/state_db/sc/hashvault/record.go diff --git a/cmd/seid/cmd/legacy_config_fuzz_test.go b/cmd/seid/cmd/legacy_config_fuzz_test.go index cf2edd898c..03c5a16860 100644 --- a/cmd/seid/cmd/legacy_config_fuzz_test.go +++ b/cmd/seid/cmd/legacy_config_fuzz_test.go @@ -176,24 +176,22 @@ var tmKeys = []tmKey{ }, } -// FuzzHashVaultDisabledUnsafeResolution pins the root-scope kill switch for the -// app-hash equivocation guard. +// FuzzHashVaultHaltOnMismatchResolution pins the root-scope switch that selects whether a hash vault +// mismatch halts the node. // -// Two things make it worth its own target. It is a bool whose safe value is the -// default, so an absent key must resolve false — setting it true removes -// equivocation protection with only a log banner. And it lives at TOML root scope, -// before any [section] header: nested under a section it parses as a different key -// and is silently ignored, which reads as "I disabled the guard" while the guard -// stays on, and would read the other way round if the scope were ever mishandled. -// The document is built from the fuzzer's choices rather than taken as free text, -// so the expected outcome follows from construction instead of being a second -// input the fuzzer can mutate out of agreement with the first. -func FuzzHashVaultDisabledUnsafeResolution(f *testing.F) { +// Two things make it worth its own target. It is a bool whose safe value is the default, so an absent +// key must resolve true — setting it false lets a node replace a recorded state hash with only an error +// log. And it lives at TOML root scope, before any [section] header: nested under a section it parses as +// a different key and is silently ignored, which reads as "I turned halting off" while the node still +// halts. The document is built from the fuzzer's choices rather than taken as free text, so the expected +// outcome follows from construction instead of being a second input the fuzzer can mutate out of +// agreement with the first. +func FuzzHashVaultHaltOnMismatchResolution(f *testing.F) { f.Add(false, false, false) - f.Add(true, true, false) // root scope, true: the guard is off - f.Add(true, false, false) // root scope, false - f.Add(true, true, true) // nested under a section: silently ignored - f.Add(true, false, true) + f.Add(true, false, false) // root scope, false: a mismatch only logs + f.Add(true, true, false) // root scope, true + f.Add(true, false, true) // nested under a section: silently ignored + f.Add(true, true, true) f.Fuzz(func(t *testing.T, present, value, underSection bool) { configtest.Isolate(t) @@ -204,37 +202,44 @@ func FuzzHashVaultDisabledUnsafeResolution(f *testing.F) { if underSection { doc.WriteString("[p2p]\n") } - fmt.Fprintf(&doc, "hash-vault-disabled-unsafe = %t\n", value) + fmt.Fprintf(&doc, "hash-vault-halt-on-mismatch = %t\n", value) } if doc.Len() > 0 { home.WriteConfigTOML(t, []byte(doc.String())) } // Root scope is the only placement that resolves. Nested under a section the - // key becomes p2p.hash-vault-disabled-unsafe, which nothing reads. - wantDisabled := present && value && !underSection + // key becomes p2p.hash-vault-halt-on-mismatch, which nothing reads. + wantHalt := true + if present && !underSection { + wantHalt = value + } got := applyLegacy(t, home, nil) if got.err != nil { t.Fatalf("Apply must succeed on a well-formed config.toml, got %v", got.err) } - if got.ctx.Config.HashVaultDisabledUnsafe != wantDisabled { - t.Fatalf("hash-vault-disabled-unsafe resolved to %v, want %v, from:\n%s", - got.ctx.Config.HashVaultDisabledUnsafe, wantDisabled, doc.String()) + if got.ctx.Config.HashVaultHaltOnMismatch != wantHalt { + t.Fatalf("hash-vault-halt-on-mismatch resolved to %v, want %v, from:\n%s", + got.ctx.Config.HashVaultHaltOnMismatch, wantHalt, doc.String()) } }) } -// TestHashVaultDisabledUnsafeDefaultsToEnabledGuard states the default on its own, -// so the guard's safe value is pinned even if every seed above were removed. -func TestHashVaultDisabledUnsafeDefaultsToEnabledGuard(t *testing.T) { +// TestHashVaultDefaultsHaltOnMismatch states the defaults on their own, so the safe value is pinned even +// if every seed above were removed. +func TestHashVaultDefaultsHaltOnMismatch(t *testing.T) { configtest.Isolate(t) got := applyLegacy(t, configtest.NewHome(t), nil) if got.err != nil { t.Fatalf("Apply: %v", got.err) } - if got.ctx.Config.HashVaultDisabledUnsafe { - t.Fatal("an empty home must leave the app-hash equivocation guard enabled") + if !got.ctx.Config.HashVaultHaltOnMismatch { + t.Fatal("an empty home must leave a hash vault mismatch halting the node") + } + if got.ctx.Config.HashVaultEmptyRollbackBlocks != 1000 { + t.Fatalf("an empty home must rewind 1000 blocks over an empty hash vault, got %d", + got.ctx.Config.HashVaultEmptyRollbackBlocks) } } diff --git a/config/tendermintbase/tendermintbase.go b/config/tendermintbase/tendermintbase.go index 59f1bca696..20cfab4837 100644 --- a/config/tendermintbase/tendermintbase.go +++ b/config/tendermintbase/tendermintbase.go @@ -57,8 +57,9 @@ var removedFromTheNode = []string{"proxy-app", "abci", "filter-peers"} type nodeRootSchema struct { tmcfg.BaseConfig `mapstructure:",squash"` - AutobahnConfigFile string `mapstructure:"autobahn-config-file"` - HashVaultDisabledUnsafe bool `mapstructure:"hash-vault-disabled-unsafe"` + AutobahnConfigFile string `mapstructure:"autobahn-config-file"` + HashVaultHaltOnMismatch bool `mapstructure:"hash-vault-halt-on-mismatch"` + HashVaultEmptyRollbackBlocks uint64 `mapstructure:"hash-vault-empty-rollback-blocks"` } // removedSettings are the consensus paths this section does not declare. Each names a field the node's @@ -289,9 +290,10 @@ func privValidatorDefaults(mode registry.Mode) any { return *forMode(mode).PrivV func rootDefaults(mode registry.Mode) any { live := forMode(mode) return nodeRootSchema{ - BaseConfig: live.BaseConfig, - AutobahnConfigFile: live.AutobahnConfigFile, - HashVaultDisabledUnsafe: live.HashVaultDisabledUnsafe, + BaseConfig: live.BaseConfig, + AutobahnConfigFile: live.AutobahnConfigFile, + HashVaultHaltOnMismatch: live.HashVaultHaltOnMismatch, + HashVaultEmptyRollbackBlocks: live.HashVaultEmptyRollbackBlocks, } } diff --git a/giga/evmonly/giga_store.go b/giga/evmonly/giga_store.go index 268799ab9d..61f1a93892 100644 --- a/giga/evmonly/giga_store.go +++ b/giga/evmonly/giga_store.go @@ -22,6 +22,10 @@ var ( var _ StateReader = gigaSnapshotStateReader{} +// placeholderBlockHashRetention is how many of the newest blocks keep their state hashes. It is a +// placeholder until a real threshold is wired in. +const placeholderBlockHashRetention = 10_000 + // NamedChangeSetEncoder converts an executor-native state result into the // on-disk changesets understood by a giga store. It is called synchronously // while the block's read snapshot is still open. It must treat the input as @@ -94,6 +98,11 @@ func (e *Executor) executePreparedBlockWithStore(ctx context.Context, req Prepar if err := stateStore.CommitStateChanges(blockNumber, changesets); err != nil { return nil, fmt.Errorf("commit state changes for block %d: %w", req.Context.Number, err) } + // PLACEHOLDER: keeps the newest placeholderBlockHashRetention blocks' hashes. A real threshold, set by + // what giga execution needs block hashes for, should be wired in here. + if err := stateStore.PruneBlockHashesBelow(blockHashesPrunedBelow(req.Context.Number)); err != nil { + return nil, fmt.Errorf("prune block hashes after block %d: %w", req.Context.Number, err) + } ok = true return result, nil } @@ -143,3 +152,12 @@ func (r gigaSnapshotStateReader) GetState(addr common.Address, key common.Hash) func (r gigaSnapshotStateReader) useMissingState(addr common.Address) bool { return r.missingState != nil && !r.snapshot.AccountExists(addr) } + +// blockHashesPrunedBelow returns the block below which state hashes may be pruned once blockNumber is +// committed. +func blockHashesPrunedBelow(blockNumber uint64) uint64 { + if blockNumber < placeholderBlockHashRetention { + return 0 + } + return blockNumber - placeholderBlockHashRetention +} diff --git a/giga/evmonly/giga_store_test.go b/giga/evmonly/giga_store_test.go index bd1759fe35..8133d26a89 100644 --- a/giga/evmonly/giga_store_test.go +++ b/giga/evmonly/giga_store_test.go @@ -21,6 +21,7 @@ type recordingGigaStore struct { commitErr error commitBlock []int64 commits [][]*proto.NamedChangeSet + pruneBelow []uint64 } func (s *recordingGigaStore) CommitStateChanges(blockNum int64, changeset []*proto.NamedChangeSet) error { @@ -42,6 +43,17 @@ func (s *recordingGigaStore) OpenViewAt(int64) (gigatypes.StateView, bool) { return nil, false } +func (s *recordingGigaStore) GetBlockHeight() uint64 { return 0 } + +func (s *recordingGigaStore) GetBlockHash(uint64) ([32]byte, gigatypes.BlockHashStatus, error) { + return [32]byte{}, gigatypes.BlockHashStatusNotReady, nil +} + +func (s *recordingGigaStore) PruneBlockHashesBelow(blockNumber uint64) error { + s.pruneBelow = append(s.pruneBelow, blockNumber) + return nil +} + func (s *recordingGigaStore) Close() error { return nil } type memoryGigaSnapshot struct { @@ -227,6 +239,8 @@ func TestExecutorCommitsGigaStoreStateChanges(t *testing.T) { require.Equal(t, 1, store.openCount) require.Equal(t, 1, snapshot.closeCount) require.Equal(t, []int64{41}, store.commitBlock) + require.Equal(t, []uint64{0}, store.pruneBelow, + "the block hashes are pruned after the commit, and none are old enough to go yet") require.Equal(t, [][]*proto.NamedChangeSet{wantChangesets}, store.commits) require.Contains(t, result.ChangeSet.Balances, BalanceChange{Address: recipient, Balance: big.NewInt(7)}) result.Release() @@ -447,3 +461,13 @@ func TestExecutorGigaStoreFailuresDoNotCommitPartialState(t *testing.T) { require.Empty(t, store.commits) }) } + +// The placeholder retention keeps the newest placeholderBlockHashRetention blocks' hashes, and never wraps +// below block 0 on a young chain. +func TestBlockHashesPrunedBelowKeepsThePlaceholderRetention(t *testing.T) { + require.Equal(t, uint64(0), blockHashesPrunedBelow(0)) + require.Equal(t, uint64(0), blockHashesPrunedBelow(placeholderBlockHashRetention-1)) + require.Equal(t, uint64(0), blockHashesPrunedBelow(placeholderBlockHashRetention)) + require.Equal(t, uint64(1), blockHashesPrunedBelow(placeholderBlockHashRetention+1)) + require.Equal(t, uint64(5_000), blockHashesPrunedBelow(placeholderBlockHashRetention+5_000)) +} diff --git a/giga/evmonly/memory_store.go b/giga/evmonly/memory_store.go index d34589454e..c0869ee049 100644 --- a/giga/evmonly/memory_store.go +++ b/giga/evmonly/memory_store.go @@ -372,6 +372,26 @@ func (s *MemoryStore) RegisterHashListener(_ gigatypes.HashListener) (lthash.Blo return lthash.BlockHash{}, fmt.Errorf("evmonly: an in-memory store computes no block hashes") } +// GetBlockHeight returns the last block committed, or 0 when none has been. +func (s *MemoryStore) GetBlockHeight() uint64 { + s.mu.RLock() + defer s.mu.RUnlock() + if !s.hasCurrentHeight { + return 0 + } + return uint64(s.currentHeight) //nolint:gosec // CommitStateChanges refuses a negative block number +} + +// GetBlockHash reports every block's hash as not ready, since this store computes no block hashes. +func (s *MemoryStore) GetBlockHash(uint64) ([32]byte, gigatypes.BlockHashStatus, error) { + return [32]byte{}, gigatypes.BlockHashStatusNotReady, nil +} + +// PruneBlockHashesBelow does nothing, since this store records no block hashes. +func (s *MemoryStore) PruneBlockHashesBelow(uint64) error { + return nil +} + // Close releases nothing. This store holds no handle outside its own maps, which go with it. func (s *MemoryStore) Close() error { return nil } diff --git a/sei-db/bench/cryptosim/block_hashes.go b/sei-db/bench/cryptosim/block_hashes.go index b10fac4961..017e62f57a 100644 --- a/sei-db/bench/cryptosim/block_hashes.go +++ b/sei-db/bench/cryptosim/block_hashes.go @@ -38,7 +38,7 @@ type blockHashWaiter struct { committed int // The block the next hash taken must describe, or 0 until the first one has been taken. - nextExpected int64 + nextExpected uint64 // How long to wait for one hash before reporting a database that has stopped hashing. waitTimeout time.Duration @@ -62,7 +62,7 @@ func newBlockHashWaiter(lagBlocks int, metrics *CryptosimMetrics) *blockHashWait // Blocking here is the backpressure: it stops a database finalizing blocks faster than the benchmark // accepts their hashes. The context is the release, cancelled when the database shuts down, since a // send with no taker left would otherwise never return. -func (w *blockHashWaiter) listen(ctx context.Context, _ int64, hash *lthash.BlockHash) error { +func (w *blockHashWaiter) listen(ctx context.Context, _ uint64, hash *lthash.BlockHash) error { select { case w.hashes <- hash: return nil @@ -112,3 +112,11 @@ func (w *blockHashWaiter) takeHash() (*lthash.BlockHash, error) { "%d blocks behind the block just committed", w.waitTimeout, w.lagBlocks) } } + +// blockHashRetention is how many of the newest blocks keep their hashes. +const blockHashRetention = 10_000 + +// blockHashesPrunedBelow returns the block below which the hashes of blocks up to blockNum may be pruned. +func blockHashesPrunedBelow(blockNum int64) uint64 { + return uint64(max(blockNum-blockHashRetention, 0)) //nolint:gosec // clamped non-negative +} diff --git a/sei-db/bench/cryptosim/block_hashes_test.go b/sei-db/bench/cryptosim/block_hashes_test.go index 0a0e7c5f85..56bd13a0d1 100644 --- a/sei-db/bench/cryptosim/block_hashes_test.go +++ b/sei-db/bench/cryptosim/block_hashes_test.go @@ -11,7 +11,7 @@ import ( ) // publish hands the waiter the hash of one block, as the database's dispatch would. -func publish(t *testing.T, w *blockHashWaiter, blockNumber int64) { +func publish(t *testing.T, w *blockHashWaiter, blockNumber uint64) { t.Helper() require.NoError(t, w.listen(t.Context(), blockNumber, <hash.BlockHash{BlockNumber: blockNumber})) } @@ -31,7 +31,7 @@ func TestTheFirstBlocksRunAheadWithoutTakingAHash(t *testing.T) { // would let the benchmark drift arbitrarily far ahead of hashing. func TestOneHashIsTakenPerBlockAfterTheWindow(t *testing.T) { waiter := newBlockHashWaiter(3, nil) - for block := int64(1); block <= 4; block++ { + for block := uint64(1); block <= 4; block++ { publish(t, waiter, block) } @@ -40,7 +40,7 @@ func TestOneHashIsTakenPerBlockAfterTheWindow(t *testing.T) { require.NoError(t, waiter.awaitBlock()) } require.Len(t, waiter.hashes, 3, "exactly one hash may be taken per block committed") - require.Equal(t, int64(2), waiter.nextExpected) + require.Equal(t, uint64(2), waiter.nextExpected) } // The wait is the point: a database that cannot hash as fast as the benchmark commits has to slow the diff --git a/sei-db/bench/cryptosim/cryptosim.go b/sei-db/bench/cryptosim/cryptosim.go index 0158ce18da..6060527f3a 100644 --- a/sei-db/bench/cryptosim/cryptosim.go +++ b/sei-db/bench/cryptosim/cryptosim.go @@ -12,6 +12,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/common/keys" crand "github.com/sei-protocol/sei-chain/sei-db/common/rand" "github.com/sei-protocol/sei-chain/sei-db/common/utils" + dbconfig "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/controller" "github.com/sei-protocol/sei-chain/sei-db/state_db/giga" ) @@ -146,7 +147,10 @@ func NewCryptoSim( // giga.NewStateDB is the node's own entry point, and the only one that leaves the state WAL // outside the live state DB: it opens the WAL itself and writes each block to it ahead of the // commit. A live state DB opened directly would own its WAL and write it inline instead. - db, err := giga.NewStateDB(ctx, config.FlatKVConfig, config.StateStoreConfig, config.CheckpointConfig) + hashVaultConfig := dbconfig.DefaultHashVaultConfig() + hashVaultConfig.DataDir = filepath.Join(config.DataDir, "hashvault") + db, err := giga.NewStateDB( + ctx, config.FlatKVConfig, config.StateStoreConfig, config.CheckpointConfig, hashVaultConfig, 0) if err != nil { cancel() return nil, fmt.Errorf("failed to open the state DB: %w", err) diff --git a/sei-db/bench/cryptosim/database.go b/sei-db/bench/cryptosim/database.go index caac90a44e..7030430d3c 100644 --- a/sei-db/bench/cryptosim/database.go +++ b/sei-db/bench/cryptosim/database.go @@ -229,6 +229,10 @@ func (d *Database) FinalizeBlock( if err := d.db.CommitStateChanges(blockNum, changeSets); err != nil { return fmt.Errorf("failed to commit block %d: %w", blockNum, err) } + // The same placeholder retention giga execution uses, until a real threshold is wired in there. + if err := d.db.PruneBlockHashesBelow(blockHashesPrunedBelow(blockNum)); err != nil { + return fmt.Errorf("failed to prune block hashes after committing block %d: %w", blockNum, err) + } d.nextBlockNumber++ d.metrics.ReportDBCommit() d.reopenView() diff --git a/sei-db/bench/gigasim/block_hashes.go b/sei-db/bench/gigasim/block_hashes.go index bfe9f71bee..b418ee9be3 100644 --- a/sei-db/bench/gigasim/block_hashes.go +++ b/sei-db/bench/gigasim/block_hashes.go @@ -34,7 +34,7 @@ type blockHashWaiter struct { committed int // The block the next hash taken must describe, or 0 until the first one has been taken. - nextExpected int64 + nextExpected uint64 // How long to wait for one hash before reporting a state DB that has stopped hashing. waitTimeout time.Duration @@ -55,7 +55,7 @@ func newBlockHashWaiter(lagBlocks int, metrics *GigasimMetrics) *blockHashWaiter // listen takes one block's hash from the state DB, blocking while the benchmark is further ahead than // its window allows. That block is the backpressure on hashing; the context releases it when the state // DB shuts down, since a send with no taker left would never return. -func (w *blockHashWaiter) listen(ctx context.Context, _ int64, hash *lthash.BlockHash) error { +func (w *blockHashWaiter) listen(ctx context.Context, _ uint64, hash *lthash.BlockHash) error { select { case w.hashes <- hash: return nil @@ -104,3 +104,11 @@ func (w *blockHashWaiter) takeHash() (*lthash.BlockHash, error) { "%d blocks behind the block just committed", w.waitTimeout, w.lagBlocks) } } + +// blockHashRetention is how many of the newest blocks keep their hashes. +const blockHashRetention = 10_000 + +// blockHashesPrunedBelow returns the block below which the hashes of blocks up to blockNum may be pruned. +func blockHashesPrunedBelow(blockNum int64) uint64 { + return uint64(max(blockNum-blockHashRetention, 0)) //nolint:gosec // clamped non-negative +} diff --git a/sei-db/bench/gigasim/execution_state.go b/sei-db/bench/gigasim/execution_state.go index 198877af84..a167594507 100644 --- a/sei-db/bench/gigasim/execution_state.go +++ b/sei-db/bench/gigasim/execution_state.go @@ -120,6 +120,11 @@ func (s *executionState) commitBlock(blockNum int64, writes blockWrites) error { } s.metrics.ReportStateCommit(int64(len(writes.changeSets[0].Changeset.Pairs))) + // The same placeholder retention giga execution uses, until a real threshold is wired in there. + if err := s.db.PruneBlockHashesBelow(blockHashesPrunedBelow(blockNum)); err != nil { + return fmt.Errorf("failed to prune block hashes after committing block %d: %w", blockNum, err) + } + // Committing a block is not finishing it: the hash of a block committed a bounded number of blocks // ago is taken here, and waited for when hashing has fallen behind execution. Reopening the view // is charged here too, being a fraction of a percent that no one reads as a stage of its own. diff --git a/sei-db/bootstrap/recovery.go b/sei-db/bootstrap/recovery.go index 3c3df6c50a..4c6c52ab28 100644 --- a/sei-db/bootstrap/recovery.go +++ b/sei-db/bootstrap/recovery.go @@ -52,7 +52,7 @@ func (m *GigaStorageManager) OpenDBWithRecovery(ctx context.Context) error { // State goes first because it is the rollback that refuses: a target its snapshots and WAL cannot span // leaves the node down for an operator to retry at a higher one, and receipts cut to the lower target // would no longer be there to reach. -func (m *GigaStorageManager) recoverStores(ctx context.Context, target int64) error { +func (m *GigaStorageManager) recoverStores(ctx context.Context, target uint64) error { if target == 0 { logger.Info("No height to converge on, opening the state DB where its files sit") return m.openStateDB(ctx) @@ -100,7 +100,7 @@ func (m *GigaStorageManager) openReceiptStore() error { // // The state and receipt heads are read from their directories, which takes the locks their open stores // hold, so this must run before either of those stores opens. -func (m *GigaStorageManager) findTargetRecoveryHeight() (int64, error) { +func (m *GigaStorageManager) findTargetRecoveryHeight() (uint64, error) { logger.Info("Reading a store head", "store", "block store") blockHeight, err := m.blockStore.GetLatestBlock() if err != nil { @@ -125,7 +125,7 @@ func (m *GigaStorageManager) findTargetRecoveryHeight() (int64, error) { "state_wal", stateHeight, "receipt_store", receiptHeight, "target", target) - return int64(target), nil //nolint:gosec // heights fit within int64 + return target, nil } // stateWALHead returns the last block the state WAL holds, or 0 when it holds none. @@ -162,10 +162,11 @@ func recoveryTarget(blockHeight, stateHeight, receiptHeight uint64) uint64 { return target } -// openStateDB opens the state commit store, the EVM state store (when enabled) and the state WAL, -// leaving them on the height the WAL holds. +// openStateDB opens the state commit store, the EVM state store (when enabled), the state WAL and the +// hash vault, leaving the stores on the height the WAL holds. func (m *GigaStorageManager) openStateDB(ctx context.Context) error { - stateDB, err := giga.NewStateDB(ctx, m.cfg.FlatKVConfig, m.cfg.SSConfig, m.cfg.CheckpointConfig) + stateDB, err := giga.NewStateDB( + ctx, m.cfg.FlatKVConfig, m.cfg.SSConfig, m.cfg.CheckpointConfig, m.cfg.HashVaultConfig, 0) if err != nil { return err } @@ -173,13 +174,20 @@ func (m *GigaStorageManager) openStateDB(ctx context.Context) error { return nil } -// openStateDBAt opens the same three stores on target, rolling them back to it first. +// openStateDBAt opens the same stores on target, rolling them back to it first. The hash vault is not +// rolled back. // // The rollback is part of the open because cutting the state WAL's tail needs the WAL closed, so an // already-open state DB would have to close and reopen it. -func (m *GigaStorageManager) openStateDBAt(ctx context.Context, target int64) error { - stateDB, err := giga.NewStateDBWithRollback( - ctx, m.cfg.FlatKVConfig, m.cfg.SSConfig, m.cfg.CheckpointConfig, target) +func (m *GigaStorageManager) openStateDBAt(ctx context.Context, target uint64) error { + if target == 0 { + // The state DB reads a target of 0 as no rollback at all, so without this a caller asking for one + // would get a plain open instead. + return fmt.Errorf("rollback target %d is invalid: version 0 means no state, so there is "+ + "nothing to roll back to", target) + } + stateDB, err := giga.NewStateDB( + ctx, m.cfg.FlatKVConfig, m.cfg.SSConfig, m.cfg.CheckpointConfig, m.cfg.HashVaultConfig, target) if err != nil { return err } @@ -191,12 +199,11 @@ func (m *GigaStorageManager) openStateDBAt(ctx context.Context, target int64) er // open store. A store already at or below target is left alone. // // It takes the locks an open receipt store holds, so it must run before openReceiptStore. -func (m *GigaStorageManager) recoverReceipt(target int64) error { +func (m *GigaStorageManager) recoverReceipt(target uint64) error { if !m.cfg.ReceiptDBConfig.Enable { return nil } - //nolint:gosec // recoverStores guards target > 0 - if err := receipt.PruneAfter(m.cfg.ReceiptDBConfig, uint64(target)); err != nil { + if err := receipt.PruneAfter(m.cfg.ReceiptDBConfig, target); err != nil { return fmt.Errorf("roll the receipt store back to %d: %w", target, err) } return nil diff --git a/sei-db/bootstrap/recovery_test.go b/sei-db/bootstrap/recovery_test.go index 34ea37be12..692177c437 100644 --- a/sei-db/bootstrap/recovery_test.go +++ b/sei-db/bootstrap/recovery_test.go @@ -114,7 +114,7 @@ func commitBlocksWithSSSnapshots(t *testing.T, manager *GigaStorageManager, thro // reconverge re-runs what a restart does: it closes every store recovery touches, recovers them onto // target — which is what opens the state DB again — and reopens the receipt store on the far side. -func reconverge(t *testing.T, manager *GigaStorageManager, target int64) { +func reconverge(t *testing.T, manager *GigaStorageManager, target uint64) { t.Helper() require.NoError(t, reconvergeErr(t, manager, target)) require.NoError(t, manager.openReceiptStore()) @@ -122,7 +122,7 @@ func reconverge(t *testing.T, manager *GigaStorageManager, target int64) { // reconvergeErr is reconverge up to the point recovery can fail, for a test that expects it to. The // receipt store is left closed, since a failed recovery leaves the manager with no state DB. -func reconvergeErr(t *testing.T, manager *GigaStorageManager, target int64) error { +func reconvergeErr(t *testing.T, manager *GigaStorageManager, target uint64) error { t.Helper() // Closing first is what a restart does, and it is also required: recovery takes file locks the // open stores hold — the state WAL's directory lock for the reads and the tail cut that precede @@ -252,7 +252,7 @@ func TestFindTargetRecoveryHeightIsZeroWithoutABlockLedger(t *testing.T) { got, err := manager.findTargetRecoveryHeight() require.NoError(t, err) - require.Equal(t, int64(0), got) + require.Equal(t, uint64(0), got) } // Recovering to a target below the WAL head drops every block above it, so the write head resumes at diff --git a/sei-db/bootstrap/storage_manager.go b/sei-db/bootstrap/storage_manager.go index d8c8f37e90..64752744c1 100644 --- a/sei-db/bootstrap/storage_manager.go +++ b/sei-db/bootstrap/storage_manager.go @@ -78,7 +78,7 @@ func (m *GigaStorageManager) startGarbageCollector(ctx context.Context, pruningC // prunableStores returns the opened stores that can join the shared prune cycle. The state stores come // from the StateDB that owns them. func (m *GigaStorageManager) prunableStores() []controller.PrunableStore { - stores := make([]controller.PrunableStore, 0, 5) + stores := make([]controller.PrunableStore, 0, 6) if m.stateDB != nil { stores = append(stores, m.stateDB.PrunableStores()...) } diff --git a/sei-db/bootstrap/storage_manager_test.go b/sei-db/bootstrap/storage_manager_test.go index 92cedfaa40..b61393d9f2 100644 --- a/sei-db/bootstrap/storage_manager_test.go +++ b/sei-db/bootstrap/storage_manager_test.go @@ -103,11 +103,11 @@ func TestStateStoreDisabled(t *testing.T) { "SC still needs the schedule that replaces its own interval") require.True(t, manager.SC().ExternalPruning()) - names := make([]string, 0, 4) + names := make([]string, 0, 5) for _, store := range manager.prunableStores() { names = append(names, store.Name()) } - require.Equal(t, []string{"FlatKV", "StateWAL", "ReceiptDB", "BlockDB"}, names, + require.Equal(t, []string{"FlatKV", "StateWAL", "HashVault", "ReceiptDB", "BlockDB"}, names, "a store this node never opened must not be offered to the collector") } @@ -243,14 +243,14 @@ func TestEveryStoreJoinsThePruneCycle(t *testing.T) { require.True(t, manager.SC().ExternalPruning()) require.True(t, manager.SS().ExternalPruning()) - names := make([]string, 0, 5) + names := make([]string, 0, 6) for _, store := range manager.prunableStores() { names = append(names, store.Name()) } - // The three state stores arrive as one group, from the StateDB that owns them. Order carries no + // The four state stores arrive as one group, from the StateDB that owns them. Order carries no // meaning to the collector: it fixes both cut lines as a minimum over every store before pruning // any of them. - require.Equal(t, []string{"FlatKV", "StateWAL", "EVM SS", "ReceiptDB", "BlockDB"}, names) + require.Equal(t, []string{"FlatKV", "StateWAL", "EVM SS", "HashVault", "ReceiptDB", "BlockDB"}, names) } // TestPrunableStoresOmitsDisabledReceipts pins that a store that was never opened is not offered diff --git a/sei-db/common/utils/path.go b/sei-db/common/utils/path.go index 6edb519c67..28e951cfb7 100644 --- a/sei-db/common/utils/path.go +++ b/sei-db/common/utils/path.go @@ -43,6 +43,11 @@ func GetFlatKVPath(homePath string) string { return filepath.Join(homePath, "data", "state_commit", "flatkv") } +// GetHashVaultPath returns the path for the hash vault that guards the live state DB's block hashes. +func GetHashVaultPath(homePath string) string { + return filepath.Join(homePath, "data", "state_commit", "hashvault") +} + // GetStateStorePath returns the path for the Cosmos state store (SS). // New nodes use data/state_store/cosmos/{backend}; existing nodes with // data/{backend} continue using the legacy path for backward compatibility. diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index b138f2db33..20b0ca9306 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -18,6 +18,7 @@ type GigaStorageConfig struct { BlockDBConfig *littblock.BlockDBConfig // required PruningConfig *StorageGarbageCollectorConfig // required CheckpointConfig CheckpointConfig + HashVaultConfig HashVaultConfig } // gigaReceiptBackend is the receipt backend Giga opens (littidx). @@ -26,6 +27,7 @@ const gigaReceiptBackend = "littidx" // DefaultGigaStorageConfig returns a config rooted at homePath: // // data/state_commit/flatkv +// data/state_commit/hashvault // data/state_store/evm/{backend} // data/ledger/receipt/{backend} // data/ledger/block @@ -46,6 +48,9 @@ func DefaultGigaStorageConfig(homePath string) (*GigaStorageConfig, error) { ssConfig.ExternalPruning = true ssConfig.DisableInternalWAL = true + hashVaultConfig := DefaultHashVaultConfig() + hashVaultConfig.DataDir = utils.GetHashVaultPath(homePath) + receiptConfig := DefaultReceiptStoreConfig() receiptConfig.Backend = gigaReceiptBackend receiptConfig.DBDirectory = utils.GetReceiptStorePath(homePath, receiptConfig.Backend) @@ -59,6 +64,7 @@ func DefaultGigaStorageConfig(homePath string) (*GigaStorageConfig, error) { BlockDBConfig: blockDBConfig, PruningConfig: DefaultStorageGarbageCollectorConfig(), CheckpointConfig: DefaultCheckpointConfig(), + HashVaultConfig: hashVaultConfig, }, nil } @@ -116,6 +122,10 @@ func (c *GigaStorageConfig) Validate() error { return fmt.Errorf("flatkv data dir is required") } + if err := c.HashVaultConfig.Validate(); err != nil { + return fmt.Errorf("hash vault config is invalid: %w", err) + } + if c.BlockDBConfig == nil { return fmt.Errorf("block db config is required") } diff --git a/sei-db/config/hashvault_config.go b/sei-db/config/hashvault_config.go new file mode 100644 index 0000000000..c30018b9cb --- /dev/null +++ b/sei-db/config/hashvault_config.go @@ -0,0 +1,44 @@ +package config + +import "fmt" + +// HashVaultConfig configures the hash vault, the equivocation guard over the live state DB's block hashes. +type HashVaultConfig struct { + // DataDir is the directory the vault keeps its hashes in. + DataDir string + + // HaltOnMismatch selects what a hash that differs from the recorded one does. When true, the state DB + // fails and the node halts. When false, the mismatch is logged as an error, the vault discards the + // recorded hashes from that block up, and the new hash is recorded in their place. + HaltOnMismatch bool + + // EmptyVaultRollbackBlocks is how many blocks the state DB rewinds and replays when it opens over an + // empty vault, so that the vault holds the hashes of recent blocks and not just the loaded one. The + // rewind is limited to what the state commit store's snapshots and the state WAL can reach. 0 records + // only the loaded block's hash. + EmptyVaultRollbackBlocks uint64 + + // LegacyPebbleDir is the directory of the Pebble-backed vault this one replaces, deleted when the vault + // opens. Empty when there is none to delete. + LegacyPebbleDir string + + // Fsync controls whether each recorded hash is fsynced before the vault reports it recorded. + Fsync bool +} + +// DefaultHashVaultConfig returns the default hash vault config. DataDir is left for the caller to set. +func DefaultHashVaultConfig() HashVaultConfig { + return HashVaultConfig{ + HaltOnMismatch: true, + EmptyVaultRollbackBlocks: 1000, + Fsync: true, + } +} + +// Validate returns an error if the config cannot open a vault. +func (c *HashVaultConfig) Validate() error { + if c.DataDir == "" { + return fmt.Errorf("hash vault data dir is required") + } + return nil +} diff --git a/sei-db/state_db/giga/state_db.go b/sei-db/state_db/giga/state_db.go index e1eb804a1e..6ecabd4822 100644 --- a/sei-db/state_db/giga/state_db.go +++ b/sei-db/state_db/giga/state_db.go @@ -17,6 +17,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashvault" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/evm" "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" ) @@ -26,17 +27,11 @@ var logger = seilog.NewLogger("db", "state-db", "giga") var _ gigatypes.StateDB = (*StateDB)(nil) // StateDB writes a committed block to the state WAL, the state commit store (SC) and the EVM state -// store (SS), and serves current-block reads from SC. +// store (SS), serves current-block reads from SC, and records SC's block hashes in the hash vault. // -// It opens all three stores, brings them onto one height, and closes them. SC and SS run without a WAL +// It opens all four stores, brings them onto one height, and closes them. SC and SS run without a WAL // of their own, so every block either of them replays is read from the WAL here. type StateDB struct { - // Where the state commit store and the state WAL live. - flatkvCfg *flatkvconfig.Config - - // Where the EVM state store lives, and whether it is enabled at all. - ssCfg config.StateStoreConfig - // The state WAL a committed block is written to. wal statewal.StateWAL @@ -46,6 +41,9 @@ type StateDB struct { // ss is nil when the EVM state store is disabled. ss *evm.EVMStateStore + // The hash vault SC's block hashes are recorded in. + vault *hashvault.HashVault + // The checkpoint schedule SC and SS take their snapshot boundaries from. checkpointer *controller.CheckpointScheduler @@ -60,242 +58,178 @@ const commitPhaseTimerName = "giga_state_commit" // gigaMeterName is the OTel meter this package's instruments are created on. const gigaMeterName = "seidb_giga" -// NewStateDB opens SC, SS and the state WAL from their configs and puts SC and SS on one checkpoint -// schedule. -// -// Both stores are put on the WAL's head — replayed up to it, and rewound onto it when a lost WAL tail -// left them above it — so the returned StateDB commits the block after it. NewStateDBWithRollback opens -// them on an earlier height instead. -// -// The returned StateDB owns all three stores and closes them on Close. A failed call closes whatever it -// had already opened. +// NewStateDB opens SC, SS, the state WAL and the hash vault, and puts SC and SS on one checkpoint schedule +// and one height: the WAL's head, or rollbackTo when it is not 0. The returned StateDB commits the block +// after that height, and its hash vault holds the hash of the block SC is on. func NewStateDB( ctx context.Context, flatkvCfg *flatkvconfig.Config, ssCfg config.StateStoreConfig, checkpointCfg config.CheckpointConfig, -) (db *StateDB, retErr error) { - s := &StateDB{ - flatkvCfg: flatkvCfg, - ssCfg: stateStoreConfigFor(ssCfg), - commitPhases: metrics.NewPhaseTimerFactory(otel.Meter(gigaMeterName), commitPhaseTimerName). - RecordLatencies().Build(), - } - defer s.closeOnFailure(&retErr) + hashVaultCfg config.HashVaultConfig, + // The height to roll back to, or 0 to load the latest block possible. Data after rollbackTo target + // may be permanently deleted. Returns an error if not possible to roll back to requested block height. + rollbackTo uint64, +) (_ *StateDB, retErr error) { + ssCfg.DisableInternalWAL = true + + if err := recoverStores(flatkvCfg, ssCfg, hashVaultCfg, rollbackTo); err != nil { + return nil, fmt.Errorf("recover the state DB's stores: %w", err) + } + + var err error + var ss *evm.EVMStateStore + var sc *flatkv.CommitStore + var vault *hashvault.HashVault + var wal statewal.StateWAL + defer func() { + if retErr == nil { + return + } + if err := closeStores(ss, sc, vault, wal); err != nil { + retErr = errors.Join(retErr, fmt.Errorf("close a partially opened state DB: %w", err)) + } + }() - wal, err := s.storedWALRange() - if err != nil { - return nil, err - } - // Before either store opens, the rewinds it may run needing their files closed. - if err := s.discardStateAboveTheWAL(wal); err != nil { - return nil, err + if ss, err = openSS(ssCfg); err != nil { + return nil, fmt.Errorf("open the state DB: %w", err) } - if err := s.openSS(); err != nil { - return nil, err + + if sc, err = openSC(ctx, flatkvCfg); err != nil { + return nil, fmt.Errorf("open the state DB: %w", err) } - if err := s.openSC(ctx); err != nil { - return nil, err + + if vault, err = hashvault.Open(hashVaultCfg); err != nil { + return nil, fmt.Errorf("open the state DB: %w", err) } - if err := s.openWAL(); err != nil { - return nil, err + // The vault must be SC's first listener, and registering it before SC is reachable from outside this + // StateDB is what makes it first. SC hands a hash to its listeners one at a time in registration order + // and stops at the first that refuses it, and the vault returns only once the hash is flushed, so no + // later listener sees a hash the vault has not recorded or has refused. + if _, err := sc.RegisterHashListener(hashVaultListener(vault)); err != nil { + return nil, fmt.Errorf("register the hash vault on the state commit store: %w", err) } - s.startCheckpointSchedule(checkpointCfg) - if err := s.catchUpToWAL(ctx); err != nil { - return nil, err + if wal, err = flatkv.OpenStateWAL(flatkvCfg); err != nil { + return nil, fmt.Errorf("open state WAL: %w", err) } - return s, nil -} -// stateStoreConfigFor is the config a StateDB opens SS with. It is settled here rather than at each -// open because the rollback path opens the same databases through DiscardStateAbove. The changelog -// is off: this StateDB's own state WAL is what catchUpTo replays into SS. -func stateStoreConfigFor(cfg config.StateStoreConfig) config.StateStoreConfig { - cfg.DisableInternalWAL = true - return cfg -} + checkpointer := startCheckpointSchedule(checkpointCfg, sc, ss) -// NewStateDBWithRollback rolls SC, SS and the state WAL back to target and then opens them, so the -// returned StateDB commits target+1. It cuts the WAL's tail to target and puts whichever of SC and SS -// sits above target on its newest snapshot at or below it, all while the stores are closed, then opens -// them the ordinary way and checks both landed on target. -// -// target must be positive, and a target the surviving snapshots and the WAL cannot span is refused. A -// refusal leaves the WAL uncut, so no target this one could reach is lost, but one from SS comes back -// with SC already rewound. -func NewStateDBWithRollback( - ctx context.Context, - flatkvCfg *flatkvconfig.Config, - ssCfg config.StateStoreConfig, - checkpointCfg config.CheckpointConfig, - target int64, -) (*StateDB, error) { - if target <= 0 { - // An empty WAL has a head of 0, which rewindTo reads as nothing to rewind, so without this a - // caller asking for a rollback would get a plain open instead. - return nil, fmt.Errorf("rollback target %d is invalid: version 0 means no state, so there is "+ - "nothing to roll back to", target) + if err := catchUpToWAL(ctx, sc, ss, wal); err != nil { + return nil, fmt.Errorf("catch the state DB up to its WAL: %w", err) } - - // rewindTo only moves files, so it needs no store open, only where they live. - offline := &StateDB{flatkvCfg: flatkvCfg, ssCfg: stateStoreConfigFor(ssCfg)} - if err := offline.rewindTo(target); err != nil { - return nil, err - } - db, err := NewStateDB(ctx, flatkvCfg, ssCfg, checkpointCfg) - if err != nil { - return nil, err - } - if err := db.matchHeight(target); err != nil { - return nil, errors.Join(fmt.Errorf("cannot roll back to %d: %w", target, err), db.Close()) + if rollbackTo > 0 { + if err := matchHeight(sc, ss, wal, int64(rollbackTo)); err != nil { //nolint:gosec // a WAL block number + return nil, fmt.Errorf("cannot roll back to %d: %w", rollbackTo, err) + } } - return db, nil -} -// closeOnFailure closes the stores a failed open had reached, so a caller that gets an error holds no -// store this StateDB left open. It is deferred against the constructor's named error. -func (s *StateDB) closeOnFailure(retErr *error) { - if *retErr == nil { - return - } - if err := s.Close(); err != nil { - *retErr = errors.Join(*retErr, fmt.Errorf("close a partially opened state DB: %w", err)) + if err := recordLoadedBlockHash(sc, vault); err != nil { + return nil, fmt.Errorf("record the loaded block's hash in the hash vault: %w", err) } -} -// openWAL opens the state WAL this StateDB commits blocks to. -func (s *StateDB) openWAL() error { - wal, err := flatkv.OpenStateWAL(s.flatkvCfg) - if err != nil { - return fmt.Errorf("open state WAL: %w", err) - } - s.wal = wal - return nil + return &StateDB{ + wal: wal, + sc: sc, + ss: ss, + vault: vault, + checkpointer: checkpointer, + commitPhases: metrics.NewPhaseTimerFactory(otel.Meter(gigaMeterName), commitPhaseTimerName). + RecordLatencies().Build(), + }, nil } // openSC opens SC with no WAL of its own, on the version its files hold: the working copy, or the // snapshot a rollback has just repointed it at. It replays nothing, so it comes up at or below the // WAL's head and catchUpTo carries it forward from there. -func (s *StateDB) openSC(ctx context.Context) error { - sc, err := flatkv.NewCommitStore(ctx, s.flatkvCfg, nil) +func openSC(ctx context.Context, flatkvCfg *flatkvconfig.Config) (*flatkv.CommitStore, error) { + sc, err := flatkv.NewCommitStore(ctx, flatkvCfg, nil) if err != nil { - return fmt.Errorf("open state commit store: %w", err) + return nil, fmt.Errorf("open state commit store: %w", err) } - s.sc = sc // Every readonly-* directory under the store is deleted, so this has to run before the process // opens a read-only view of its own: after that, the ones a crashed process left are no longer // the only ones there. - if err := s.sc.CleanupOrphanedReadOnlyDirs(); err != nil { - return fmt.Errorf("clean up orphaned state commit read-only dirs: %w", err) + if err := sc.CleanupOrphanedReadOnlyDirs(); err != nil { + return nil, errors.Join( + fmt.Errorf("clean up orphaned state commit read-only dirs: %w", err), sc.Close()) } - if err := s.sc.LoadWorkingCopy(); err != nil { - return fmt.Errorf("load the state commit store: %w", err) + if err := sc.LoadWorkingCopy(); err != nil { + return nil, errors.Join(fmt.Errorf("load the state commit store: %w", err), sc.Close()) } - return nil + return sc, nil } -// openSS opens the EVM state store and its snapshot manager, leaving it nil when the store is disabled. -func (s *StateDB) openSS() error { - if !s.ssCfg.Enable { - return nil +// openSS opens the EVM state store and its snapshot manager, returning nil when the store is disabled. +func openSS(ssCfg config.StateStoreConfig) (*evm.EVMStateStore, error) { + if !ssCfg.Enable { + return nil, nil } - ss, err := evm.NewEVMStateStore(s.ssCfg.EVMDBDirectory, s.ssCfg) + ss, err := evm.NewEVMStateStore(ssCfg.EVMDBDirectory, ssCfg) if err != nil { - return fmt.Errorf("open EVM state store: %w", err) + return nil, fmt.Errorf("open EVM state store: %w", err) } - s.ss = ss - if err := s.ss.StartSnapshots(s.ssSnapshotRoot(), s.ssCfg, nil); err != nil { - return fmt.Errorf("start EVM state store snapshot manager: %w", err) + snapshotRoot := utils.GetStateStoreSnapshotsSiblingPath(ssCfg.EVMDBDirectory) + if err := ss.StartSnapshots(snapshotRoot, ssCfg, nil); err != nil { + return nil, errors.Join(fmt.Errorf("start EVM state store snapshot manager: %w", err), ss.Close()) } - return nil + return ss, nil } -// startCheckpointSchedule puts SC and SS on one snapshot cadence. It runs before either store is on a -// height, so the blocks SC replays offer themselves to the schedule as live commits do. -func (s *StateDB) startCheckpointSchedule(cfg config.CheckpointConfig) { - s.checkpointer = controller.NewCheckpointScheduler(cfg) - s.sc.SetCheckpointScheduler(s.checkpointer) - if s.ss != nil { - s.ss.SetCheckpointScheduler(s.checkpointer) - } -} - -// ssSnapshotRoot returns the directory SS keeps its snapshots in. -func (s *StateDB) ssSnapshotRoot() string { - return utils.GetStateStoreSnapshotsSiblingPath(s.ssCfg.EVMDBDirectory) -} - -// storedWALRange is the block range a state WAL holds on disk: the lowest and highest blocks in it, -// both 0 when it holds none. -type storedWALRange struct { - first, last int64 -} - -// walConfig returns the config that locates the state WAL on disk. -func (s *StateDB) walConfig() *statewal.Config { - return flatkv.StateWALConfig(s.flatkvCfg.DataDir) -} - -// storedWALRange reads the state WAL's block range from its directory. It takes that directory's -// exclusive lock, so it is only for the window before the WAL opens; GetStoredRange on the open handle -// answers the same question afterwards. -func (s *StateDB) storedWALRange() (storedWALRange, error) { - stored, first, last, err := statewal.GetRange(s.walConfig()) - if err != nil { - return storedWALRange{}, fmt.Errorf("read state WAL range: %w", err) - } - if !stored { - return storedWALRange{}, nil - } - //nolint:gosec // a block number never approaches the int64 ceiling - return storedWALRange{first: int64(first), last: int64(last)}, nil +// startCheckpointSchedule puts SC and SS on one snapshot cadence, and returns it. ss is nil when SS is +// disabled. It runs before either store is on a height, so the blocks SC replays offer themselves to the +// schedule as live commits do. +func startCheckpointSchedule( + cfg config.CheckpointConfig, + sc *flatkv.CommitStore, + ss *evm.EVMStateStore, +) *controller.CheckpointScheduler { + checkpointer := controller.NewCheckpointScheduler(cfg) + sc.SetCheckpointScheduler(checkpointer) + if ss != nil { + ss.SetCheckpointScheduler(checkpointer) + } + return checkpointer } -// openWALRange reads the block range from the open WAL handle, which storedWALRange's directory lock -// rules out reading once the WAL is open. -func (s *StateDB) openWALRange() (storedWALRange, error) { - stored, first, last, err := s.wal.GetStoredRange() - if err != nil { - return storedWALRange{}, fmt.Errorf("read state WAL range: %w", err) - } - if !stored { - return storedWALRange{}, nil - } - //nolint:gosec // a block number never approaches the int64 ceiling - return storedWALRange{first: int64(first), last: int64(last)}, nil -} - -// truncateWAL drops every WAL block above target so the next commit is target+1. A live WAL prunes only -// from its start, so this cuts the tail through the directory, which requires that no WAL be open on it. -func (s *StateDB) truncateWAL(target int64) error { - //nolint:gosec // target > 0 here, checked by NewStateDBWithRollback - if err := statewal.PruneAfter(s.walConfig(), uint64(target)); err != nil { - return fmt.Errorf("truncate state WAL to %d: %w", target, err) - } - return nil +// Close closes SC, SS, the hash vault and the state WAL, reporting every failure rather than stopping at +// the first. +func (s *StateDB) Close() error { + return closeStores(s.ss, s.sc, s.vault, s.wal) } -// Close closes SC, SS and the state WAL, reporting every failure rather than stopping at the first. -// The WAL closes last, since SC replays through it. +// closeStores closes whichever of the stores are not nil, reporting every failure rather than stopping at +// the first. The hash vault closes after SC, which hands it the hashes of the blocks it drains, and the +// WAL closes last, since SC replays through it. // -// How long each of the three took is logged, since each drains its own write queue and waits on the +// How long each store took is logged, since each drains its own write queue and waits on the // compactions behind it, and those dominate the time a shutdown takes. -func (s *StateDB) Close() error { +func closeStores( + ss *evm.EVMStateStore, + sc *flatkv.CommitStore, + vault *hashvault.HashVault, + wal statewal.StateWAL, +) error { var errs error var timer utils.CloseTimer - if s.ss != nil { - if err := timer.Close("ss", s.ss.Close); err != nil { + if ss != nil { + if err := timer.Close("ss", ss.Close); err != nil { errs = errors.Join(errs, fmt.Errorf("close EVM state store: %w", err)) } } - if s.sc != nil { - if err := timer.Close("sc", s.sc.Close); err != nil { + if sc != nil { + if err := timer.Close("sc", sc.Close); err != nil { errs = errors.Join(errs, fmt.Errorf("close state commit store: %w", err)) } } - if s.wal != nil { - if err := timer.Close("wal", s.wal.Close); err != nil { + if vault != nil { + if err := timer.Close("hashvault", vault.Close); err != nil { + errs = errors.Join(errs, fmt.Errorf("close hash vault: %w", err)) + } + } + if wal != nil { + if err := timer.Close("wal", wal.Close); err != nil { errs = errors.Join(errs, fmt.Errorf("close state WAL: %w", err)) } } @@ -318,7 +252,7 @@ func (s *StateDB) CheckpointScheduler() *controller.CheckpointScheduler { return // PrunableStores returns the opened stores that can join a prune cycle. func (s *StateDB) PrunableStores() []controller.PrunableStore { - stores := make([]controller.PrunableStore, 0, 3) + stores := make([]controller.PrunableStore, 0, 4) if s.sc != nil { stores = append(stores, s.sc) } @@ -328,6 +262,9 @@ func (s *StateDB) PrunableStores() []controller.PrunableStore { if s.ss != nil { stores = append(stores, s.ss) } + if s.vault != nil { + stores = append(stores, s.vault) + } return stores } @@ -389,3 +326,23 @@ func (s *StateDB) RegisterHashListener(listener gigatypes.HashListener) (lthash. } return mostRecentHash, nil } + +// GetBlockHeight returns the version SC is on. +func (s *StateDB) GetBlockHeight() uint64 { + return uint64(s.sc.Version()) //nolint:gosec // a committed version is never negative +} + +// GetBlockHash returns the hash the hash vault holds for blockNumber. +func (s *StateDB) GetBlockHash(blockNumber uint64) ([32]byte, gigatypes.BlockHashStatus, error) { + hash, status, err := s.vault.Get(blockNumber) + if err != nil { + return hash, status, fmt.Errorf("get the hash of block %d: %w", blockNumber, err) + } + return hash, status, nil +} + +// PruneBlockHashesBelow permits the hash vault to delete the hashes of blocks below blockNumber. +func (s *StateDB) PruneBlockHashesBelow(blockNumber uint64) error { + s.vault.PruneBelow(blockNumber) + return nil +} diff --git a/sei-db/state_db/giga/state_db_hash_listener_test.go b/sei-db/state_db/giga/state_db_hash_listener_test.go index bd999fdb83..7201698382 100644 --- a/sei-db/state_db/giga/state_db_hash_listener_test.go +++ b/sei-db/state_db/giga/state_db_hash_listener_test.go @@ -16,20 +16,20 @@ import ( func TestRegisterHashListenerReachesTheLiveStateDB(t *testing.T) { stateDB, _, liveStateDB := newTestStateDB(t) - var seen []int64 + var seen []uint64 mostRecent, err := stateDB.RegisterHashListener( - func(_ context.Context, blockNumber int64, _ *lthash.BlockHash) error { + func(_ context.Context, blockNumber uint64, _ *lthash.BlockHash) error { seen = append(seen, blockNumber) return nil }) require.NoError(t, err) - require.Equal(t, int64(0), mostRecent.BlockNumber, "a fresh store has hashed nothing") + require.Equal(t, uint64(0), mostRecent.BlockNumber, "a fresh store has hashed nothing") require.NoError(t, stateDB.CommitStateChanges(1, changeset("key", "one"))) require.NoError(t, stateDB.CommitStateChanges(2, changeset("key", "two"))) require.NoError(t, liveStateDB.FlushHashes()) - require.Equal(t, []int64{1, 2}, seen) + require.Equal(t, []uint64{1, 2}, seen) } // The hash logger is wired in as a listener, and this is that wiring end to end: blocks committed diff --git a/sei-db/state_db/giga/state_db_hash_vault.go b/sei-db/state_db/giga/state_db_hash_vault.go new file mode 100644 index 0000000000..c9aca724e8 --- /dev/null +++ b/sei-db/state_db/giga/state_db_hash_vault.go @@ -0,0 +1,52 @@ +package giga + +import ( + "context" + "fmt" + + gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashvault" +) + +// hashVaultListener returns the listener that records each block's hash in vault. +func hashVaultListener(vault *hashvault.HashVault) gigatypes.HashListener { + return func(_ context.Context, blockNumber uint64, hash *lthash.BlockHash) error { + if hash.Global == nil { + return fmt.Errorf("record the hash of block %d: it carries no global hash", blockNumber) + } + if err := vault.Commit(blockNumber, hash.Global.Checksum()); err != nil { + return fmt.Errorf("record the hash of block %d in the hash vault: %w", blockNumber, err) + } + return nil + } +} + +// recordLoadedBlockHash records, or checks, the hash of the block SC opened on, so that the vault holds +// it once the open returns. SC dispatches that hash while it loads, before the vault is registered, and +// no replay re-dispatches it when SC was already on the WAL's head. +func recordLoadedBlockHash(sc *flatkv.CommitStore, vault *hashvault.HashVault) error { + if err := sc.FlushHashes(); err != nil { + return fmt.Errorf("wait for the replayed blocks to reach the hash vault: %w", err) + } + loaded := sc.Version() + if loaded == 0 { + // A store that has committed nothing has no block to record, and the first one it commits may be + // any height the chain starts at. + return nil + } + current, err := sc.RegisterHashListener(nil) + if err != nil { + return fmt.Errorf("read the hash of the loaded block %d: %w", loaded, err) + } + //nolint:gosec // a committed version is never negative + if current.BlockNumber != uint64(loaded) { + return fmt.Errorf("the state commit store is on block %d but last produced the hash of block %d", + loaded, current.BlockNumber) + } + if err := hashVaultListener(vault)(context.Background(), current.BlockNumber, ¤t); err != nil { + return fmt.Errorf("record the loaded block's hash: %w", err) + } + return nil +} diff --git a/sei-db/state_db/giga/state_db_hash_vault_test.go b/sei-db/state_db/giga/state_db_hash_vault_test.go new file mode 100644 index 0000000000..1f94d8d9c5 --- /dev/null +++ b/sei-db/state_db/giga/state_db_hash_vault_test.go @@ -0,0 +1,320 @@ +package giga + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/controller" + gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" + flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashvault" +) + +// vaultTestStores is the set of configs a StateDB under test opens, kept so the test can reopen it. +type vaultTestStores struct { + // Where SC and the state WAL live. + flatkvCfg *flatkvconfig.Config + + // SS stays disabled: it produces no hashes, so nothing here depends on it. + ssCfg config.StateStoreConfig + + // The schedule SC snapshots on when a test does not install one of its own. + checkpointCfg config.CheckpointConfig + + // Where the hash vault lives, and what it does on a mismatch. + hashVaultCfg config.HashVaultConfig +} + +// newVaultTestStores returns configs for a fresh StateDB whose vault halts on a mismatch. +func newVaultTestStores(t *testing.T) *vaultTestStores { + t.Helper() + flatkvCfg := flatkvconfig.DefaultTestConfig(t) + // The snapshots the rewinds land on are kept, since no collector runs here to prune them. + flatkvCfg.ExternalPruning = true + hashVaultCfg := config.DefaultHashVaultConfig() + hashVaultCfg.DataDir = filepath.Join(t.TempDir(), "hashvault") + hashVaultCfg.Fsync = false + return &vaultTestStores{ + flatkvCfg: flatkvCfg, + ssCfg: config.StateStoreConfig{Enable: false}, + checkpointCfg: config.CheckpointConfig{TimeInterval: time.Hour}, + hashVaultCfg: hashVaultCfg, + } +} + +// open opens the StateDB, failing the test if it cannot. +func (c *vaultTestStores) open(t *testing.T) *StateDB { + t.Helper() + db, err := c.openErr() + require.NoError(t, err) + return db +} + +// openErr opens the StateDB. +func (c *vaultTestStores) openErr() (*StateDB, error) { + return NewStateDB(context.Background(), c.flatkvCfg, c.ssCfg, c.checkpointCfg, c.hashVaultCfg, 0) +} + +// commitBlocks commits blocks first through last, each writing its own value, and snapshots SC at every +// one of them so any of them can be rewound to. +func commitBlocks(t *testing.T, db *StateDB, first int64, last int64) { + t.Helper() + require.NoError(t, db.SC().FlushSnapshots()) + db.SC().SetCheckpointScheduler(controller.NewCheckpointScheduler(config.CheckpointConfig{BlockInterval: 1})) + for block := first; block <= last; block++ { + require.NoError(t, db.CommitStateChanges(block, changeset("key", fmt.Sprintf("value-%d", block)))) + } + require.NoError(t, db.SC().FlushSnapshots()) + require.NoError(t, db.SC().FlushHashes()) +} + +// recordedHashes returns the vault's hashes for blocks first through last, failing the test if any is +// not recorded. +func recordedHashes(t *testing.T, db *StateDB, first uint64, last uint64) map[uint64][32]byte { + t.Helper() + hashes := make(map[uint64][32]byte) + for block := first; block <= last; block++ { + hash, status, err := db.GetBlockHash(block) + require.NoError(t, err) + require.Equal(t, gigatypes.BlockHashStatusFound, status, "block %d", block) + hashes[block] = hash + } + return hashes +} + +// requireStatus asserts the status GetBlockHash reports for blockNumber. +func requireStatus(t *testing.T, db *StateDB, blockNumber uint64, want gigatypes.BlockHashStatus) { + t.Helper() + _, status, err := db.GetBlockHash(blockNumber) + require.NoError(t, err) + require.Equal(t, want, status, "block %d", blockNumber) +} + +// Every block committed has its hash recorded, and the hash recorded is the one SC hands its listeners. +func TestStateDBRecordsTheHashOfEveryCommittedBlock(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + defer func() { require.NoError(t, db.Close()) }() + + var mu sync.Mutex + dispatched := make(map[uint64][32]byte) + _, err := db.RegisterHashListener(func(_ context.Context, blockNumber uint64, hash *lthash.BlockHash) error { + mu.Lock() + defer mu.Unlock() + dispatched[blockNumber] = hash.Global.Checksum() + return nil + }) + require.NoError(t, err) + + commitBlocks(t, db, 1, 5) + + require.Equal(t, uint64(5), db.GetBlockHeight()) + recorded := recordedHashes(t, db, 1, 5) + mu.Lock() + require.Equal(t, dispatched, recorded) + mu.Unlock() + requireStatus(t, db, 6, gigatypes.BlockHashStatusNotReady) +} + +// A reopened StateDB holds the hash of the block it opened on, whether or not anything was replayed. +func TestAReopenedStateDBHoldsTheLoadedBlocksHash(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 5) + before := recordedHashes(t, db, 1, 5) + require.NoError(t, db.Close()) + + reopened := stores.open(t) + defer func() { require.NoError(t, reopened.Close()) }() + require.Equal(t, uint64(5), reopened.GetBlockHeight()) + require.Equal(t, before, recordedHashes(t, reopened, 1, 5)) +} + +// A vault that lost its tail is behind SC, and the blocks it lost can only be hashed again by replaying +// them, so the open rewinds SC to the vault's newest block and replays from there. +func TestAVaultBehindSCIsRefilledByReplay(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 6) + before := recordedHashes(t, db, 1, 6) + require.NoError(t, db.Close()) + + require.NoError(t, hashvault.PruneAfter(stores.hashVaultCfg, 3)) + + reopened := stores.open(t) + defer func() { require.NoError(t, reopened.Close()) }() + require.Equal(t, uint64(6), reopened.GetBlockHeight()) + require.Equal(t, before, recordedHashes(t, reopened, 1, 6)) +} + +// An empty vault is refilled by rewinding SC the configured number of blocks and replaying them, so it +// holds the hashes of the newest blocks rather than just the loaded one. +func TestAnEmptyVaultIsRefilledByRewinding(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 8) + before := recordedHashes(t, db, 1, 8) + require.NoError(t, db.Close()) + + require.NoError(t, os.RemoveAll(stores.hashVaultCfg.DataDir)) + stores.hashVaultCfg.EmptyVaultRollbackBlocks = 3 + + reopened := stores.open(t) + defer func() { require.NoError(t, reopened.Close()) }() + require.Equal(t, uint64(8), reopened.GetBlockHeight()) + require.Equal(t, map[uint64][32]byte{6: before[6], 7: before[7], 8: before[8]}, + recordedHashes(t, reopened, 6, 8)) + requireStatus(t, reopened, 5, gigatypes.BlockHashStatusTooOld) +} + +// A rewind deeper than SC's snapshots and the WAL can replay is shortened to the deepest one they can. +func TestAnEmptyVaultRewindIsShortenedToWhatIsReachable(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 8) + before := recordedHashes(t, db, 1, 8) + require.NoError(t, db.Close()) + + require.NoError(t, os.RemoveAll(stores.hashVaultCfg.DataDir)) + stores.hashVaultCfg.EmptyVaultRollbackBlocks = 1000 + + reopened := stores.open(t) + defer func() { require.NoError(t, reopened.Close()) }() + require.Equal(t, uint64(8), reopened.GetBlockHeight()) + recorded := recordedHashes(t, reopened, 2, 8) + for block, hash := range recorded { + require.Equal(t, before[block], hash, "block %d", block) + } +} + +// With no rewind configured, an empty vault records only the loaded block's hash. +func TestAnEmptyVaultWithNoRewindRecordsOnlyTheLoadedBlock(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 4) + before := recordedHashes(t, db, 1, 4) + require.NoError(t, db.Close()) + + require.NoError(t, os.RemoveAll(stores.hashVaultCfg.DataDir)) + stores.hashVaultCfg.EmptyVaultRollbackBlocks = 0 + + reopened := stores.open(t) + defer func() { require.NoError(t, reopened.Close()) }() + require.Equal(t, map[uint64][32]byte{4: before[4]}, recordedHashes(t, reopened, 4, 4)) + requireStatus(t, reopened, 3, gigatypes.BlockHashStatusTooOld) +} + +// tamperLoadedBlock replaces the vault's hash for blockNumber with one SC will never produce. +func tamperLoadedBlock(t *testing.T, stores *vaultTestStores, blockNumber uint64) { + t.Helper() + cfg := stores.hashVaultCfg + cfg.HaltOnMismatch = false + vault, err := hashvault.Open(cfg) + require.NoError(t, err) + require.NoError(t, vault.Commit(blockNumber, [32]byte{0xEE})) + require.NoError(t, vault.Close()) +} + +// The loaded block's hash is checked against the vault even when nothing replays, so a vault that +// disagrees with the state on disk stops the open when halting is selected. +func TestAMismatchAtTheLoadedBlockFailsTheOpenWhenHalting(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 3) + require.NoError(t, db.Close()) + + tamperLoadedBlock(t, stores, 3) + + _, err := stores.openErr() + require.ErrorContains(t, err, "mismatch") +} + +// With halting off, the same disagreement is logged and the state's own hash replaces the vault's. +func TestAMismatchAtTheLoadedBlockIsReplacedWhenNotHalting(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 3) + before := recordedHashes(t, db, 1, 3) + require.NoError(t, db.Close()) + + tamperLoadedBlock(t, stores, 3) + stores.hashVaultCfg.HaltOnMismatch = false + + reopened := stores.open(t) + defer func() { require.NoError(t, reopened.Close()) }() + require.Equal(t, before, recordedHashes(t, reopened, 1, 3)) +} + +// A rollback leaves the vault alone: the blocks above the target are re-executed, and it is exactly their +// recorded hashes the re-execution is held to. +func TestARollbackKeepsTheHashesAboveItsTarget(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 5) + before := recordedHashes(t, db, 1, 5) + require.NoError(t, db.Close()) + + rolledBack, err := NewStateDB(context.Background(), + stores.flatkvCfg, stores.ssCfg, stores.checkpointCfg, stores.hashVaultCfg, 3) + require.NoError(t, err) + defer func() { require.NoError(t, rolledBack.Close()) }() + + require.Equal(t, uint64(3), rolledBack.GetBlockHeight()) + require.Equal(t, before, recordedHashes(t, rolledBack, 1, 5)) + + commitBlocks(t, rolledBack, 4, 5) + require.Equal(t, before, recordedHashes(t, rolledBack, 1, 5), "re-executing the same blocks matches") +} + +// Re-executing a block into a different state is the equivocation the vault exists to stop. The vault is +// SC's first listener, so a hash it refuses reaches no listener registered after it. +func TestADifferentReexecutionIsRefusedBeforeAnyOtherListenerSeesIt(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 5) + require.NoError(t, db.Close()) + + rolledBack, err := NewStateDB(context.Background(), + stores.flatkvCfg, stores.ssCfg, stores.checkpointCfg, stores.hashVaultCfg, 3) + require.NoError(t, err) + defer func() { _ = rolledBack.Close() }() + + var mu sync.Mutex + var seen []uint64 + record := func(_ context.Context, blockNumber uint64, _ *lthash.BlockHash) error { + mu.Lock() + defer mu.Unlock() + seen = append(seen, blockNumber) + return nil + } + _, err = rolledBack.RegisterHashListener(record) + require.NoError(t, err) + + require.NoError(t, rolledBack.CommitStateChanges(4, changeset("key", "a different value"))) + require.ErrorContains(t, rolledBack.SC().FlushHashes(), "mismatch") + mu.Lock() + require.Empty(t, seen, "a hash the vault refused must reach no later listener") + mu.Unlock() +} + +// The vault joins the prune cycle, which is how the storage garbage collector's permission reaches it. +func TestTheVaultJoinsThePruneCycle(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + defer func() { require.NoError(t, db.Close()) }() + + var names []string + for _, store := range db.PrunableStores() { + names = append(names, store.Name()) + } + require.Contains(t, names, "HashVault") +} diff --git a/sei-db/state_db/giga/state_db_recovery.go b/sei-db/state_db/giga/state_db_recovery.go new file mode 100644 index 0000000000..43558e7cc2 --- /dev/null +++ b/sei-db/state_db/giga/state_db_recovery.go @@ -0,0 +1,263 @@ +package giga + +import ( + "fmt" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashvault" + "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/evm" + "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" +) + +// What the stores hold on disk before any of them opens. +type recoverySurvey struct { + walStored bool + + // Only meaningful when walStored is true. + walFirst uint64 + + // Only meaningful when walStored is true. + walLast uint64 + + // Only meaningful when vaultRecorded is true. + vaultHead uint64 + + vaultRecorded bool + + // The live state DB's snapshot versions, lowest first. + scSnapshots []uint64 +} + +// Where each store is put before the stores open. +type recoveryPlan struct { + // The height the open replays every store up to, or 0 when there is nothing to move or replay. + head uint64 + + // The live state DB is put on its newest snapshot at or below this when it holds state above it. + scTarget uint64 + + // The historical state DB is put on its newest snapshot at or below this when it holds state above it. + ssTarget uint64 + + // Whether the snapshots and WAL blocks above head are dropped. + rollback bool + + // Whether an empty hash vault's refill was cut short by how far back the snapshots and WAL reach. + emptyVaultRefillShortened bool + + // Whether an empty hash vault cannot be refilled, and records only the loaded block's hash. + emptyVaultRefillUnreachable bool +} + +// Puts each store where it belongs, as decided from what the stores hold on disk. Every store must be +// closed. +func recoverStores( + flatkvCfg *flatkvconfig.Config, + ssCfg config.StateStoreConfig, + hashVaultCfg config.HashVaultConfig, + rollbackTo uint64, +) error { + survey, err := surveyStores(flatkvCfg, hashVaultCfg) + if err != nil { + return fmt.Errorf("survey the stores: %w", err) + } + plan, err := planRecovery(survey, rollbackTo, hashVaultCfg.EmptyVaultRollbackBlocks) + if err != nil { + return fmt.Errorf("plan the recovery: %w", err) + } + logRecoveryPlan(plan, survey, hashVaultCfg.EmptyVaultRollbackBlocks) + if err := applyRecoveryPlan(flatkvCfg, ssCfg, survey.walFirst, plan); err != nil { + return fmt.Errorf("apply the recovery plan: %w", err) + } + return nil +} + +// Reads what the state WAL, the hash vault and the live state DB's snapshots hold. Every store must be +// closed. +func surveyStores(flatkvCfg *flatkvconfig.Config, hashVaultCfg config.HashVaultConfig) (recoverySurvey, error) { + // This takes the WAL directory's exclusive lock, so it only works before the WAL opens. + walStored, walFirst, walLast, err := statewal.GetRange(flatkv.StateWALConfig(flatkvCfg.DataDir)) + if err != nil { + return recoverySurvey{}, fmt.Errorf("survey the state WAL: %w", err) + } + versions, err := flatkv.SnapshotVersions(flatkvCfg.DataDir) + if err != nil { + return recoverySurvey{}, fmt.Errorf("survey the state commit store's snapshots: %w", err) + } + scSnapshots := make([]uint64, len(versions)) + for i, version := range versions { + scSnapshots[i] = uint64(version) //nolint:gosec // a snapshot version is never negative + } + _, vaultHead, vaultRecorded, err := hashvault.StoredRange(hashVaultCfg) + if err != nil { + return recoverySurvey{}, fmt.Errorf("survey the hash vault: %w", err) + } + return recoverySurvey{ + walStored: walStored, + walFirst: walFirst, + walLast: walLast, + vaultHead: vaultHead, + vaultRecorded: vaultRecorded, + scSnapshots: scSnapshots, + }, nil +} + +// Decides where each store is put: on the WAL's head, or on rollbackTo when it is not 0, which must be at +// or below the head. The live state DB goes lower when the hash vault is missing hashes only a replay can +// produce. +func planRecovery( + survey recoverySurvey, + rollbackTo uint64, + emptyVaultRollbackBlocks uint64, +) (recoveryPlan, error) { + if !survey.walStored { + // An empty WAL says nothing about where state belongs, so it moves nothing. + if rollbackTo > 0 { + return recoveryPlan{}, fmt.Errorf("cannot roll back to %d: the state WAL is empty, so no "+ + "replay reaches the target", rollbackTo) + } + return recoveryPlan{}, nil + } + // A crash can lose the WAL's unflushed tail while the state above it survives. Those blocks are + // re-executed, so that state is discarded. + plan := recoveryPlan{head: survey.walLast} + if rollbackTo > 0 { + if survey.walLast < rollbackTo { + return recoveryPlan{}, fmt.Errorf("cannot roll back to %d: the state WAL ends at %d, so no "+ + "replay reaches the target", rollbackTo, survey.walLast) + } + plan.head = rollbackTo + plan.rollback = true + } + plan.scTarget = plan.head + plan.ssTarget = plan.head + + if plan.head == 0 || plan.head < survey.walFirst { + // No WAL block is left to replay once the plan is applied, so a lower SC would stay lower. + return plan, nil + } + planHashVaultRefill(&plan, survey, emptyVaultRollbackBlocks) + return plan, nil +} + +// Lowers the live state DB's target so the replay produces the hashes the vault is missing. plan.head must +// be a height the WAL can replay up to. +func planHashVaultRefill(plan *recoveryPlan, survey recoverySurvey, emptyVaultRollbackBlocks uint64) { + if survey.vaultRecorded { + if survey.vaultHead < plan.head { + // The rewind lands on the snapshot at or below the target, so a target of 1 still reaches the + // state a vault holding only block 0 needs. + plan.scTarget = max(survey.vaultHead, 1) + } + return + } + + if emptyVaultRollbackBlocks == 0 { + return + } + earliest, reachable := earliestSCRewindTarget(survey.scSnapshots, survey.walFirst) + if !reachable { + plan.emptyVaultRefillUnreachable = true + return + } + target := earliest + if emptyVaultRollbackBlocks < plan.head && plan.head-emptyVaultRollbackBlocks >= earliest { + target = plan.head - emptyVaultRollbackBlocks + } else { + plan.emptyVaultRefillShortened = true + } + plan.scTarget = min(target, plan.head) +} + +// Returns the lowest height the live state DB can be rewound to and replayed from a WAL starting at +// walFirst, and false when there is none. +func earliestSCRewindTarget(scSnapshots []uint64, walFirst uint64) (uint64, bool) { + for _, version := range scSnapshots { + if version+1 >= walFirst { + // A rewind target of 0 is refused, and 1 lands on a snapshot at 0 just the same. + return max(version, 1), true + } + } + return 0, false +} + +// Reports a plan that puts the live state DB below the head, and why. +func logRecoveryPlan(plan recoveryPlan, survey recoverySurvey, emptyVaultRollbackBlocks uint64) { + switch { + case plan.emptyVaultRefillUnreachable: + logger.Warn("The hash vault is empty and no snapshot of the state commit store can be replayed "+ + "from the state WAL, so it records only the loaded block's hash", + "walFirst", survey.walFirst, "walLast", survey.walLast) + case plan.emptyVaultRefillShortened: + logger.Warn("The hash vault is empty and the requested rewind reaches further back than the state "+ + "commit store's snapshots and the state WAL can replay, so it is shortened", + "requestedBlocks", emptyVaultRollbackBlocks, "head", plan.head, "target", plan.scTarget) + } + if plan.scTarget < plan.head { + logger.Info("Rewinding the state commit store so the replay refills the hash vault", + "target", plan.scTarget, "head", plan.head, + "vaultRecorded", survey.vaultRecorded, "vaultHead", survey.vaultHead) + } +} + +// Puts the live and historical state DBs on the plan's targets and, for a rollback, drops the snapshots and +// WAL blocks above the head. The stores must be closed, and walFirst is the WAL's lowest block when the +// plan was made. A refused target leaves the WAL uncut. +func applyRecoveryPlan( + flatkvCfg *flatkvconfig.Config, + ssCfg config.StateStoreConfig, + walFirst uint64, + plan recoveryPlan, +) error { + if plan.head == 0 { + return nil + } + //nolint:gosec // WAL block numbers never approach the int64 ceiling + if _, err := flatkv.DiscardStateAbove(flatkvCfg.DataDir, int64(plan.scTarget), int64(walFirst)); err != nil { + return plan.refusal(fmt.Errorf("the state commit store cannot reach %d: %w", plan.scTarget, err)) + } + if err := discardSSAbove(ssCfg, walFirst, plan.ssTarget); err != nil { + return plan.refusal(err) + } + if !plan.rollback { + return nil + } + if err := dropSnapshotsAbove(flatkvCfg, ssCfg, plan.head); err != nil { + return fmt.Errorf("cannot roll back to %d: %w", plan.head, err) + } + // Last, so that an interruption leaves the WAL still above the target and a restart comes back here. A + // live WAL prunes only from its start, so the tail is cut through the directory, with no WAL open on it. + if err := statewal.PruneAfter(flatkv.StateWALConfig(flatkvCfg.DataDir), plan.head); err != nil { + return fmt.Errorf("cannot roll back to %d: truncate state WAL: %w", plan.head, err) + } + return nil +} + +// Wraps a store's refusal of its target in the rollback it refused, or in the WAL head when no rollback +// was asked for. +func (p recoveryPlan) refusal(err error) error { + if p.scTarget < p.head { + err = fmt.Errorf("rewinding the state commit store to %d to refill the hash vault: %w", p.scTarget, err) + } + if p.rollback { + return fmt.Errorf("cannot roll back to %d: %w", p.head, err) + } + return fmt.Errorf("cannot open on the state WAL's head %d: %w", p.head, err) +} + +// Puts an enabled historical state DB on its newest snapshot at or below target when it holds state above +// it. The store must be closed, and a target the WAL cannot replay it back up to is refused. +func discardSSAbove(ssCfg config.StateStoreConfig, walFirst uint64, target uint64) error { + if !ssCfg.Enable { + return nil + } + snapshotRoot := utils.GetStateStoreSnapshotsSiblingPath(ssCfg.EVMDBDirectory) + //nolint:gosec // WAL block numbers never approach the int64 ceiling + if _, err := evm.DiscardStateAbove(ssCfg, snapshotRoot, int64(target), int64(walFirst)); err != nil { + return fmt.Errorf("the EVM state store cannot reach %d: %w", target, err) + } + return nil +} diff --git a/sei-db/state_db/giga/state_db_recovery_test.go b/sei-db/state_db/giga/state_db_recovery_test.go new file mode 100644 index 0000000000..dab4f0f205 --- /dev/null +++ b/sei-db/state_db/giga/state_db_recovery_test.go @@ -0,0 +1,172 @@ +package giga + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Where each store is put is decided once, from what the stores hold, so every rule the open follows is +// visible here without opening anything. +func TestPlanRecovery(t *testing.T) { + snapshots := []uint64{0, 4, 8} + for _, tc := range []struct { + name string + survey recoverySurvey + rollbackTo uint64 + depth uint64 + want recoveryPlan + }{ + { + name: "a plain open puts every store on the WAL's head", + survey: recoverySurvey{ + walStored: true, + walFirst: 1, + walLast: 10, + vaultRecorded: true, + vaultHead: 10, + scSnapshots: snapshots, + }, + want: recoveryPlan{head: 10, scTarget: 10, ssTarget: 10}, + }, + { + name: "an empty WAL moves nothing", + survey: recoverySurvey{ + vaultRecorded: true, + vaultHead: 10, + scSnapshots: snapshots, + }, + want: recoveryPlan{}, + }, + { + name: "a rollback puts every store on its target", + survey: recoverySurvey{ + walStored: true, + walFirst: 1, + walLast: 10, + vaultRecorded: true, + vaultHead: 10, + scSnapshots: snapshots, + }, + rollbackTo: 6, + want: recoveryPlan{head: 6, scTarget: 6, ssTarget: 6, rollback: true}, + }, + { + name: "a vault ahead of the head keeps its hashes and moves nothing", + survey: recoverySurvey{ + walStored: true, + walFirst: 1, + walLast: 10, + vaultRecorded: true, + vaultHead: 12, + scSnapshots: snapshots, + }, + want: recoveryPlan{head: 10, scTarget: 10, ssTarget: 10}, + }, + { + name: "a vault behind the head puts SC alone on the vault's newest block", + survey: recoverySurvey{ + walStored: true, + walFirst: 1, + walLast: 10, + vaultRecorded: true, + vaultHead: 7, + scSnapshots: snapshots, + }, + want: recoveryPlan{head: 10, scTarget: 7, ssTarget: 10}, + }, + { + name: "a vault behind a rollback target is caught up the same way", + survey: recoverySurvey{ + walStored: true, + walFirst: 1, + walLast: 10, + vaultRecorded: true, + vaultHead: 3, + scSnapshots: snapshots, + }, + rollbackTo: 6, + want: recoveryPlan{head: 6, scTarget: 3, ssTarget: 6, rollback: true}, + }, + { + name: "a vault holding only block 0 is caught up from 1", + survey: recoverySurvey{ + walStored: true, + walFirst: 1, + walLast: 10, + vaultRecorded: true, + vaultHead: 0, + scSnapshots: snapshots, + }, + want: recoveryPlan{head: 10, scTarget: 1, ssTarget: 10}, + }, + { + name: "an empty vault puts SC the configured depth below the head", + survey: recoverySurvey{ + walStored: true, + walFirst: 1, + walLast: 10, + scSnapshots: snapshots, + }, + depth: 3, + want: recoveryPlan{head: 10, scTarget: 7, ssTarget: 10}, + }, + { + name: "an empty vault with no depth configured moves nothing", + survey: recoverySurvey{ + walStored: true, + walFirst: 1, + walLast: 10, + scSnapshots: snapshots, + }, + want: recoveryPlan{head: 10, scTarget: 10, ssTarget: 10}, + }, + { + name: "an empty vault's rewind is shortened to the lowest snapshot the WAL replays from", + survey: recoverySurvey{ + walStored: true, + walFirst: 5, + walLast: 10, + scSnapshots: snapshots, + }, + depth: 1000, + want: recoveryPlan{head: 10, scTarget: 4, ssTarget: 10, emptyVaultRefillShortened: true}, + }, + { + name: "an empty vault with no replayable snapshot records only the loaded block", + survey: recoverySurvey{ + walStored: true, + walFirst: 9, + walLast: 10, + scSnapshots: []uint64{0, + 4}, + }, + depth: 3, + want: recoveryPlan{head: 10, scTarget: 10, ssTarget: 10, emptyVaultRefillUnreachable: true}, + }, + { + name: "a rollback that empties the WAL leaves nothing to refill the vault from", + survey: recoverySurvey{ + walStored: true, + walFirst: 5, + walLast: 10, + scSnapshots: snapshots, + }, + rollbackTo: 4, + depth: 3, + want: recoveryPlan{head: 4, scTarget: 4, ssTarget: 4, rollback: true}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := planRecovery(tc.survey, tc.rollbackTo, tc.depth) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +// A rollback above the WAL's head is a target no replay reaches, refused before anything is planned. +func TestPlanRecoveryRefusesARollbackAboveTheWAL(t *testing.T) { + _, err := planRecovery(recoverySurvey{walStored: true, walFirst: 1, walLast: 3}, 5, 0) + require.ErrorContains(t, err, "the state WAL ends at 3") +} diff --git a/sei-db/state_db/giga/state_db_replay.go b/sei-db/state_db/giga/state_db_replay.go index 01ffd48573..a9969f11ff 100644 --- a/sei-db/state_db/giga/state_db_replay.go +++ b/sei-db/state_db/giga/state_db_replay.go @@ -5,174 +5,124 @@ import ( "fmt" "time" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/evm" + "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" ) // replayLogInterval bounds how often a running replay reports how far it has got. const replayLogInterval = 30 * time.Second -// rewindTo puts whichever of SC and SS holds state above target on its newest snapshot at or below it, -// drops every snapshot of both above target, and cuts the WAL's tail to it. All three stores must be -// closed, and a store holding nothing above target is left where it is, for the replay to carry forward. -// -// A target the surviving snapshots and the WAL cannot span is refused before the WAL is cut, so every -// target this one could reach is still reachable on a retry. SS is asked once SC has moved, so a -// refusal from SS leaves SC on its snapshot and a retry replays from there. -func (s *StateDB) rewindTo(target int64) error { - wal, err := s.storedWALRange() - if err != nil { - return err - } - if wal.last < target { - return fmt.Errorf("cannot roll back to %d: the state WAL ends at %d, so no replay reaches the "+ - "target", target, wal.last) - } - - // First, so that a refusal from SC comes back with every snapshot still on disk. Once SC has moved, - // its own snapshots above where it landed are gone. - if err := s.discardStateAbove(wal, target); err != nil { - return fmt.Errorf("cannot roll back to %d: %w", target, err) - } - if err := s.dropSnapshotsAbove(target); err != nil { - return err - } - // Last, so that an interruption leaves the WAL still above target and a restart comes back here. - return s.truncateWAL(target) -} - -// discardStateAboveTheWAL puts each store back on the WAL's head when it sits above it, onto its -// newest snapshot at or below the head for the replay to carry forward. Every store must be closed. -// -// A commit writes the WAL unflushed, so a crash can lose its tail while the state committed above that -// tail survives. Those blocks are re-executed from the block store, which a store still holding them -// cannot accept, so the state above the WAL is dropped rather than kept. -func (s *StateDB) discardStateAboveTheWAL(wal storedWALRange) error { - head := wal.last - if head == 0 { - // An empty WAL says nothing about where state belongs: one pruned away behind a snapshot leaves - // the state it covered as the only record of it. - return nil - } - if err := s.discardStateAbove(wal, head); err != nil { - // Named for the open, not for a rollback: nobody asked for one, and an operator sent looking for - // the rollback they did not run is an operator not looking at the WAL head that refused. - return fmt.Errorf("cannot open on the state WAL's head %d: %w", head, err) - } - return nil -} - -// discardStateAbove puts whichever of SC and SS holds state above target onto its newest snapshot at or -// below it. Both stores must be closed, and a store holding nothing above target is left where it is, -// for the replay to carry forward. -// -// Each store is handed the WAL's first block and refuses, without moving, a target this WAL cannot -// replay it back up to. SC is put back first, so a refusal from SS can leave SC already rewound. -func (s *StateDB) discardStateAbove(wal storedWALRange, target int64) error { - if _, err := flatkv.DiscardStateAbove(s.flatkvCfg.DataDir, target, wal.first); err != nil { - return fmt.Errorf("the state commit store cannot reach %d: %w", target, err) - } - if !s.ssCfg.Enable { - return nil - } - if _, err := evm.DiscardStateAbove( - s.ssCfg, s.ssSnapshotRoot(), target, wal.first); err != nil { - return fmt.Errorf("the EVM state store cannot reach %d: %w", target, err) - } - return nil -} - -// dropSnapshotsAbove removes the snapshots of SC and SS above target. +// dropSnapshotsAbove removes the snapshots of SC and SS above target. Both stores must be closed. // // It runs whether or not either store is above target, because an interrupted rollback leaves exactly a // store that is not: it reads as the snapshot it was repointed at, with the branch above it still on // disk. Left there, a later rollback lands on a snapshot from the branch this one abandoned. -func (s *StateDB) dropSnapshotsAbove(target int64) error { - if err := flatkv.DropSnapshotsAbove(s.flatkvCfg.DataDir, target); err != nil { +func dropSnapshotsAbove(flatkvCfg *flatkvconfig.Config, ssCfg config.StateStoreConfig, target uint64) error { + //nolint:gosec // a WAL block number never approaches the int64 ceiling + if err := flatkv.DropSnapshotsAbove(flatkvCfg.DataDir, int64(target)); err != nil { return fmt.Errorf("cannot roll back the state commit store to %d: %w", target, err) } - if !s.ssCfg.Enable { + if !ssCfg.Enable { return nil } - if err := evm.DropSnapshotsAbove(s.ssSnapshotRoot(), target); err != nil { + snapshotRoot := utils.GetStateStoreSnapshotsSiblingPath(ssCfg.EVMDBDirectory) + //nolint:gosec // a WAL block number never approaches the int64 ceiling + if err := evm.DropSnapshotsAbove(snapshotRoot, int64(target)); err != nil { return fmt.Errorf("cannot roll back the EVM state store to %d: %w", target, err) } return nil } // catchUpToWAL replays the WAL into SC and SS up to the last block it holds, which is the height state -// committed to. An empty WAL leaves both stores where they are. +// committed to. ss is nil when SS is disabled. An empty WAL leaves both stores where they are. // // A commit writes the WAL before either store, so a crash between the two leaves one of them a block // behind. Committing from behind the WAL is rejected, so this is what makes an opened StateDB able to // commit. -func (s *StateDB) catchUpToWAL(ctx context.Context) error { - wal, err := s.openWALRange() +func catchUpToWAL( + ctx context.Context, + sc *flatkv.CommitStore, + ss *evm.EVMStateStore, + wal statewal.StateWAL, +) error { + stored, _, last, err := wal.GetStoredRange() if err != nil { - return err + return fmt.Errorf("find the head to catch up to: %w", err) } - if wal.last == 0 { - // Neither store is carried forward, for the reason discardStateAboveTheWAL gives: with no head to - // measure against, a working copy above the current snapshot is the only record of the blocks it - // holds, and dropping it on one store alone would leave the two at different heights. A rollback - // that empties the WAL brings both down in rewindTo, where the target says where they belong. + if !stored { + // Neither store is carried forward, for the reason planRecovery gives: with no head to measure + // against, a working copy above the current snapshot is the only record of the blocks it holds, and + // dropping it on one store alone would leave the two at different heights. A rollback that empties + // the WAL brings both down in applyRecoveryPlan, where the target says where they belong. // // An interrupted commit is still repaired, since the disagreement it leaves needs no head to be // recognised. Nothing below reaches the repair catchUpTo runs. - if err := s.sc.RebuildIfTorn(); err != nil { + if err := sc.RebuildIfTorn(); err != nil { return fmt.Errorf("repair the state commit store's working copy: %w", err) } return nil } - head := wal.last - if err := s.catchUpTo(ctx, head); err != nil { - return err + head := int64(last) //nolint:gosec // a WAL block number never approaches the int64 ceiling + if err := catchUpTo(ctx, sc, ss, wal, head); err != nil { + return fmt.Errorf("catch up to the state WAL's head %d: %w", head, err) } - if err := s.matchHeight(head); err != nil { - // Named for the open, as discardStateAboveTheWAL's refusals are: no rollback ran, and an - // operator sent looking for one is an operator not looking at the head that was not reached. + if err := matchHeight(sc, ss, wal, head); err != nil { + // Named for the open, as a plan's refusals are when no rollback ran: an operator sent looking for + // one is an operator not looking at the head that was not reached. return fmt.Errorf("cannot open on the state WAL's head %d: %w", head, err) } return nil } -// catchUpTo replays the WAL into SC and SS up to target. +// catchUpTo replays the WAL into SC and SS up to target. ss is nil when SS is disabled. // // One pass feeds both. It spans from the lower of their two versions, and each block goes only to the // store still below it, so the WAL is read once rather than once per store. -func (s *StateDB) catchUpTo(ctx context.Context, target int64) error { +func catchUpTo( + ctx context.Context, + sc *flatkv.CommitStore, + ss *evm.EVMStateStore, + wal statewal.StateWAL, + target int64, +) error { // Ahead of the pass, which is what erases the evidence it works from, and here rather than in the // open because every replay of this WAL comes through this function. - if err := s.sc.RebuildIfUnreachable(target); err != nil { + if err := sc.RebuildIfUnreachable(target); err != nil { return fmt.Errorf("rebuild the state commit store's working copy: %w", err) } - scFrom := s.sc.Version() - ssFrom, ssReplays, err := s.ssReplayStart(target) + scFrom := sc.Version() + ssFrom, ssReplays, err := ssReplayStart(ss, wal, target) if err != nil { - return err + return fmt.Errorf("find where the EVM state store replays from: %w", err) } from := scFrom if ssReplays { from = min(from, ssFrom) } - s.logReplayPlan(scFrom, ssFrom, ssReplays, target) + logReplayPlan(ss, scFrom, ssFrom, ssReplays, target) - if err := s.replay(ctx, from, target, func(block int64, changesets []*proto.NamedChangeSet) error { + if err := replay(ctx, wal, from, target, func(block int64, changesets []*proto.NamedChangeSet) error { if block > scFrom { // SC owns no WAL, so re-committing a block read from this one appends nothing. It does run // SC's commit path, so the checkpoint schedule is asked at each block SC takes. - if err := s.sc.CommitStateChanges(block, changesets); err != nil { - return err + if err := sc.CommitStateChanges(block, changesets); err != nil { + return fmt.Errorf("commit the block to the state commit store: %w", err) } } if ssReplays && block > ssFrom { - return s.ss.ApplyReplayedBlock(block, changesets) + if err := ss.ApplyReplayedBlock(block, changesets); err != nil { + return fmt.Errorf("apply the block to the EVM state store: %w", err) + } } return nil }); err != nil { - return err + return fmt.Errorf("replay the state WAL up to %d: %w", target, err) } return nil } @@ -182,13 +132,13 @@ func (s *StateDB) catchUpTo(ctx context.Context, target int64) error { // // It is logged even when nothing is replayed, because an open that had no catching up to do is // otherwise indistinguishable from one still working through a long pass. -func (s *StateDB) logReplayPlan(scFrom int64, ssFrom int64, ssReplays bool, target int64) { +func logReplayPlan(ss *evm.EVMStateStore, scFrom int64, ssFrom int64, ssReplays bool, target int64) { fields := append([]any{"target", target}, replayPlanFields("sc", scFrom, target)...) - if s.ss != nil { + if ss != nil { // A store left out of the pass replays nothing, which is a range ending where it already sits. to := target if !ssReplays { - ssFrom = s.ss.GetLatestVersion() + ssFrom = ss.GetLatestVersion() to = ssFrom } fields = append(fields, replayPlanFields("ss", ssFrom, to)...) @@ -214,12 +164,14 @@ func replayPlanFields(store string, from int64, to int64) []any { // // A cancelled ctx stops the replay between blocks and is reported as an error. The blocks already // applied stay applied, and the next open resumes from the version they left behind. -func (s *StateDB) replay( +func replay( ctx context.Context, - from, target int64, + wal statewal.StateWAL, + from int64, + target int64, apply func(int64, []*proto.NamedChangeSet) error, ) error { - stored, first, last, err := s.wal.GetStoredRange() + stored, first, last, err := wal.GetStoredRange() if err != nil { return fmt.Errorf("read state WAL range: %w", err) } @@ -237,7 +189,7 @@ func (s *StateDB) replay( "are missing (data loss or corruption)", first, start, start, first-1) } - it, err := s.wal.Iterator(start, end) + it, err := wal.Iterator(start, end) if err != nil { return fmt.Errorf("state WAL iterator [%d,%d]: %w", start, end, err) } @@ -318,18 +270,19 @@ func estimate(remaining int64, rate float64) time.Duration { } // ssReplayStart returns the version SS replays forward from, and whether it replays at all. SS is left -// out when it is disabled, already on target, or empty with a WAL that can no longer rebuild it. -func (s *StateDB) ssReplayStart(target int64) (from int64, replays bool, err error) { - if s.ss == nil { +// out when it is disabled (ss is nil), already on target, or empty with a WAL that can no longer rebuild +// it. +func ssReplayStart(ss *evm.EVMStateStore, wal statewal.StateWAL, target int64) (from int64, replays bool, err error) { + if ss == nil { return 0, false, nil } - from = s.ss.GetLatestVersion() + from = ss.GetLatestVersion() if from >= target { return 0, false, nil } - fillForward, err := s.ssFillsForward() + fillForward, err := ssFillsForward(ss, wal) if err != nil { - return 0, false, err + return 0, false, fmt.Errorf("decide whether the EVM state store fills forward: %w", err) } if fillForward { logger.Info("EVM state store left empty to fill forward: it holds no history and the state WAL "+ @@ -343,37 +296,37 @@ func (s *StateDB) ssReplayStart(target int64) (from int64, replays bool, err err // treatment recoveryTarget gives an empty receipt store. It covers a store with no history of its own // behind a WAL that has had a retention cut, where no replay rebuilds it and the alternative is // refusing to start over a store that is merely new. -func (s *StateDB) ssFillsForward() (bool, error) { - if s.ss == nil { +func ssFillsForward(ss *evm.EVMStateStore, wal statewal.StateWAL) (bool, error) { + if ss == nil { return false, nil } - wal, err := s.openWALRange() + stored, first, _, err := wal.GetStoredRange() if err != nil { - return false, err + return false, fmt.Errorf("find whether the state WAL reaches block 1: %w", err) } - return s.ss.GetLatestVersion() == 0 && (wal.last == 0 || wal.first > 1), nil + return ss.GetLatestVersion() == 0 && (!stored || first > 1), nil } -// matchHeight checks SC and SS against blockNum and reports the one that is not on it. An SS left empty -// to fill forward is not held to blockNum. +// matchHeight checks SC and SS against blockNum and reports the one that is not on it. ss is nil when SS +// is disabled, and an SS left empty to fill forward is not held to blockNum. // // The error names no path, since both the open and a rollback converge here; each caller supplies the // height it asked for. -func (s *StateDB) matchHeight(blockNum int64) error { - if got := s.sc.Version(); got != blockNum { +func matchHeight(sc *flatkv.CommitStore, ss *evm.EVMStateStore, wal statewal.StateWAL, blockNum int64) error { + if got := sc.Version(); got != blockNum { return fmt.Errorf("the state commit store landed on %d", got) } - if s.ss == nil { + if ss == nil { return nil } - got := s.ss.GetLatestVersion() + got := ss.GetLatestVersion() if got == blockNum { return nil } if got == 0 { - fillForward, err := s.ssFillsForward() + fillForward, err := ssFillsForward(ss, wal) if err != nil { - return err + return fmt.Errorf("check the EVM state store's height: %w", err) } if fillForward { return nil diff --git a/sei-db/state_db/giga/state_db_replay_test.go b/sei-db/state_db/giga/state_db_replay_test.go index cfd41c9e8c..d64955c8eb 100644 --- a/sei-db/state_db/giga/state_db_replay_test.go +++ b/sei-db/state_db/giga/state_db_replay_test.go @@ -7,53 +7,24 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/config" - flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/evm" "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" "github.com/stretchr/testify/require" ) // SS keeps no changelog of its own under giga, the state WAL being what catchUpTo replays into it. -// The absence is pinned here rather than left to the config, since recovery rests on it. +// The absence is pinned here rather than left to the config, since recovery rests on it: a config that +// leaves the changelog on still opens without one. func TestGigaOpensSSWithoutAChangelog(t *testing.T) { - newStateDB := func(t *testing.T) *StateDB { - t.Helper() - ssCfg := config.DefaultStateStoreConfig() - ssCfg.Enable = true - ssCfg.EVMDBDirectory = filepath.Join(t.TempDir(), "ss") - return &StateDB{ - flatkvCfg: flatkvconfig.DefaultTestConfig(t), - // As the constructors settle it, which is what makes both paths below agree. - ssCfg: stateStoreConfigFor(ssCfg), - } - } - - t.Run("opened to commit", func(t *testing.T) { - s := newStateDB(t) - require.NoError(t, s.openSS()) - t.Cleanup(func() { _ = s.ss.Close() }) - requireNoSSChangelog(t, s.ssCfg.EVMDBDirectory) - }) - - // The rollback path opens the same databases through DiscardStateAbove rather than openSS, so a - // config settled per-open would miss it. StoredVersions opens nothing when the directory is - // absent, so the store has to exist first. - t.Run("opened to roll back", func(t *testing.T) { - s := newStateDB(t) - require.NoError(t, s.openSS()) - require.NoError(t, s.ss.Close()) - - require.NoError(t, s.discardStateAbove(storedWALRange{first: 1, last: 9}, 7)) - requireNoSSChangelog(t, s.ssCfg.EVMDBDirectory) - }) -} - -// TestStateStoreConfigForDisablesTheInternalWAL pins what the constructors apply, every path that -// opens SS reading the config they settled rather than disabling the log for itself. -func TestStateStoreConfigForDisablesTheInternalWAL(t *testing.T) { - handedIn := config.DefaultStateStoreConfig() - require.False(t, handedIn.DisableInternalWAL, "a caller is not expected to have set it") - require.True(t, stateStoreConfigFor(handedIn).DisableInternalWAL) + stores := newVaultTestStores(t) + stores.ssCfg = config.DefaultStateStoreConfig() + stores.ssCfg.Enable = true + stores.ssCfg.EVMDBDirectory = filepath.Join(t.TempDir(), "ss") + require.False(t, stores.ssCfg.DisableInternalWAL, "a caller is not expected to have set it") + + db := stores.open(t) + require.NoError(t, db.Close()) + requireNoSSChangelog(t, stores.ssCfg.EVMDBDirectory) } func requireNoSSChangelog(t *testing.T, evmDBDirectory string) { @@ -68,14 +39,11 @@ func requireNoSSChangelog(t *testing.T, evmDBDirectory string) { // The directory is one an earlier run with SS on could have left, and the WAL reaches block 1, so a // rollback that read it would come back with a rewind to run. func TestDiscardStateAboveLeavesANodeThatKeepsNoEVMStoreAlone(t *testing.T) { - const target = int64(7) + const target = uint64(7) dir := t.TempDir() - s := &StateDB{ - flatkvCfg: flatkvconfig.DefaultTestConfig(t), - ssCfg: config.StateStoreConfig{Enable: false, EVMDBDirectory: dir}, - } + ssCfg := config.StateStoreConfig{Enable: false, EVMDBDirectory: dir} - require.NoError(t, s.discardStateAbove(storedWALRange{first: 1, last: 9}, target)) + require.NoError(t, discardSSAbove(ssCfg, 1, target)) entries, err := os.ReadDir(dir) require.NoError(t, err) @@ -136,23 +104,20 @@ func TestCatchUpRefusesAWALMissingTheBlocksAStoreNeeds(t *testing.T) { t.Run("the state commit store", func(t *testing.T) { _, _, sc := newTestStateDB(t) - s := &StateDB{wal: &gapWAL{first: 3, last: 4}, sc: sc} - - require.ErrorContains(t, s.catchUpTo(t.Context(), 4), missingBlocks) + require.ErrorContains(t, catchUpTo(t.Context(), sc, nil, &gapWAL{first: 3, last: 4}, 4), missingBlocks) }) t.Run("the EVM state store", func(t *testing.T) { - _, _, sc := newTestStateDB(t) - s := &StateDB{wal: &gapWAL{first: 3, last: 4}, sc: sc, ss: &evm.EVMStateStore{}} + ss := &evm.EVMStateStore{} // The store holds nothing, so the gap is its whole history rather than a hole in it. Refusing // here would report data loss for a store that is merely new, and would do it on every node // past its first retention cut, so it is left out of the replay to fill forward from the target. - _, replays, err := s.ssReplayStart(4) + _, replays, err := ssReplayStart(ss, &gapWAL{first: 3, last: 4}, 4) require.NoError(t, err) require.False(t, replays) - require.Zero(t, s.ss.GetLatestVersion()) + require.Zero(t, ss.GetLatestVersion()) }) } @@ -163,9 +128,7 @@ func TestMatchHeightExcusesAStoreLeftToFillForward(t *testing.T) { for block := int64(1); block <= 4; block++ { require.NoError(t, sc.CommitStateChanges(block, changeset("k", "v"))) } - s := &StateDB{wal: &gapWAL{first: 3, last: 4}, sc: sc, ss: &evm.EVMStateStore{}} - - require.NoError(t, s.matchHeight(4)) + require.NoError(t, matchHeight(sc, &evm.EVMStateStore{}, &gapWAL{first: 3, last: 4}, 4)) } // An empty SS is only excused when the WAL cannot rebuild it. Excusing every version-0 store would @@ -175,9 +138,7 @@ func TestMatchHeightDoesNotExcuseAnEmptyStoreTheWALCanRebuild(t *testing.T) { for block := int64(1); block <= 4; block++ { require.NoError(t, sc.CommitStateChanges(block, changeset("k", "v"))) } - s := &StateDB{wal: &gapWAL{first: 1, last: 4}, sc: sc, ss: &evm.EVMStateStore{}} - - err := s.matchHeight(4) + err := matchHeight(sc, &evm.EVMStateStore{}, &gapWAL{first: 1, last: 4}, 4) require.ErrorContains(t, err, "EVM state store") // Both the open and a rollback converge here, so the height belongs to whichever asked. Naming a @@ -190,10 +151,7 @@ func TestMatchHeightDoesNotExcuseAnEmptyStoreTheWALCanRebuild(t *testing.T) { // reaches back far enough for comes out of recovery holding real history, which is strictly better, and // is how a store that lagged the WAL is populated on restart. func TestCatchUpRebuildsAnEmptyStoreTheWALStillCovers(t *testing.T) { - _, _, sc := newTestStateDB(t) - s := &StateDB{wal: &gapWAL{first: 1, last: 4}, sc: sc, ss: &evm.EVMStateStore{}} - - fillForward, err := s.ssFillsForward() + fillForward, err := ssFillsForward(&evm.EVMStateStore{}, &gapWAL{first: 1, last: 4}) require.NoError(t, err) require.False(t, fillForward, "a WAL starting at block 1 can rebuild an empty store") } diff --git a/sei-db/state_db/giga/state_db_test.go b/sei-db/state_db/giga/state_db_test.go index a0b03552c2..eb0669b295 100644 --- a/sei-db/state_db/giga/state_db_test.go +++ b/sei-db/state_db/giga/state_db_test.go @@ -54,7 +54,7 @@ func newTestStateDB(t *testing.T) (gigatypes.StateDB, *fakeStateWAL, *flatkv.Com require.NoError(t, liveStateDB.LoadLatest()) wal := &fakeStateWAL{} - return &StateDB{wal: wal, sc: liveStateDB, flatkvCfg: cfg}, wal, liveStateDB + return &StateDB{wal: wal, sc: liveStateDB}, wal, liveStateDB } // changeset builds a changeset setting key to value in the test module. diff --git a/sei-db/state_db/giga/types/state_db.go b/sei-db/state_db/giga/types/state_db.go index ef4aa80cfb..8b903a8f26 100644 --- a/sei-db/state_db/giga/types/state_db.go +++ b/sei-db/state_db/giga/types/state_db.go @@ -11,7 +11,23 @@ import ( ) // A callback function for getting the hash of each block. -type HashListener func(ctx context.Context, blockNum int64, hash *lthash.BlockHash) error +type HashListener func(ctx context.Context, blockNum uint64, hash *lthash.BlockHash) error + +// BlockHashStatus is the outcome of a block hash lookup. +type BlockHashStatus uint8 + +// BlockHashStatusError means the lookup failed, and the accompanying error says why. +const BlockHashStatusError BlockHashStatus = 0 + +// BlockHashStatusFound means the hash was found. +const BlockHashStatusFound BlockHashStatus = 1 + +// BlockHashStatusTooOld means the block is below the oldest block whose hash is still kept. +const BlockHashStatusTooOld BlockHashStatus = 2 + +// BlockHashStatusNotReady means the block's hash is not recorded yet: the block has not been hashed, or +// has not been committed. +const BlockHashStatusNotReady BlockHashStatus = 3 // StateDB is the top-level API used by the Giga EVM executor for // read and write. Writes commit into both SC and SS; reads can be served for @@ -40,6 +56,18 @@ type StateDB interface { // This may be useful at startup time to determine the initial hash of the database. RegisterHashListener(listener HashListener) (mostRecentHash lthash.BlockHash, err error) + // GetBlockHeight returns the number of the last block passed to CommitStateChanges, or the block the + // StateDB opened on when none has been passed since. + GetBlockHeight() uint64 + + // GetBlockHash returns the state hash of a recent block without blocking. The hash is valid only when + // the status is BlockHashStatusFound, and the error is non-nil exactly when the status is + // BlockHashStatusError. A returned hash is crash durable. + GetBlockHash(blockNumber uint64) (hash [32]byte, status BlockHashStatus, err error) + + // PruneBlockHashesBelow permits the hashes of blocks below blockNumber to be deleted. + PruneBlockHashesBelow(blockNumber uint64) error + // Close releases everything this StateDB was built over, reporting every failure rather than // stopping at the first. Close() error diff --git a/sei-db/state_db/sc/composite/hashlog.go b/sei-db/state_db/sc/composite/hashlog.go index 1543591941..03f8332365 100644 --- a/sei-db/state_db/sc/composite/hashlog.go +++ b/sei-db/state_db/sc/composite/hashlog.go @@ -34,8 +34,7 @@ func (cs *CompositeCommitStore) RecordHashes(hl hashlog.HashLogger, blockNumber // Keyed on the block cosmos committed rather than the hash's own height, which is what keeps // this row complete: a block whose writes never reached flatKV leaves its hash on the height // before, and the AppHash reports that same hash for this block. - //nolint:gosec // commit versions are non-negative - if err := hl.HashListener(cs.ctx, int64(blockNumber), cs.flatKVHash.Load()); err != nil { + if err := hl.HashListener(cs.ctx, blockNumber, cs.flatKVHash.Load()); err != nil { return fmt.Errorf("record flatkv hashes for block %d: %w", blockNumber, err) } } diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index ad1c42d306..39147a792f 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -228,7 +228,7 @@ func (cs *CompositeCommitStore) adoptFlatKV(store gigatypes.LiveStateStore) erro // recordFlatKVHash keeps flatKVHash current. It is the listener registered on every flatKV instance // this store adopts. -func (cs *CompositeCommitStore) recordFlatKVHash(_ context.Context, _ int64, hash *lthash.BlockHash) error { +func (cs *CompositeCommitStore) recordFlatKVHash(_ context.Context, _ uint64, hash *lthash.BlockHash) error { cs.flatKVHash.Store(hash) return nil } @@ -1210,7 +1210,7 @@ func (cs *CompositeCommitStore) latticeHash(version int64) ([]byte, error) { } hash := cs.flatKVHash.Load() - if hash.BlockNumber != version { + if hash.BlockNumber != uint64(version) { //nolint:gosec // a committed version is never negative // Block version+1 has not been handed to flatKV yet, so the hash just flushed is version's. // Asserted rather than assumed: this value reaches the AppHash, where a hash for the wrong // height is indistinguishable from the right one. diff --git a/sei-db/state_db/sc/flatkv/finalization_manager.go b/sei-db/state_db/sc/flatkv/finalization_manager.go index de8692312e..9080617d75 100644 --- a/sei-db/state_db/sc/flatkv/finalization_manager.go +++ b/sei-db/state_db/sc/flatkv/finalization_manager.go @@ -222,7 +222,7 @@ func (fm *FinalizationManager) finalize(pending *pendingFinalization) (stopped b if hash.Error != nil { return false, fmt.Errorf("hash block %d: %w", pending.blockNumber, hash.Error) } - if hash.BlockNumber != pending.blockNumber { + if hash.BlockNumber != uint64(pending.blockNumber) { //nolint:gosec // an offered block is never negative return false, fmt.Errorf("finalization is out of step: holding block %d, hashed block %d", pending.blockNumber, hash.BlockNumber) } diff --git a/sei-db/state_db/sc/flatkv/hash_listeners_test.go b/sei-db/state_db/sc/flatkv/hash_listeners_test.go index 900afa1dcf..6700ec7210 100644 --- a/sei-db/state_db/sc/flatkv/hash_listeners_test.go +++ b/sei-db/state_db/sc/flatkv/hash_listeners_test.go @@ -37,9 +37,9 @@ func commitBlocks(t *testing.T, s *CommitStore, count int) { // recordBlocks returns a listener that records the block number of every hash it is handed, and the // slice it records into. The slice is only safe to read once FlushHashes has returned. -func recordBlocks() (func(context.Context, int64, *lthash.BlockHash) error, *[]int64) { - blocks := &[]int64{} - return func(_ context.Context, blockNumber int64, _ *lthash.BlockHash) error { +func recordBlocks() (func(context.Context, uint64, *lthash.BlockHash) error, *[]uint64) { + blocks := &[]uint64{} + return func(_ context.Context, blockNumber uint64, _ *lthash.BlockHash) error { *blocks = append(*blocks, blockNumber) return nil }, blocks @@ -67,13 +67,13 @@ func TestAListenerSeesEveryBlockInOrder(t *testing.T) { listener, seen := recordBlocks() mostRecent, err := s.RegisterHashListener(listener) require.NoError(t, err) - require.Equal(t, int64(0), mostRecent.BlockNumber, "a fresh store has hashed nothing") + require.Equal(t, uint64(0), mostRecent.BlockNumber, "a fresh store has hashed nothing") const blocks = 8 commitBlocks(t, s, blocks) require.NoError(t, s.FlushHashes()) - require.Equal(t, []int64{1, 2, 3, 4, 5, 6, 7, 8}, *seen) + require.Equal(t, []uint64{1, 2, 3, 4, 5, 6, 7, 8}, *seen) } // FlushHashes is how a caller waits for hashing to catch up, and a hash that has been computed but @@ -107,13 +107,13 @@ func TestRegisterReportsTheBlockTheFirstDeliveryFollows(t *testing.T) { listener, seen := recordBlocks() mostRecent, err := s.RegisterHashListener(listener) require.NoError(t, err) - require.Equal(t, int64(3), mostRecent.BlockNumber) + require.Equal(t, uint64(3), mostRecent.BlockNumber) require.Equal(t, rootHash(s), checksumOf(mostRecent.Global)) commitBlocks(t, s, 2) require.NoError(t, s.FlushHashes()) - require.Equal(t, []int64{4, 5}, *seen, "a listener starts at the block after the one it was told") + require.Equal(t, []uint64{4, 5}, *seen, "a listener starts at the block after the one it was told") } // Listeners are independent: one of them consuming a hash must not take it away from another. @@ -131,8 +131,8 @@ func TestEveryListenerSeesEveryBlock(t *testing.T) { commitBlocks(t, s, 3) require.NoError(t, s.FlushHashes()) - require.Equal(t, []int64{1, 2, 3}, *seenByFirst) - require.Equal(t, []int64{1, 2, 3}, *seenBySecond) + require.Equal(t, []uint64{1, 2, 3}, *seenByFirst) + require.Equal(t, []uint64{1, 2, 3}, *seenBySecond) } // A listener that refuses a block is a caller that cannot keep up with the state it is deriving. The @@ -141,7 +141,7 @@ func TestAListenerThatFailsBricksTheStore(t *testing.T) { s := setupTestStoreWithConfig(t, tightHashPipelineConfig(t)) defer func() { _ = s.Close() }() - _, err := s.RegisterHashListener(func(context.Context, int64, *lthash.BlockHash) error { + _, err := s.RegisterHashListener(func(context.Context, uint64, *lthash.BlockHash) error { return fmt.Errorf("injected listener failure") }) require.NoError(t, err) @@ -171,7 +171,7 @@ func TestANilHashListenerRegistersNothing(t *testing.T) { mostRecent, err := s.RegisterHashListener(nil) require.NoError(t, err) - require.Equal(t, int64(2), mostRecent.BlockNumber, "a nil listener still reports the current hash") + require.Equal(t, uint64(2), mostRecent.BlockNumber, "a nil listener still reports the current hash") // Nothing was registered, so the block below has nobody to deliver to and must still commit. commitBlocks(t, s, 1) @@ -196,7 +196,7 @@ func TestRegistrationsSurviveARollback(t *testing.T) { commitBlocks(t, s, 5) require.NoError(t, s.FlushHashes()) - require.Equal(t, []int64{1, 2, 3, 4, 5}, *seen) + require.Equal(t, []uint64{1, 2, 3, 4, 5}, *seen) require.NoError(t, s.Rollback(3)) @@ -204,7 +204,7 @@ func TestRegistrationsSurviveARollback(t *testing.T) { require.NoError(t, s.FlushHashes()) // 0 is the height the rollback reopened at, then 1 to 3 are replayed, then 4 and 5 re-executed. - require.Equal(t, []int64{1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5}, *seen, + require.Equal(t, []uint64{1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5}, *seen, "the listener registered before the rollback must still be given the blocks after it") } @@ -216,7 +216,7 @@ func TestDispatchedPerDBHashesMatchWhatEachDatabaseRecorded(t *testing.T) { defer func() { require.NoError(t, s.Close()) }() var dispatched *lthash.BlockHash - _, err := s.RegisterHashListener(func(_ context.Context, _ int64, hash *lthash.BlockHash) error { + _, err := s.RegisterHashListener(func(_ context.Context, _ uint64, hash *lthash.BlockHash) error { dispatched = hash return nil }) diff --git a/sei-db/state_db/sc/flatkv/lthash/hash_engine.go b/sei-db/state_db/sc/flatkv/lthash/hash_engine.go index 2a3c5da127..917e13dad1 100644 --- a/sei-db/state_db/sc/flatkv/lthash/hash_engine.go +++ b/sei-db/state_db/sc/flatkv/lthash/hash_engine.go @@ -107,7 +107,7 @@ func (he *HashEngine) ScheduleHash( return fmt.Errorf("schedule hash: current and previous views are both required") } request := &hashRequest{ - blockNumber: current.BlockHeight(), + blockNumber: uint64(current.BlockHeight()), //nolint:gosec // a sealed view's height is never negative current: current, previous: previous, } diff --git a/sei-db/state_db/sc/flatkv/lthash/hash_engine_messages.go b/sei-db/state_db/sc/flatkv/lthash/hash_engine_messages.go index e708db644c..ce4b67c580 100644 --- a/sei-db/state_db/sc/flatkv/lthash/hash_engine_messages.go +++ b/sei-db/state_db/sc/flatkv/lthash/hash_engine_messages.go @@ -12,7 +12,7 @@ import ( // hashRequest is one sealed block for the engine to hash. type hashRequest struct { // blockNumber is the height being hashed. - blockNumber int64 + blockNumber uint64 // current is the block's own sealed view. The gatherer reads this block's diff from it. current *sview.StoreView @@ -54,7 +54,7 @@ func (r *hashRequest) release() error { // running hash. type gatheredBlock struct { // blockNumber is the height this job hashes. - blockNumber int64 + blockNumber uint64 // hashes is this block's leaf hashing in flight, which the combiner drains to completion. hashes leafHashes diff --git a/sei-db/state_db/sc/flatkv/lthash/hash_engine_test.go b/sei-db/state_db/sc/flatkv/lthash/hash_engine_test.go index 5971454b8c..933cd8a149 100644 --- a/sei-db/state_db/sc/flatkv/lthash/hash_engine_test.go +++ b/sei-db/state_db/sc/flatkv/lthash/hash_engine_test.go @@ -183,7 +183,7 @@ func TestHashEngineAgreesWithSynchronousCompute(t *testing.T) { require.Equal(t, want.Global.Checksum(), got.Global.Checksum(), "the pipeline must produce the hash a single-call fold produces") - require.Equal(t, int64(1), got.BlockNumber) + require.Equal(t, uint64(1), got.BlockNumber) } // mustViews is blockViews without the stub handles, for a caller that only wants the views. @@ -210,7 +210,7 @@ func TestHashEngineStreamsOneHashPerBlockInOrder(t *testing.T) { for height := int64(1); height <= blocks; height++ { got := <-engine.AwaitHash() require.NoError(t, got.Error) - require.Equal(t, height, got.BlockNumber, "hashes must arrive in block order with no gaps") + require.Equal(t, uint64(height), got.BlockNumber, "hashes must arrive in block order with no gaps") } require.NoError(t, engine.Close()) @@ -284,7 +284,7 @@ func TestHashEngineFlushWaitsForScheduledBlocks(t *testing.T) { for height := int64(1); height <= blocks; height++ { select { case got := <-engine.AwaitHash(): - require.Equal(t, height, got.BlockNumber) + require.Equal(t, uint64(height), got.BlockNumber) default: t.Fatalf("Flush returned before block %d was published", height) } @@ -401,7 +401,7 @@ func TestHashEngineDeliversFailureAndStops(t *testing.T) { got := <-engine.AwaitHash() require.Error(t, got.Error) require.ErrorContains(t, got.Error, "injected diff failure") - require.Equal(t, int64(1), got.BlockNumber) + require.Equal(t, uint64(1), got.BlockNumber) require.ErrorContains(t, engine.Close(), "injected diff failure") diff --git a/sei-db/state_db/sc/flatkv/lthash/hash_types.go b/sei-db/state_db/sc/flatkv/lthash/hash_types.go index 6ee983da5d..9b9f235bd0 100644 --- a/sei-db/state_db/sc/flatkv/lthash/hash_types.go +++ b/sei-db/state_db/sc/flatkv/lthash/hash_types.go @@ -41,7 +41,7 @@ type ModuleHashInfo struct { // and later blocks do not disturb it. type BlockHash struct { // BlockNumber is the height this state describes. - BlockNumber int64 + BlockNumber uint64 // PerDB is each data database's lattice hash root, with an entry for every database the engine was // configured with, so a caller can swap the map in wholesale. diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index 2c505a8038..4ce0615061 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -769,6 +769,18 @@ func repointAtSnapshot(dir string, version int64) error { return nil } +// SnapshotVersions returns the versions of the snapshots of the closed store under dir, lowest first. +func SnapshotVersions(dir string) ([]int64, error) { + var versions []int64 + if err := traverseSnapshots(dir, true, func(version int64) (bool, error) { + versions = append(versions, version) + return false, nil + }); err != nil { + return nil, fmt.Errorf("list snapshots under %q: %w", dir, err) + } + return versions, nil +} + // DiscardStateAbove puts the closed store under dir on its newest snapshot at or below target when it // holds any state above target, and reports the version its files hold once it returns. A store holding // nothing above target is left alone, reported at the version it opens on, for a replay to carry it diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index 146e2f43fc..7b68ea2755 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -1182,7 +1182,7 @@ func (s *CommitStore) deriveGlobalState() { } s.committedVersion = version - s.loadedHashes.BlockNumber = version + s.loadedHashes.BlockNumber = uint64(version) //nolint:gosec // a loaded version is never negative s.loadedHashes.Global = lthash.SumDBHashes(dataDBDirs, s.loadedHashes.PerDB) } diff --git a/sei-db/state_db/sc/flatkv/store_meta.go b/sei-db/state_db/sc/flatkv/store_meta.go index a9d79843c2..0545c97f76 100644 --- a/sei-db/state_db/sc/flatkv/store_meta.go +++ b/sei-db/state_db/sc/flatkv/store_meta.go @@ -406,7 +406,7 @@ func (s *CommitStore) SetInitialVersion(initialVersion int64) error { } s.committedVersion = seededVersion - s.loadedHashes.BlockNumber = seededVersion + s.loadedHashes.BlockNumber = uint64(seededVersion) //nolint:gosec // a seeded version is positive // The engine must carry back what this established, or the first real block would be measured // against different state than was persisted. diff --git a/sei-db/state_db/sc/flatkv/store_write.go b/sei-db/state_db/sc/flatkv/store_write.go index 0f8a68e890..449296c6a7 100644 --- a/sei-db/state_db/sc/flatkv/store_write.go +++ b/sei-db/state_db/sc/flatkv/store_write.go @@ -369,7 +369,7 @@ func (s *CommitStore) FinalizeImport(version int64) error { } s.loadedHashes.Global = lthash.SumDBHashes(dataDBDirs, s.loadedHashes.PerDB) - s.loadedHashes.BlockNumber = version + s.loadedHashes.BlockNumber = uint64(version) //nolint:gosec // an imported version is never negative s.committedVersion = version // The engine's accumulator described the databases this import has just replaced wholesale, so it is diff --git a/sei-db/state_db/sc/flatkv/store_write_test.go b/sei-db/state_db/sc/flatkv/store_write_test.go index e62d984bb5..5a3e9738ee 100644 --- a/sei-db/state_db/sc/flatkv/store_write_test.go +++ b/sei-db/state_db/sc/flatkv/store_write_test.go @@ -1927,8 +1927,8 @@ func TestHashFailureSurfacesToACallerAndStopsDispatch(t *testing.T) { defer func() { _ = s.Close() }() // Registered before the first block, since a listener only ever sees the blocks after it. - dispatched := make(chan int64, 8) - _, err := s.RegisterHashListener(func(_ context.Context, blockNumber int64, _ *lthash.BlockHash) error { + dispatched := make(chan uint64, 8) + _, err := s.RegisterHashListener(func(_ context.Context, blockNumber uint64, _ *lthash.BlockHash) error { dispatched <- blockNumber return nil }) @@ -1942,7 +1942,7 @@ func TestHashFailureSurfacesToACallerAndStopsDispatch(t *testing.T) { commitAndCheck(t, s) require.NoError(t, s.FlushHashes()) - require.Equal(t, int64(1), <-dispatched, "the good block hashes normally") + require.Equal(t, uint64(1), <-dispatched, "the good block hashes normally") s.moduleOf = func([]byte) (string, error) { return "", fmt.Errorf("injected moduleOf failure") @@ -1985,14 +1985,14 @@ func TestAReadOnlyStoreReportsItsHeight(t *testing.T) { require.NoError(t, err) defer func() { _ = ro.Close() }() - delivered := make(chan int64, 4) + delivered := make(chan uint64, 4) mostRecent, err := ro.RegisterHashListener( - func(_ context.Context, blockNumber int64, _ *lthash.BlockHash) error { + func(_ context.Context, blockNumber uint64, _ *lthash.BlockHash) error { delivered <- blockNumber return nil }) require.NoError(t, err) - require.Equal(t, ro.Version(), mostRecent.BlockNumber, + require.Equal(t, uint64(ro.Version()), mostRecent.BlockNumber, "registration must report the height the read-only store was opened at") require.NoError(t, ro.FlushHashes()) diff --git a/sei-db/state_db/sc/hashlog/flatkv_listener_test.go b/sei-db/state_db/sc/hashlog/flatkv_listener_test.go index a938d3ee5e..df793dd883 100644 --- a/sei-db/state_db/sc/hashlog/flatkv_listener_test.go +++ b/sei-db/state_db/sc/hashlog/flatkv_listener_test.go @@ -39,7 +39,7 @@ func checksumOf(hash *lthash.LtHash) []byte { // flatKVBlockHash returns a block hash with a distinct root and a distinct hash for each of flatKV's // data databases. -func flatKVBlockHash(t *testing.T, blockNumber int64) *lthash.BlockHash { +func flatKVBlockHash(t *testing.T, blockNumber uint64) *lthash.BlockHash { t.Helper() return <hash.BlockHash{ BlockNumber: blockNumber, diff --git a/sei-db/state_db/sc/hashlog/hash_logger.go b/sei-db/state_db/sc/hashlog/hash_logger.go index 9f76f7a1a4..e347c34a70 100644 --- a/sei-db/state_db/sc/hashlog/hash_logger.go +++ b/sei-db/state_db/sc/hashlog/hash_logger.go @@ -75,7 +75,7 @@ type HashLogger interface { // // The columns reported here are fixed, and a node declares them when it constructs the logger // (see HashLoggerConfig.HashTypes). Nothing registers a column per block. - HashListener(ctx context.Context, blockNumber int64, hash *lthash.BlockHash) error + HashListener(ctx context.Context, blockNumber uint64, hash *lthash.BlockHash) error // Shut down the HashLogger and release any resources. Flushes pending writes before returning. Only blocks // that are complete (a hash has been reported for every configured type) are written; a block still missing a diff --git a/sei-db/state_db/sc/hashlog/hash_logger_impl.go b/sei-db/state_db/sc/hashlog/hash_logger_impl.go index d892af8f4f..aa4d74fefe 100644 --- a/sei-db/state_db/sc/hashlog/hash_logger_impl.go +++ b/sei-db/state_db/sc/hashlog/hash_logger_impl.go @@ -448,9 +448,7 @@ func (h *hashLoggerImpl) ReportHash(blockNumber uint64, hashType string, hash [] // HashListener records one block's flatKV hashes: the store-wide root and each data database's root. // Its signature is gigatypes.HashListener, so it registers as one directly: // stateDB.RegisterHashListener(hashLogger.HashListener). -func (h *hashLoggerImpl) HashListener(_ context.Context, blockNumber int64, hash *lthash.BlockHash) error { - block := uint64(blockNumber) //nolint:gosec // commit versions are non-negative - +func (h *hashLoggerImpl) HashListener(_ context.Context, block uint64, hash *lthash.BlockHash) error { root := hash.Global.Checksum() if err := h.ReportHash(block, FlatKVRootHashType, root[:]); err != nil { return fmt.Errorf("record the flatkv root hash of block %d: %w", block, err) diff --git a/sei-db/state_db/sc/hashlog/noop_hash_logger.go b/sei-db/state_db/sc/hashlog/noop_hash_logger.go index 8b097340ec..a2aa7af543 100644 --- a/sei-db/state_db/sc/hashlog/noop_hash_logger.go +++ b/sei-db/state_db/sc/hashlog/noop_hash_logger.go @@ -37,7 +37,7 @@ func (n *noOpHashLogger) ReportHash(uint64, string, []byte) error { return nil } -func (n *noOpHashLogger) HashListener(context.Context, int64, *lthash.BlockHash) error { +func (n *noOpHashLogger) HashListener(context.Context, uint64, *lthash.BlockHash) error { // intentional no-op return nil } diff --git a/sei-db/state_db/sc/hashvault/hashvault.go b/sei-db/state_db/sc/hashvault/hashvault.go index 3a40600039..5e3a807060 100644 --- a/sei-db/state_db/sc/hashvault/hashvault.go +++ b/sei-db/state_db/sc/hashvault/hashvault.go @@ -1,58 +1,366 @@ +// Package hashvault records the live state DB's block hashes and refuses to let a recorded hash change, +// so that a node cannot commit to two different states for the same block without human intervention. package hashvault import ( - "context" - "errors" + "encoding/hex" + "fmt" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/sei-protocol/seilog" + + "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable/keymap" + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/littbuilder" + gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" ) -// HashVault is a safety mechanism to prevent a validator from "changing its mind" about the hash of a block -// without human intervention. -type HashVault interface { - - // CommitToHash takes a provided hash for a block and writes it to disk. This method blocks until the hash is - // crash durable. - // - // This utility may be passed the hash for a block multiple times, but it will refuse to allow the hash to change - // for a particular block height. If this method returns nil, then it means that the hash is either the first - // one observed by the HashVault, or that the hash is the same as one for this block that was previously reported. - // - // If this method returns an error, DO NOT ATTEMPT TO RECOVER WITHOUT HUMAN INTERVENTION! - CommitToHash(ctx context.Context, blockHeight uint64, hash []byte) error - - // Prune deletes all data for blocks below the specified height. Keeps data for the specified block height. - // Note that reporting the hash for a block below the pruning boundary will result in an error - // (as it is impossible to validate the correctness of the hash for a block below the pruning boundary). - Prune(ctx context.Context, blockHeight uint64) error - - // Close shuts the HashVault down and frees all resources (but does not delete the data from disk). - Close(ctx context.Context) error -} - -// BlockHashSize is the required byte length for hashes passed to CommitToHash (CometBFT block ID / header hash). -const BlockHashSize = 32 - -// ErrInvalidHashLength is returned when CommitToHash is called with a hash whose length is not BlockHashSize. -var ErrInvalidHashLength = errors.New("block hash must be 32 bytes") - -// ErrHashMismatch is returned by CommitToHash when the caller provides a hash that differs from the -// hash previously committed for the same block height. This is the primary "node changed its mind" -// signal and MUST cause the calling node to halt. -var ErrHashMismatch = errors.New("block hash mismatch") - -// ErrBelowPruneBoundary is returned when an operation targets a block height that has already been -// pruned. Reporting or rolling back through pruned heights is impossible to validate and so is rejected. -var ErrBelowPruneBoundary = errors.New("block height below prune boundary") - -// ErrClosed is returned when a method is called after Close. -var ErrClosed = errors.New("hashvault is closed") - -// ErrCorruption is returned when the on-disk integrity check (SHA-256 trailer bound to (height, hash)) -// fails. This indicates either disk corruption or a bug in the encoding layer. Callers MUST treat -// this as fatal and require human intervention. -var ErrCorruption = errors.New("hashvault on-disk integrity check failed") - -// ErrRollbackHeightOverflow is returned by HardRollbackPebbleHashVault when blockHeight is -// math.MaxUint64. The partial-rollback path deletes hashes strictly above blockHeight via -// DeleteRange(hashKey(blockHeight+1), ...); at MaxUint64 that addition wraps to zero and would -// delete the entire vault instead of none. -var ErrRollbackHeightOverflow = errors.New("rollback block height overflows uint64") +var logger = seilog.NewLogger("db", "state-db", "sc", "hashvault") + +// tableName is the LittDB table the hashes are recorded in. +const tableName = "hashes" + +// HashVault records one hash per block for a contiguous range of blocks, and holds each hash fixed once +// it is recorded. +// +// Every method is safe to call from any goroutine. +type HashVault struct { + // The config the vault was opened with. + config config.HashVaultConfig + + // Guards db, table, empty, head and closed, and so the table against being replaced while it is read. + // Commit, Reset and Close take it exclusively; lookups share it. + mu sync.RWMutex + + // The database the table lives in. Replaced when a mismatch or Reset rewrites the files. + db litt.DB + + // The table the hashes are recorded in. + table litt.Table + + // True when the vault holds no hashes. + empty bool + + // The newest recorded block. Meaningless when empty is true. + head uint64 + + // True once Close has run. + closed bool + + // The floor PruneBlockHashesBelow() has raised. Only ever rises. + outerFloor atomic.Uint64 + + // The floor the storage garbage collector has raised through PruneHistory(). Only ever rises. + gcFloor atomic.Uint64 +} + +// Open opens the vault under cfg.DataDir, creating it if it does not exist, and deletes +// cfg.LegacyPebbleDir if it is present. +func Open(cfg config.HashVaultConfig) (*HashVault, error) { + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("invalid hash vault config: %w", err) + } + if err := deleteLegacyPebbleVault(cfg.LegacyPebbleDir); err != nil { + return nil, fmt.Errorf("open the hash vault: %w", err) + } + v := &HashVault{config: cfg} + if err := v.openTable(); err != nil { + return nil, fmt.Errorf("open the hash vault: %w", err) + } + return v, nil +} + +// deleteLegacyPebbleVault deletes the Pebble-backed vault this one replaced, if it is present. Its hashes +// are app hashes rather than state hashes, so none of them can be carried over. +// +// This can be deleted once every node that ran the Pebble-backed vault has started on this one. +func deleteLegacyPebbleVault(dir string) error { + if dir == "" { + return nil + } + if _, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("stat legacy hash vault dir %q: %w", dir, err) + } + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("delete legacy hash vault dir %q: %w", dir, err) + } + logger.Info("Deleted the legacy Pebble hash vault; its app hashes cannot be compared with state hashes", + "dir", dir) + return nil +} + +// littConfig returns the config a vault's LittDB is opened, surveyed and pruned with. +func littConfig(vaultCfg config.HashVaultConfig) (*litt.Config, error) { + cfg, err := litt.DefaultConfig(vaultCfg.DataDir) + if err != nil { + return nil, fmt.Errorf("build hash vault littdb config: %w", err) + } + cfg.Fsync = vaultCfg.Fsync + // The table holds a few thousand small keys, so an in-memory keymap rebuilt at open is cheap, and it + // spares every flush a second database to sync. + cfg.KeymapType = keymap.MemKeymapType + cfg.DoubleWriteProtection = true + return cfg, nil +} + +// openTable opens the database and its table, and loads the newest recorded block. +func (v *HashVault) openTable() error { + cfg, err := littConfig(v.config) + if err != nil { + return fmt.Errorf("open the hash vault table: %w", err) + } + db, err := littbuilder.NewDB(cfg) + if err != nil { + return fmt.Errorf("open hash vault littdb at %q: %w", v.config.DataDir, err) + } + tableConfig := litt.DefaultTableConfig(tableName) + // A single write shard is what makes the writes that survive a crash a prefix of the ones issued, so + // a crash can shorten the recorded range but never leave a gap in it. + tableConfig.ShardingFactor = 1 + // A TTL is required for LittDB to collect at all. This one is shorter than any block, so the GC filter + // alone decides what is deleted. + tableConfig.TTL = time.Nanosecond + tableConfig.GCFilter = v.gcFilter + table, err := db.BuildTable(tableConfig) + if err != nil { + _ = db.Close() + return fmt.Errorf("open hash vault table: %w", err) + } + v.db = db + v.table = table + if err := v.loadRange(); err != nil { + _ = db.Close() + return fmt.Errorf("load the hash vault's range: %w", err) + } + return nil +} + +// loadRange reads the newest recorded block and checks that the recorded blocks are contiguous. +func (v *HashVault) loadRange() error { + newestKey, found, err := v.table.GetNewestKey() + if err != nil { + return fmt.Errorf("read the newest hash vault key: %w", err) + } + if !found { + v.empty = true + v.head = 0 + return nil + } + newest, err := decodeKey(newestKey) + if err != nil { + return fmt.Errorf("decode the newest hash vault key: %w", err) + } + oldestKey, found, err := v.table.GetOldestKey() + if err != nil { + return fmt.Errorf("read the oldest hash vault key: %w", err) + } + if !found { + return fmt.Errorf("hash vault has a newest key but no oldest key") + } + oldest, err := decodeKey(oldestKey) + if err != nil { + return fmt.Errorf("decode the oldest hash vault key: %w", err) + } + if count := v.table.KeyCount(); oldest > newest || newest-oldest+1 != count { + return fmt.Errorf("hash vault at %q is corrupt: it holds %d hashes for blocks %d to %d, which is not "+ + "one per block", v.config.DataDir, count, oldest, newest) + } + v.empty = false + v.head = newest + return nil +} + +// Head returns the newest recorded block, and false when the vault holds no hashes. +func (v *HashVault) Head() (uint64, bool) { + v.mu.RLock() + defer v.mu.RUnlock() + return v.head, !v.empty +} + +// Commit records hash as blockNumber's hash, or checks it against the hash already recorded for that +// block. It returns once the hash is recorded, and the recording is crash durable when cfg.Fsync is set. +// +// An empty vault takes any block, and a vault that is not empty takes a block at or below its newest +// recorded block, or the one after it. A block further ahead is an error. A hash that differs from the +// recorded one, or a block below the oldest recorded one, is an error when cfg.HaltOnMismatch is set; +// otherwise the vault discards its hashes from that block up and records this one in their place. +func (v *HashVault) Commit(blockNumber uint64, hash [32]byte) error { + v.mu.Lock() + defer v.mu.Unlock() + if v.closed { + return fmt.Errorf("commit the hash of block %d: the hash vault is closed", blockNumber) + } + + if v.empty || blockNumber == v.head+1 { + if err := v.append(blockNumber, hash); err != nil { + return fmt.Errorf("commit the hash of block %d: %w", blockNumber, err) + } + return nil + } + if blockNumber > v.head { + return fmt.Errorf("commit the hash of block %d: the hash vault's newest block is %d, so block %d "+ + "would leave a gap", blockNumber, v.head, blockNumber) + } + + recorded, found, err := v.read(blockNumber) + if err != nil { + return fmt.Errorf("commit the hash of block %d: %w", blockNumber, err) + } + if found && recorded == hash { + return nil + } + if err := v.resolveMismatch(blockNumber, hash, recorded, found); err != nil { + return fmt.Errorf("commit the hash of block %d: %w", blockNumber, err) + } + return nil +} + +// resolveMismatch handles a hash for blockNumber that differs from the recorded one, or a block below the +// oldest recorded one, which found reports. It halts or replaces the recorded hashes, as +// cfg.HaltOnMismatch selects. +func (v *HashVault) resolveMismatch(blockNumber uint64, hash [32]byte, recorded [32]byte, found bool) error { + recordedHex := "" + if found { + recordedHex = hex.EncodeToString(recorded[:]) + } + fields := []any{ + "blockNumber", blockNumber, + "recordedHex", recordedHex, + "incomingHex", hex.EncodeToString(hash[:]), + "newestRecordedBlock", v.head, + "hashVaultDir", v.config.DataDir, + } + + if v.config.HaltOnMismatch { + logger.Error("HASH VAULT MISMATCH: the node computed a different state hash for a block it already "+ + "recorded, or a block older than any it keeps. Halting. DO NOT RESTART WITHOUT HUMAN "+ + "INVESTIGATION. To continue past this instead, set hash-vault-halt-on-mismatch = false.", + fields...) + return fmt.Errorf("hash vault mismatch at block %d: recorded %s, computed %x", + blockNumber, recordedHex, hash) + } + + logger.Error("HASH VAULT MISMATCH: the node computed a different state hash for a block it already "+ + "recorded, or a block older than any it keeps. hash-vault-halt-on-mismatch is false, so the "+ + "recorded hashes from this block up are discarded and the new hash replaces them.", fields...) + if err := v.discardFrom(blockNumber); err != nil { + return fmt.Errorf("discard the hash vault from block %d after a mismatch: %w", blockNumber, err) + } + if err := v.append(blockNumber, hash); err != nil { + return fmt.Errorf("record the replacing hash of block %d: %w", blockNumber, err) + } + return nil +} + +// discardFrom closes the database, deletes every hash from blockNumber up, and reopens it. When no hash +// below blockNumber is recorded, the vault is left empty. +func (v *HashVault) discardFrom(blockNumber uint64) error { + if err := v.db.Close(); err != nil { + return fmt.Errorf("close the hash vault before pruning it: %w", err) + } + if err := pruneFrom(v.config, blockNumber); err != nil { + return fmt.Errorf("prune the hash vault from block %d: %w", blockNumber, err) + } + if err := v.openTable(); err != nil { + return fmt.Errorf("reopen the hash vault after pruning it: %w", err) + } + return nil +} + +// Reset deletes every recorded hash and records hash as blockNumber's, leaving it the only one. +func (v *HashVault) Reset(blockNumber uint64, hash [32]byte) error { + v.mu.Lock() + defer v.mu.Unlock() + if v.closed { + return fmt.Errorf("reset the hash vault to block %d: the hash vault is closed", blockNumber) + } + if err := v.db.Close(); err != nil { + return fmt.Errorf("close the hash vault before resetting it: %w", err) + } + if err := os.RemoveAll(v.config.DataDir); err != nil { + return fmt.Errorf("delete the hash vault at %q: %w", v.config.DataDir, err) + } + if err := v.openTable(); err != nil { + return fmt.Errorf("reopen the hash vault after deleting it: %w", err) + } + logger.Info("Reset the hash vault", "blockNumber", blockNumber, "hashVaultDir", v.config.DataDir) + if err := v.append(blockNumber, hash); err != nil { + return fmt.Errorf("record the hash of block %d after a reset: %w", blockNumber, err) + } + return nil +} + +// append records hash as blockNumber's hash and flushes it. blockNumber must be the block after the +// newest recorded one, or any block when the vault is empty. +func (v *HashVault) append(blockNumber uint64, hash [32]byte) error { + if err := v.table.Put(encodeKey(blockNumber), encodeValue(hash)); err != nil { + return fmt.Errorf("record the hash of block %d: %w", blockNumber, err) + } + if err := v.table.Flush(); err != nil { + return fmt.Errorf("flush the hash of block %d: %w", blockNumber, err) + } + v.empty = false + v.head = blockNumber + return nil +} + +// read returns the hash recorded for blockNumber, and false when none is. +func (v *HashVault) read(blockNumber uint64) ([32]byte, bool, error) { + value, found, err := v.table.Get(encodeKey(blockNumber)) + if err != nil { + return [32]byte{}, false, fmt.Errorf("read the hash of block %d: %w", blockNumber, err) + } + if !found { + return [32]byte{}, false, nil + } + hash, err := decodeValue(value) + if err != nil { + return [32]byte{}, false, fmt.Errorf("decode the hash of block %d: %w", blockNumber, err) + } + return hash, true, nil +} + +// Get returns the hash recorded for blockNumber, without blocking. +func (v *HashVault) Get(blockNumber uint64) ([32]byte, gigatypes.BlockHashStatus, error) { + v.mu.RLock() + defer v.mu.RUnlock() + if v.closed { + return [32]byte{}, gigatypes.BlockHashStatusError, + fmt.Errorf("get the hash of block %d: the hash vault is closed", blockNumber) + } + if v.empty || blockNumber > v.head { + return [32]byte{}, gigatypes.BlockHashStatusNotReady, nil + } + hash, found, err := v.read(blockNumber) + if err != nil { + return [32]byte{}, gigatypes.BlockHashStatusError, + fmt.Errorf("get the hash of block %d: %w", blockNumber, err) + } + if !found { + // The recorded blocks are contiguous up to head, so a block at or below it that is missing has + // been pruned. + return [32]byte{}, gigatypes.BlockHashStatusTooOld, nil + } + return hash, gigatypes.BlockHashStatusFound, nil +} + +// Close closes the vault. Every later call fails. +func (v *HashVault) Close() error { + v.mu.Lock() + defer v.mu.Unlock() + if v.closed { + return nil + } + v.closed = true + if err := v.db.Close(); err != nil { + return fmt.Errorf("close the hash vault: %w", err) + } + return nil +} diff --git a/sei-db/state_db/sc/hashvault/hashvault_config.go b/sei-db/state_db/sc/hashvault/hashvault_config.go deleted file mode 100644 index 39394afda7..0000000000 --- a/sei-db/state_db/sc/hashvault/hashvault_config.go +++ /dev/null @@ -1,41 +0,0 @@ -package hashvault - -import ( - "fmt" -) - -// HashVaultConfig is the configuration for a HashVault. -type HashVaultConfig struct { - // DataDir is the directory in which the PebbleDB-backed HashVault stores its data. - DataDir string - - // Fsync controls whether the underlying Pebble writes are fsynced. - // - // This field is test-only. Production callers should construct via NewPebbleHashVault, which forces - // fsync on regardless of this value. NewUnsafePebbleHashVault honors this flag and is intended for - // tests that exercise enough writes that fsync would dominate runtime. - Fsync bool - - // CacheSize is the number of recent (height -> verified hash) entries in the in-process LRU cache. - CacheSize int -} - -// DefaultHashVaultConfig returns a HashVaultConfig with production defaults. -func DefaultHashVaultConfig() HashVaultConfig { - return HashVaultConfig{ - Fsync: false, - CacheSize: 1024, - } -} - -// Validate returns a non-nil error if the configuration is missing required fields or has values -// that the HashVault cannot accept. -func (c *HashVaultConfig) Validate() error { - if c.DataDir == "" { - return fmt.Errorf("data directory is required") - } - if c.CacheSize <= 0 { - return fmt.Errorf("cache size must be greater than zero") - } - return nil -} diff --git a/sei-db/state_db/sc/hashvault/hashvault_test.go b/sei-db/state_db/sc/hashvault/hashvault_test.go index 3eff0ee259..2462864271 100644 --- a/sei-db/state_db/sc/hashvault/hashvault_test.go +++ b/sei-db/state_db/sc/hashvault/hashvault_test.go @@ -1,229 +1,274 @@ package hashvault import ( - "context" - "errors" - "fmt" - "sync" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/config" + gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" ) -// Contract-level tests for HashVault. These exercise the externally-visible behavior promised by -// the HashVault interface against the PebbleHashVault implementation. Pebble-specific surface -// (encoding, restart recovery, on-disk inspection, the static rollback function, etc.) is tested -// per-implementation in pebble_hashvault_test.go and pebble_hashvault_rollback_test.go. +// testConfig returns a config for a vault in a fresh directory. Fsync is off, since the tests flush after +// every hash and the durability is LittDB's to prove, not this package's. +func testConfig(t *testing.T, haltOnMismatch bool) config.HashVaultConfig { + t.Helper() + cfg := config.DefaultHashVaultConfig() + cfg.DataDir = filepath.Join(t.TempDir(), "hashvault") + cfg.HaltOnMismatch = haltOnMismatch + cfg.Fsync = false + return cfg +} + +// openVault opens a vault from cfg, closed when the test ends. +func openVault(t *testing.T, cfg config.HashVaultConfig) *HashVault { + t.Helper() + v, err := Open(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, v.Close()) }) + return v +} -func bytesOfLen(b byte, n int) []byte { - out := make([]byte, n) - for i := range out { - out[i] = b +// hashOf returns a hash that differs for every distinct seed. +func hashOf(seed byte) [32]byte { + var hash [32]byte + for i := range hash { + hash[i] = seed } - return out + return hash } -func TestCommitRejectsInvalidHashLength(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) +// commitRange commits the hashes of blocks first to last, each seeded with its own block number. +func commitRange(t *testing.T, v *HashVault, first uint64, last uint64) { + t.Helper() + for block := first; block <= last; block++ { + require.NoError(t, v.Commit(block, hashOf(byte(block)))) + } +} - require.ErrorIs(t, v.CommitToHash(ctx, 1, nil), ErrInvalidHashLength) - require.ErrorIs(t, v.CommitToHash(ctx, 1, []byte{}), ErrInvalidHashLength) - require.ErrorIs(t, v.CommitToHash(ctx, 1, bytesOfLen(0xAA, 31)), ErrInvalidHashLength) - require.ErrorIs(t, v.CommitToHash(ctx, 1, bytesOfLen(0xAA, 33)), ErrInvalidHashLength) +// requireHash asserts the vault holds want for blockNumber. +func requireHash(t *testing.T, v *HashVault, blockNumber uint64, want [32]byte) { + t.Helper() + got, status, err := v.Get(blockNumber) + require.NoError(t, err) + require.Equal(t, gigatypes.BlockHashStatusFound, status, "block %d", blockNumber) + require.Equal(t, want, got, "block %d", blockNumber) } -func TestCommitFirstTime(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - hash := bytesOfLen(0xAA, 32) - require.NoError(t, v.CommitToHash(ctx, 7, hash)) +// An empty vault has no range to hold a block to, so the first block may be any height, and nothing is +// ready to be read until it is recorded. +func TestAnEmptyVaultTakesAnyFirstBlock(t *testing.T) { + v := openVault(t, testConfig(t, true)) + + _, recorded := v.Head() + require.False(t, recorded) + _, status, err := v.Get(5) + require.NoError(t, err) + require.Equal(t, gigatypes.BlockHashStatusNotReady, status) + + require.NoError(t, v.Commit(100, hashOf(1))) + head, recorded := v.Head() + require.True(t, recorded) + require.Equal(t, uint64(100), head) + requireHash(t, v, 100, hashOf(1)) } -func TestCommitIdempotent(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - hash := bytesOfLen(0xAB, 32) - require.NoError(t, v.CommitToHash(ctx, 7, hash)) - require.NoError(t, v.CommitToHash(ctx, 7, hash)) - require.NoError(t, v.CommitToHash(ctx, 7, hash)) +// Blocks above the newest recorded one are not ready, whether or not they have been committed yet. +func TestABlockAboveTheHeadIsNotReady(t *testing.T) { + v := openVault(t, testConfig(t, true)) + commitRange(t, v, 1, 3) + + _, status, err := v.Get(4) + require.NoError(t, err) + require.Equal(t, gigatypes.BlockHashStatusNotReady, status) } -func TestCommitMismatch(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - a := bytesOfLen(0x01, 32) - b := bytesOfLen(0x02, 32) - require.NoError(t, v.CommitToHash(ctx, 42, a)) +// The recorded range is contiguous, so a block that would leave a gap is refused whatever the mismatch +// policy is. +func TestAGapIsRefused(t *testing.T) { + for _, halt := range []bool{true, false} { + v := openVault(t, testConfig(t, halt)) + commitRange(t, v, 1, 3) - err := v.CommitToHash(ctx, 42, b) - require.Error(t, err) - require.ErrorIs(t, err, ErrHashMismatch) -} - -func TestCommitMismatchAfterRepeatedCommitIsSticky(t *testing.T) { - // Even after re-committing the same hash many times, a single mismatch still surfaces. This - // is essentially a regression check that the cache fast path also enforces the mismatch. - ctx := context.Background() - v := newTestPebbleVault(t) - a := bytesOfLen(0x55, 32) - b := bytesOfLen(0x66, 32) - for i := 0; i < 10; i++ { - require.NoError(t, v.CommitToHash(ctx, 5, a)) + require.ErrorContains(t, v.Commit(5, hashOf(5)), "gap") + head, _ := v.Head() + require.Equal(t, uint64(3), head, "a refused block must not be recorded") } - err := v.CommitToHash(ctx, 5, b) - require.ErrorIs(t, err, ErrHashMismatch) } -func TestPruneRemovesData(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) +// Re-execution reproduces the hashes it recorded before, so committing the same hash again is a check +// that passes, not a write. +func TestRecommittingTheSameHashPasses(t *testing.T) { + v := openVault(t, testConfig(t, true)) + commitRange(t, v, 1, 5) - // Commit a handful of heights, prune below 5, then probe around the boundary. - for h := uint64(1); h <= 10; h++ { - require.NoError(t, v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32))) - } - require.NoError(t, v.Prune(ctx, 5)) - - // Below the boundary is rejected. - require.ErrorIs(t, - v.CommitToHash(ctx, 3, bytesOfLen(0x03, 32)), - ErrBelowPruneBoundary, - ) - // At the boundary is allowed (and the previously-committed hash is still locked in). - require.NoError(t, v.CommitToHash(ctx, 5, bytesOfLen(0x05, 32))) - require.ErrorIs(t, - v.CommitToHash(ctx, 5, bytesOfLen(0x55, 32)), - ErrHashMismatch, - ) - // Above the boundary is allowed and still locked. - require.NoError(t, v.CommitToHash(ctx, 7, bytesOfLen(0x07, 32))) - require.ErrorIs(t, - v.CommitToHash(ctx, 7, bytesOfLen(0x77, 32)), - ErrHashMismatch, - ) -} - -func TestCommitBelowPruneBoundary(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - - require.NoError(t, v.Prune(ctx, 100)) - // Strictly below the boundary is rejected. - require.ErrorIs(t, - v.CommitToHash(ctx, 99, bytesOfLen(0xAA, 32)), - ErrBelowPruneBoundary, - ) - require.ErrorIs(t, - v.CommitToHash(ctx, 50, bytesOfLen(0xAA, 32)), - ErrBelowPruneBoundary, - ) - // At the boundary is allowed: Prune keeps the boundary block per the godoc. - require.NoError(t, v.CommitToHash(ctx, 100, bytesOfLen(0xAA, 32))) - // Above is also obviously fine. - require.NoError(t, v.CommitToHash(ctx, 101, bytesOfLen(0xAA, 32))) -} - -func TestPruneMonotonic(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - - require.NoError(t, v.Prune(ctx, 50)) - require.NoError(t, v.Prune(ctx, 25)) // no-op - // Committing at 30 still errors: the effective boundary is still 50. - require.ErrorIs(t, - v.CommitToHash(ctx, 30, bytesOfLen(0xAA, 32)), - ErrBelowPruneBoundary, - ) - // Just below the boundary still errors. - require.ErrorIs(t, - v.CommitToHash(ctx, 49, bytesOfLen(0xAA, 32)), - ErrBelowPruneBoundary, - ) - // At and above the boundary succeed. - require.NoError(t, v.CommitToHash(ctx, 50, bytesOfLen(0x50, 32))) - require.NoError(t, v.CommitToHash(ctx, 51, bytesOfLen(0xAA, 32))) -} - -func TestCloseIsIdempotent(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - require.NoError(t, v.Close(ctx)) - require.NoError(t, v.Close(ctx)) - require.NoError(t, v.Close(ctx)) -} - -func TestCallsAfterCloseError(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - require.NoError(t, v.Close(ctx)) - require.ErrorIs(t, v.CommitToHash(ctx, 1, bytesOfLen(0xAA, 32)), ErrClosed) - require.ErrorIs(t, v.Prune(ctx, 1), ErrClosed) -} - -func TestConcurrentCommits(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - - // 100 goroutines, each commits a distinct height. All should succeed. - var wg sync.WaitGroup - const N = 100 - errs := make(chan error, N) - for i := 0; i < N; i++ { - wg.Add(1) - go func(h uint64) { - defer wg.Done() - errs <- v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32)) - }(uint64(i + 1)) - } - wg.Wait() - close(errs) - for err := range errs { - require.NoError(t, err) - } + commitRange(t, v, 2, 5) + head, _ := v.Head() + require.Equal(t, uint64(5), head) + requireHash(t, v, 3, hashOf(3)) +} - // Re-committing the same (height, hash) from many goroutines should also all succeed. - errs2 := make(chan error, N) - for i := 0; i < N; i++ { - wg.Add(1) - go func(h uint64) { - defer wg.Done() - errs2 <- v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32)) - }(uint64(i + 1)) - } - wg.Wait() - close(errs2) - for err := range errs2 { +// With halting selected, a different hash for a recorded block fails and leaves the record as it was. +func TestAMismatchHaltsWhenHaltingIsSelected(t *testing.T) { + v := openVault(t, testConfig(t, true)) + commitRange(t, v, 1, 5) + + require.ErrorContains(t, v.Commit(3, hashOf(0xEE)), "mismatch") + requireHash(t, v, 3, hashOf(3)) + head, _ := v.Head() + require.Equal(t, uint64(5), head) +} + +// With halting off, a different hash replaces the recorded one, and the hashes above it go with it: they +// were derived from the state the new hash disowns. +func TestAMismatchReplacesTheRecordWhenHaltingIsOff(t *testing.T) { + v := openVault(t, testConfig(t, false)) + commitRange(t, v, 1, 5) + + require.NoError(t, v.Commit(3, hashOf(0xEE))) + requireHash(t, v, 2, hashOf(2)) + requireHash(t, v, 3, hashOf(0xEE)) + head, _ := v.Head() + require.Equal(t, uint64(3), head) + _, status, err := v.Get(4) + require.NoError(t, err) + require.Equal(t, gigatypes.BlockHashStatusNotReady, status) + + require.NoError(t, v.Commit(4, hashOf(0xEF)), "commits carry on from the replaced block") + requireHash(t, v, 4, hashOf(0xEF)) +} + +// A mismatch at the oldest recorded block discards every hash, which the vault survives as an empty one. +func TestAMismatchAtTheOldestBlockLeavesOnlyTheNewHash(t *testing.T) { + v := openVault(t, testConfig(t, false)) + commitRange(t, v, 10, 12) + + require.NoError(t, v.Commit(10, hashOf(0xEE))) + requireHash(t, v, 10, hashOf(0xEE)) + head, _ := v.Head() + require.Equal(t, uint64(10), head) +} + +// What the vault records survives a restart, including a record a mismatch rewrote. +func TestTheRecordSurvivesAReopen(t *testing.T) { + cfg := testConfig(t, false) + v, err := Open(cfg) + require.NoError(t, err) + commitRange(t, v, 1, 5) + require.NoError(t, v.Commit(4, hashOf(0xEE))) + require.NoError(t, v.Close()) + + reopened := openVault(t, cfg) + head, recorded := reopened.Head() + require.True(t, recorded) + require.Equal(t, uint64(4), head) + requireHash(t, reopened, 3, hashOf(3)) + requireHash(t, reopened, 4, hashOf(0xEE)) + require.ErrorContains(t, reopened.Commit(6, hashOf(6)), "gap", "the reopened vault still refuses gaps") +} + +// Reset leaves the block it is given as the only one recorded, and commits carry on from it. +func TestResetLeavesOnlyTheGivenBlock(t *testing.T) { + v := openVault(t, testConfig(t, true)) + commitRange(t, v, 1, 5) + + require.NoError(t, v.Reset(1000, hashOf(0xAB))) + head, recorded := v.Head() + require.True(t, recorded) + require.Equal(t, uint64(1000), head) + requireHash(t, v, 1000, hashOf(0xAB)) + require.Equal(t, uint64(1), v.table.KeyCount()) + + require.NoError(t, v.Commit(1001, hashOf(0xAC))) + requireHash(t, v, 1001, hashOf(0xAC)) +} + +// The Pebble vault this one replaced holds app hashes nothing can use, so opening deletes it. +func TestOpenDeletesTheLegacyPebbleVault(t *testing.T) { + cfg := testConfig(t, true) + cfg.LegacyPebbleDir = filepath.Join(t.TempDir(), "hashvault") + require.NoError(t, os.MkdirAll(cfg.LegacyPebbleDir, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(cfg.LegacyPebbleDir, "000001.log"), []byte("x"), 0o600)) + + openVault(t, cfg) + _, err := os.Stat(cfg.LegacyPebbleDir) + require.True(t, os.IsNotExist(err), "the legacy vault must be gone, got %v", err) +} + +// A legacy dir that is not there is the common case once the testnet has run this build, not an error. +func TestOpenWithoutALegacyPebbleVault(t *testing.T) { + cfg := testConfig(t, true) + cfg.LegacyPebbleDir = filepath.Join(t.TempDir(), "absent") + openVault(t, cfg) +} + +// A hash may be deleted only once both the owner and the storage garbage collector permit it, and neither +// permission is taken back by a later, lower one. +func TestGCDeletesOnlyBelowBothFloors(t *testing.T) { + v := openVault(t, testConfig(t, true)) + deletable := func(blockNumber uint64) bool { + t.Helper() + ok, err := v.gcFilter(encodeKey(blockNumber), true) require.NoError(t, err) + return ok } - // Committing a *different* hash at any of those heights from many goroutines should yield - // at least one mismatch error and never a hidden success. - errs3 := make(chan error, N) - for i := 0; i < N; i++ { - wg.Add(1) - go func(h uint64) { - defer wg.Done() - errs3 <- v.CommitToHash(ctx, h, bytesOfLen(0xFF, 32)) - }(uint64(i + 1)) - } - wg.Wait() - close(errs3) - mismatches := 0 - for err := range errs3 { - require.Error(t, err) - if errors.Is(err, ErrHashMismatch) { - mismatches++ - } - } - require.Equal(t, N, mismatches, "every concurrent different-hash commit must return ErrHashMismatch") + require.False(t, deletable(0), "nothing is deletable before either floor is raised") + + v.PruneBelow(100) + require.False(t, deletable(50), "the owner alone cannot delete a hash") + + require.NoError(t, v.PruneHistory(60)) + require.True(t, deletable(59)) + require.False(t, deletable(60), "the lower floor bounds what is deleted") + require.False(t, deletable(99)) + + require.NoError(t, v.PruneHistory(200)) + require.True(t, deletable(99)) + require.False(t, deletable(100), "the owner's floor now bounds what is deleted") + + v.PruneBelow(10) + require.NoError(t, v.PruneHistory(10)) + require.True(t, deletable(99), "a lower floor must not take back a permission already given") } -// Sanity check that fmt.Errorf wrapping of our sentinels via %w stays Is-compatible. Defends -// against accidental future refactors of the codec or handlers that lose the sentinel. -func TestErrorWrappingIsCompatible(t *testing.T) { - wrapped := fmt.Errorf("outer: %w", ErrCorruption) - require.ErrorIs(t, wrapped, ErrCorruption) - wrappedLen := fmt.Errorf("outer: %w", ErrInvalidHashLength) - require.ErrorIs(t, wrappedLen, ErrInvalidHashLength) +// The vault restores nothing from snapshots, so its rollback floor is its newest block less the window. +func TestRollbackFloorIsTheHeadLessTheWindow(t *testing.T) { + v := openVault(t, testConfig(t, true)) + require.Equal(t, uint64(0), v.GetRollbackFloor(10), "an empty vault constrains nothing") + + commitRange(t, v, 1, 30) + require.Equal(t, uint64(20), v.GetRollbackFloor(10)) + require.Equal(t, uint64(0), v.GetRollbackFloor(40), "a window deeper than the history floors at 0") + latest, err := v.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(30), latest) +} + +// A record written in a format this build does not know is refused rather than read as a hash. +func TestAnUnknownRecordFormatIsRefused(t *testing.T) { + value := encodeValue(hashOf(1)) + value[0] = recordFormatVersion + 1 + _, err := decodeValue(value) + require.ErrorContains(t, err, "format version") + + _, err = decodeValue(value[:10]) + require.ErrorContains(t, err, "bytes") +} + +// Every method fails once the vault is closed, rather than reading a table that is gone. +func TestAClosedVaultRefusesEverything(t *testing.T) { + v, err := Open(testConfig(t, true)) + require.NoError(t, err) + commitRange(t, v, 1, 2) + require.NoError(t, v.Close()) + require.NoError(t, v.Close(), "closing twice is harmless") + + require.Error(t, v.Commit(3, hashOf(3))) + require.Error(t, v.Reset(3, hashOf(3))) + _, status, err := v.Get(1) + require.Error(t, err) + require.Equal(t, gigatypes.BlockHashStatusError, status) } diff --git a/sei-db/state_db/sc/hashvault/noop_hashvault.go b/sei-db/state_db/sc/hashvault/noop_hashvault.go deleted file mode 100644 index 65df820cd8..0000000000 --- a/sei-db/state_db/sc/hashvault/noop_hashvault.go +++ /dev/null @@ -1,30 +0,0 @@ -package hashvault - -import "context" - -var _ HashVault = (*NoopHashVault)(nil) - -// NoopHashVault is a HashVault implementation that does nothing. It provides no equivocation -// protection whatsoever. It exists for two purposes: -// - tests that construct a BlockExecutor but do not exercise the vault, and -// - the explicit, operator-opted-in "hash-vault-disabled-unsafe" escape hatch. -// -// Production code must never substitute this for a real vault without a deliberate human decision. -type NoopHashVault struct{} - -// NewNoopHashVault returns a HashVault whose methods are all no-ops. -func NewNoopHashVault() *NoopHashVault { - return &NoopHashVault{} -} - -func (n *NoopHashVault) CommitToHash(_ context.Context, _ uint64, _ []byte) error { - return nil -} - -func (n *NoopHashVault) Prune(_ context.Context, _ uint64) error { - return nil -} - -func (n *NoopHashVault) Close(_ context.Context) error { - return nil -} diff --git a/sei-db/state_db/sc/hashvault/offline.go b/sei-db/state_db/sc/hashvault/offline.go new file mode 100644 index 0000000000..d0f63afbd0 --- /dev/null +++ b/sei-db/state_db/sc/hashvault/offline.go @@ -0,0 +1,119 @@ +package hashvault + +import ( + "errors" + "fmt" + "math" + "os" + + "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/offline" +) + +// StoredRange returns the oldest and newest blocks the closed vault under cfg.DataDir records, and false +// when it records none. A vault that has never been created records none. +func StoredRange(cfg config.HashVaultConfig) (oldest uint64, newest uint64, recorded bool, err error) { + exists, err := vaultExists(cfg) + if err != nil { + return 0, 0, false, fmt.Errorf("read the hash vault's range: %w", err) + } + if !exists { + return 0, 0, false, nil + } + oldest, recorded, err = firstStoredBlock(cfg, false) + if err != nil { + return 0, 0, false, fmt.Errorf("read the oldest hash vault block: %w", err) + } + if !recorded { + return 0, 0, false, nil + } + newest, _, err = firstStoredBlock(cfg, true) + if err != nil { + return 0, 0, false, fmt.Errorf("read the newest hash vault block: %w", err) + } + return oldest, newest, true, nil +} + +// PruneAfter deletes every hash the closed vault under cfg.DataDir records above blockNumber. +func PruneAfter(cfg config.HashVaultConfig, blockNumber uint64) error { + if blockNumber == math.MaxUint64 { + return nil + } + if err := pruneFrom(cfg, blockNumber+1); err != nil { + return fmt.Errorf("prune the hash vault after block %d: %w", blockNumber, err) + } + return nil +} + +// pruneFrom deletes every hash the closed vault under cfg.DataDir records at or above blockNumber. A +// vault left with no hash is left empty. +func pruneFrom(cfg config.HashVaultConfig, blockNumber uint64) error { + exists, err := vaultExists(cfg) + if err != nil { + return fmt.Errorf("prune the hash vault: %w", err) + } + if !exists { + return nil + } + littCfg, err := littConfig(cfg) + if err != nil { + return fmt.Errorf("prune the hash vault: %w", err) + } + // The rollback walks from the newest hash down and keeps everything from the first one this accepts. + keep := func(_ string, key []byte, _ bool) (bool, error) { + recordedBlock, err := decodeKey(key) + if err != nil { + return false, fmt.Errorf("decode a hash vault key: %w", err) + } + return recordedBlock < blockNumber, nil + } + if err := offline.RollbackLittDB(littCfg, keep); err != nil { + return fmt.Errorf("prune the hash vault from block %d: %w", blockNumber, err) + } + return nil +} + +// firstStoredBlock returns the first block the closed vault yields in the direction reverse selects: the +// newest when reverse is true, the oldest otherwise. It returns false when the vault records none. +func firstStoredBlock(cfg config.HashVaultConfig, reverse bool) (blockNumber uint64, found bool, err error) { + littCfg, err := littConfig(cfg) + if err != nil { + return 0, false, fmt.Errorf("iterate the hash vault offline: %w", err) + } + iterator, err := offline.NewIterator(littCfg, tableName, reverse) + if err != nil { + return 0, false, fmt.Errorf("open an offline iterator over the hash vault: %w", err) + } + defer func() { + if closeErr := iterator.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("close the offline hash vault iterator: %w", closeErr)) + } + }() + found, err = iterator.Next() + if err != nil { + return 0, false, fmt.Errorf("iterate the hash vault offline: %w", err) + } + if !found { + return 0, false, nil + } + key, _, err := iterator.GetKey() + if err != nil { + return 0, false, fmt.Errorf("read a hash vault key: %w", err) + } + blockNumber, err = decodeKey(key) + if err != nil { + return 0, false, fmt.Errorf("decode a hash vault key: %w", err) + } + return blockNumber, true, nil +} + +// vaultExists reports whether the vault's directory exists. +func vaultExists(cfg config.HashVaultConfig) (bool, error) { + if _, err := os.Stat(cfg.DataDir); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("stat the hash vault dir %q: %w", cfg.DataDir, err) + } + return true, nil +} diff --git a/sei-db/state_db/sc/hashvault/offline_test.go b/sei-db/state_db/sc/hashvault/offline_test.go new file mode 100644 index 0000000000..dfa7681ccc --- /dev/null +++ b/sei-db/state_db/sc/hashvault/offline_test.go @@ -0,0 +1,77 @@ +package hashvault + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// A vault that was never created records nothing, which is what a node booting for the first time finds. +func TestStoredRangeOfAMissingVault(t *testing.T) { + _, _, recorded, err := StoredRange(testConfig(t, true)) + require.NoError(t, err) + require.False(t, recorded) +} + +// A vault that exists but holds no hashes records nothing either. +func TestStoredRangeOfAnEmptyVault(t *testing.T) { + cfg := testConfig(t, true) + v, err := Open(cfg) + require.NoError(t, err) + require.NoError(t, v.Close()) + + _, _, recorded, err := StoredRange(cfg) + require.NoError(t, err) + require.False(t, recorded) +} + +// The range read offline is the one the vault holds when open. +func TestStoredRangeOfAPopulatedVault(t *testing.T) { + cfg := testConfig(t, true) + v, err := Open(cfg) + require.NoError(t, err) + commitRange(t, v, 4, 9) + require.NoError(t, v.Close()) + + oldest, newest, recorded, err := StoredRange(cfg) + require.NoError(t, err) + require.True(t, recorded) + require.Equal(t, uint64(4), oldest) + require.Equal(t, uint64(9), newest) +} + +// PruneAfter keeps the hashes up to its block and drops the rest, and the vault reopens on what is kept. +func TestPruneAfterKeepsThePrefix(t *testing.T) { + cfg := testConfig(t, true) + v, err := Open(cfg) + require.NoError(t, err) + commitRange(t, v, 1, 9) + require.NoError(t, v.Close()) + + require.NoError(t, PruneAfter(cfg, 5)) + + _, newest, recorded, err := StoredRange(cfg) + require.NoError(t, err) + require.True(t, recorded) + require.Equal(t, uint64(5), newest) + + reopened := openVault(t, cfg) + requireHash(t, reopened, 5, hashOf(5)) + require.NoError(t, reopened.Commit(6, hashOf(0xEE)), "commits carry on from the kept prefix") +} + +// Pruning below every recorded hash leaves an empty vault, and pruning a missing one does nothing. +func TestPruneAfterBelowEveryHashEmptiesTheVault(t *testing.T) { + cfg := testConfig(t, true) + require.NoError(t, PruneAfter(cfg, 0), "a vault that was never created has nothing to prune") + + v, err := Open(cfg) + require.NoError(t, err) + commitRange(t, v, 3, 5) + require.NoError(t, v.Close()) + + require.NoError(t, PruneAfter(cfg, 2)) + _, _, recorded, err := StoredRange(cfg) + require.NoError(t, err) + require.False(t, recorded) +} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault.go b/sei-db/state_db/sc/hashvault/pebble_hashvault.go deleted file mode 100644 index 0c1a8d7722..0000000000 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault.go +++ /dev/null @@ -1,253 +0,0 @@ -package hashvault - -import ( - "bytes" - "context" - "encoding/hex" - "errors" - "fmt" - "os" - "sync" - - "github.com/cockroachdb/pebble/v2" - "github.com/ethereum/go-ethereum/common/lru" - "github.com/sei-protocol/seilog" -) - -var _ HashVault = (*PebbleHashVault)(nil) - -var logger = seilog.NewLogger("db", "state-db", "sc", "hashvault") - -// PebbleHashVault is a PebbleDB-backed implementation of the HashVault interface. -type PebbleHashVault struct { - config HashVaultConfig - db *pebble.DB - writeOpts *pebble.WriteOptions - - mu sync.Mutex - // closed is true after Close. Every other public method returns ErrClosed once set. - closed bool - // pruneBoundary is the lowest height that may still be committed. - pruneBoundary uint64 - cache *lru.Cache[uint64, []byte] -} - -// NewPebbleHashVault opens (or creates) a PebbleHashVault rooted at config.DataDir. -func NewPebbleHashVault(ctx context.Context, config HashVaultConfig) (*PebbleHashVault, error) { - if !config.Fsync { - logger.Info("forcing fsync on for production PebbleHashVault", "dataDir", config.DataDir) - } - config.Fsync = true - return newPebbleHashVault(ctx, config) -} - -// NewUnsafePebbleHashVault opens (or creates) a PebbleHashVault rooted at config.DataDir. Honors -// config.Fsync as set; intended for tests only. Never use in production: disabling fsync means a -// well-timed crash can lose the most recent committed hash and let the node vote a different hash -// for that block on the next boot. -func NewUnsafePebbleHashVault(ctx context.Context, config HashVaultConfig) (*PebbleHashVault, error) { - return newPebbleHashVault(ctx, config) -} - -func newPebbleHashVault(_ context.Context, config HashVaultConfig) (*PebbleHashVault, error) { - if err := config.Validate(); err != nil { - return nil, fmt.Errorf("invalid hashvault config: %w", err) - } - - if err := os.MkdirAll(config.DataDir, 0o750); err != nil { - return nil, fmt.Errorf("failed to create hashvault data dir %q: %w", config.DataDir, err) - } - - db, err := pebble.Open(config.DataDir, &pebble.Options{}) - if err != nil { - return nil, fmt.Errorf("failed to open hashvault pebble db at %q: %w", config.DataDir, err) - } - - writeOpts := pebble.Sync - if !config.Fsync { - writeOpts = pebble.NoSync - } - - p := &PebbleHashVault{ - config: config, - db: db, - writeOpts: writeOpts, - cache: lru.NewCache[uint64, []byte](config.CacheSize), - } - - if err := p.loadPruneBoundary(); err != nil { - _ = db.Close() - return nil, err - } - - empty, err := p.isEmpty() - if err != nil { - _ = db.Close() - return nil, err - } - if empty { - // Surface the fresh-start case: an operator who expected this node to already have an - // equivocation history on disk (e.g. after a restart) should notice an empty vault. - logger.Info("opened hashvault with no data on disk; starting with an empty equivocation history", - "dataDir", config.DataDir) - } - - return p, nil -} - -// isEmpty reports whether the underlying DB holds no keys at all (a freshly created vault with no -// committed hashes and no prune boundary). -func (p *PebbleHashVault) isEmpty() (bool, error) { - iter, err := p.db.NewIter(nil) - if err != nil { - return false, fmt.Errorf("failed to open hashvault iterator: %w", err) - } - defer func() { _ = iter.Close() }() - return !iter.First(), nil -} - -// loadPruneBoundary reads the on-disk prune boundary (if any) and populates p.pruneBoundary. -func (p *PebbleHashVault) loadPruneBoundary() error { - raw, closer, err := p.db.Get(pruneBoundaryKey) - if err != nil { - if errors.Is(err, pebble.ErrNotFound) { - return nil - } - return fmt.Errorf("failed to read prune boundary: %w", err) - } - defer func() { _ = closer.Close() }() - - boundary, err := decodeBoundaryValue(raw) - if err != nil { - logger.Error("hashvault prune boundary is malformed; refusing to start", - "dataDir", p.config.DataDir, "rawHex", hex.EncodeToString(raw), "err", err) - return err - } - p.pruneBoundary = boundary - return nil -} - -// CommitToHash implements HashVault. -func (p *PebbleHashVault) CommitToHash(ctx context.Context, blockHeight uint64, hash []byte) error { - if err := ctx.Err(); err != nil { - return err - } - p.mu.Lock() - defer p.mu.Unlock() - - if p.closed { - return ErrClosed - } - if blockHeight < p.pruneBoundary { - return ErrBelowPruneBoundary - } - if len(hash) != BlockHashSize { - return ErrInvalidHashLength - } - - if cached, ok := p.cache.Get(blockHeight); ok { - if !bytes.Equal(cached, hash) { - p.logHashMismatch(blockHeight, cached, hash) - return ErrHashMismatch - } - return nil - } - - key := hashKey(blockHeight) - raw, closer, err := p.db.Get(key) - switch { - case errors.Is(err, pebble.ErrNotFound): - // First commit for this height: write it. - value := encodeHashValue(blockHeight, hash) - if werr := p.db.Set(key, value, p.writeOpts); werr != nil { - return fmt.Errorf("failed to persist hash for block %d: %w", blockHeight, werr) - } - p.cache.Add(blockHeight, bytes.Clone(hash)) - return nil - case err != nil: - return fmt.Errorf("failed to read hash for block %d: %w", blockHeight, err) - } - // Found an existing entry; clone the raw bytes so we can release the closer before doing - // further work. - cloned := bytes.Clone(raw) - _ = closer.Close() - - existing, err := decodeHashValue(blockHeight, cloned) - if err != nil { - logger.Error("hashvault detected on-disk corruption; DO NOT RESTART WITHOUT HUMAN INVESTIGATION", - "blockHeight", blockHeight, "rawHex", hex.EncodeToString(cloned), "err", err) - return err - } - if !bytes.Equal(existing, hash) { - p.logHashMismatch(blockHeight, existing, hash) - return ErrHashMismatch - } - p.cache.Add(blockHeight, existing) - return nil -} - -// Prune implements HashVault. The boundary advance and range deletion are written in a single -// atomic Pebble batch: a crash mid-Prune either rolls forward to the new boundary (with the -// deletions applied) or leaves the old state intact. On return, every height strictly below -// blockHeight is guaranteed durable-deleted (subject to config.Fsync). -func (p *PebbleHashVault) Prune(ctx context.Context, blockHeight uint64) error { - if err := ctx.Err(); err != nil { - return err - } - p.mu.Lock() - defer p.mu.Unlock() - - if p.closed { - return ErrClosed - } - if blockHeight <= p.pruneBoundary { - return nil - } - - batch := p.db.NewBatch() - defer func() { _ = batch.Close() }() - if err := batch.Set(pruneBoundaryKey, encodeBoundaryValue(blockHeight), nil); err != nil { - return fmt.Errorf("failed to stage prune boundary advance to %d: %w", blockHeight, err) - } - // DeleteRange's upper bound is exclusive, so hashKey(blockHeight) keeps the boundary block - // itself per the HashVault.Prune contract. - if err := batch.DeleteRange(hashKey(0), hashKey(blockHeight), nil); err != nil { - return fmt.Errorf("failed to stage prune deletion below %d: %w", blockHeight, err) - } - if err := batch.Commit(p.writeOpts); err != nil { - return fmt.Errorf("failed to commit prune to %d: %w", blockHeight, err) - } - - p.pruneBoundary = blockHeight - return nil -} - -// Close implements HashVault. Subsequent calls return nil. After Close, every other public method -// returns ErrClosed. -func (p *PebbleHashVault) Close(_ context.Context) error { - p.mu.Lock() - defer p.mu.Unlock() - if p.closed { - return nil - } - p.closed = true - p.cache.Purge() - if err := p.db.Close(); err != nil { - return fmt.Errorf("failed to close hashvault pebble db: %w", err) - } - return nil -} - -func (p *PebbleHashVault) logHashMismatch(blockHeight uint64, existing, incoming []byte) { - logger.Error("Hashvault detected app hash mismatch; node attempted to change its mind. "+ - "DO NOT RESTART WITHOUT HUMAN INVESTIGATION. If you are CERTAIN this is not a real "+ - "equivocation, you can bypass this guard by stopping the node and deleting the HashVault "+ - "data directory (hashVaultDir below), then restarting. WARNING: deleting it removes "+ - "equivocation protection — if the node then commits a conflicting hash for a height it has "+ - "already finalized, the validator may be SLASHED.", - "blockHeight", blockHeight, - "existingHex", hex.EncodeToString(existing), - "incomingHex", hex.EncodeToString(incoming), - "hashVaultDir", p.config.DataDir, - ) -} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_codec.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_codec.go deleted file mode 100644 index 139632d8d5..0000000000 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault_codec.go +++ /dev/null @@ -1,160 +0,0 @@ -package hashvault - -import ( - "bytes" - "crypto/sha256" - "encoding/binary" - "fmt" - "math" - "strconv" -) - -// Wire format -// -// Hash entries live under keys "h" , where is fixed-width 20-digit zero-padded -// ASCII decimal (twenty digits is the exact width of math.MaxUint64). Fixed width is what makes -// Pebble's lexicographic key order match numeric height order, which is what lets Prune wipe a -// contiguous range with a single DeleteRange. ASCII (vs binary BE) is chosen so a raw Pebble dump -// shows recognizable numbers (e.g. "h00000000000000000042") rather than opaque bytes. -// -// Each hash entry's value is the raw block hash followed by a 32-byte SHA-256 trailer computed -// over (be-uint64 height || hash). The trailer is what protects against the validator -// double-voting after silent corruption: the height is folded into the SHA so a stale entry -// returned from the wrong key path also fails verification. The trailer's height encoding is -// binary BE because that representation never appears on disk in raw form (it's consumed entirely -// by the SHA), so the human-readability argument doesn't apply there. -// -// The prune boundary lives under the single key "prune_boundary". Its value is the boundary -// height as variable-width unpadded ASCII decimal (e.g. "5" or "18446744073709551615"). No -// padding because there's only one such row and no range scan to satisfy. No checksum: the -// boundary is GC bookkeeping, a silent flip is not slashable, and Pebble's own block-level CRC -// catches bit-rot in normal operation. -// -// "h" and "prune_boundary" have disjoint first bytes ('h' vs 'p'), so the two namespaces can never -// alias regardless of what digits follow the hash prefix. - -const checksumSize = sha256.Size - -// heightDigits is the on-disk width (in ASCII bytes) of every encoded height. math.MaxUint64 is -// 18446744073709551615, exactly 20 digits. -const heightDigits = 20 - -var ( - hashKeyPrefix = []byte("h") - pruneBoundaryKey = []byte("prune_boundary") -) - -// hashKey returns the Pebble key for the given block height: hashKeyPrefix followed by the height -// as 20-digit zero-padded ASCII decimal. -func hashKey(height uint64) []byte { - out := make([]byte, 0, len(hashKeyPrefix)+heightDigits) - out = append(out, hashKeyPrefix...) - return appendHeight(out, height) -} - -// decodeHashKey is the inverse of hashKey: validates the length and prefix, then parses the -// trailing decimal digits. Returns ErrCorruption on any malformedness. -func decodeHashKey(key []byte) (uint64, error) { - if len(key) != len(hashKeyPrefix)+heightDigits { - return 0, fmt.Errorf("%w: unexpected hash key length %d", ErrCorruption, len(key)) - } - if !bytes.HasPrefix(key, hashKeyPrefix) { - return 0, fmt.Errorf("%w: hash key missing prefix", ErrCorruption) - } - return parseHeight(key[len(hashKeyPrefix):]) -} - -// hashKeyUpperBound returns an end-exclusive Pebble key that is strictly greater than every key -// hashKey can produce (i.e. up to and including hashKey(math.MaxUint64)). Safe to use as an -// IterOptions.UpperBound or as the upper end of a DeleteRange covering the entire hash namespace. -func hashKeyUpperBound() []byte { - // One byte longer than any valid hash key, so lex-greater than all of them. - return append(hashKey(math.MaxUint64), 0x00) -} - -// encodeHashValue returns the on-disk value for the given (height, hash) pair: the hash bytes -// followed by SHA-256(be(height) || hash). -func encodeHashValue(height uint64, hash []byte) []byte { - out := make([]byte, 0, len(hash)+checksumSize) - out = append(out, hash...) - out = append(out, hashChecksum(height, hash)...) - return out -} - -// decodeHashValue verifies the trailing SHA-256 of raw against (height, hash[:len(raw)-32]) and -// returns the hash bytes on success. Returns ErrCorruption if the trailer is missing or wrong. -func decodeHashValue(height uint64, raw []byte) ([]byte, error) { - if len(raw) < checksumSize { - return nil, fmt.Errorf("%w: value too short for height %d (%d bytes)", ErrCorruption, height, len(raw)) - } - split := len(raw) - checksumSize - hash := raw[:split] - trailer := raw[split:] - expected := hashChecksum(height, hash) - if !bytes.Equal(trailer, expected) { - return nil, fmt.Errorf("%w: checksum mismatch for height %d", ErrCorruption, height) - } - return bytes.Clone(hash), nil -} - -// encodeBoundaryValue returns the on-disk value for the prune boundary: variable-width unpadded -// ASCII decimal. There's only one boundary row in the DB and nothing range-scans the value, so -// fixed-width padding (as used for keys) buys nothing here. -func encodeBoundaryValue(boundary uint64) []byte { - return strconv.AppendUint(nil, boundary, 10) -} - -// decodeBoundaryValue parses an ASCII-decimal boundary value. Empty/oversized/non-digit inputs all -// trip ErrCorruption; Pebble's own CRC handles bit-rot within an otherwise-valid value. -func decodeBoundaryValue(raw []byte) (uint64, error) { - // math.MaxUint64 is 20 digits; anything longer can't be a valid uint64 and is suspect. - if len(raw) == 0 || len(raw) > heightDigits { - return 0, fmt.Errorf("%w: unexpected boundary value length %d", ErrCorruption, len(raw)) - } - n, err := strconv.ParseUint(string(raw), 10, 64) - if err != nil { - return 0, fmt.Errorf("%w: invalid boundary digits %q: %v", ErrCorruption, raw, err) - } - return n, nil -} - -// hashChecksum returns SHA-256(be(height) || hash). The height is encoded as binary BE here, not -// ASCII, because the result is hashed in place and never appears on disk in raw form. -func hashChecksum(height uint64, hash []byte) []byte { - h := sha256.New() - var buf [8]byte - binary.BigEndian.PutUint64(buf[:], height) - _, _ = h.Write(buf[:]) - _, _ = h.Write(hash) - return h.Sum(nil) -} - -// appendHeight appends 20-digit zero-padded decimal to dst and returns the result. -func appendHeight(dst []byte, height uint64) []byte { - var buf [heightDigits]byte - i := len(buf) - for height > 0 { - i-- - buf[i] = byte('0' + height%10) - height /= 10 - } - for i > 0 { - i-- - buf[i] = '0' - } - return append(dst, buf[:]...) -} - -// parseHeight parses exactly heightDigits decimal bytes into a uint64. Returns ErrCorruption on -// any non-digit byte; uint64 cannot overflow because 20 digits is the exact width of math.MaxUint64 -// and ParseUint with bitSize=64 rejects values above MaxUint64. -func parseHeight(raw []byte) (uint64, error) { - if len(raw) != heightDigits { - return 0, fmt.Errorf("%w: expected %d digits, got %d", ErrCorruption, heightDigits, len(raw)) - } - n, err := strconv.ParseUint(string(raw), 10, 64) - if err != nil { - return 0, fmt.Errorf("%w: invalid height digits %q: %v", ErrCorruption, raw, err) - } - return n, nil -} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback.go deleted file mode 100644 index da57093aa2..0000000000 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback.go +++ /dev/null @@ -1,111 +0,0 @@ -package hashvault - -import ( - "context" - "encoding/hex" - "errors" - "fmt" - "math" - "os" - - "github.com/cockroachdb/pebble/v2" -) - -// HardRollbackPebbleHashVault deletes every recorded hash strictly above blockHeight from the -// on-disk vault rooted at config.DataDir and clears the prune boundary. This is a break-glass -// operator tool: after it returns, commits at any height are allowed until Prune is run again. -func HardRollbackPebbleHashVault(_ context.Context, config HashVaultConfig, blockHeight uint64) error { - if err := config.Validate(); err != nil { - return fmt.Errorf("invalid hashvault config: %w", err) - } - - // Refuse if the data dir doesn't already exist. pebble.Open would otherwise silently create a - // fresh empty DB at a typo'd path and report a "successful" no-op rollback, which is exactly - // the kind of operator-error-eaten-by-tooling we want to avoid in a CLI tool. - if _, err := os.Stat(config.DataDir); err != nil { - return fmt.Errorf("hashvault data dir %q is not accessible: %w", config.DataDir, err) - } - - db, err := pebble.Open(config.DataDir, &pebble.Options{}) - if err != nil { - return fmt.Errorf("failed to open hashvault pebble db at %q: %w", config.DataDir, err) - } - defer func() { _ = db.Close() }() - - boundary, err := readPersistedBoundary(db) - if err != nil { - return err - } - - if blockHeight < boundary { - return wipeEntireStore(db, config.DataDir, blockHeight, boundary) - } - - // Partial rollback uses DeleteRange(hashKey(blockHeight+1), ...). At math.MaxUint64 the +1 - // wraps to 0, so hashKey(0) becomes the range start and every hash entry is deleted. - if blockHeight == math.MaxUint64 { - return fmt.Errorf("cannot hard rollback above block %d: %w", blockHeight, ErrRollbackHeightOverflow) - } - - return hardRollbackAbove(db, config.DataDir, blockHeight) -} - -// hardRollbackAbove deletes hashes strictly above blockHeight and clears the prune boundary in one -// atomic batch. -func hardRollbackAbove(db *pebble.DB, dataDir string, blockHeight uint64) error { - batch := db.NewBatch() - defer func() { _ = batch.Close() }() - if err := batch.DeleteRange(hashKey(blockHeight+1), hashKeyUpperBound(), nil); err != nil { - return fmt.Errorf("failed to stage hard rollback above block %d: %w", blockHeight, err) - } - if err := batch.Delete(pruneBoundaryKey, nil); err != nil { - return fmt.Errorf("failed to stage prune boundary clear during hard rollback: %w", err) - } - if err := batch.Commit(pebble.Sync); err != nil { - return fmt.Errorf("failed to hard rollback above block %d: %w", blockHeight, err) - } - logger.Info("hashvault hard rollback completed", - "dataDir", dataDir, "blockHeight", blockHeight) - return nil -} - -// readPersistedBoundary returns the on-disk prune boundary, or zero if none has ever been written. -// A malformed boundary record is logged and surfaced as ErrCorruption so the operator must -// investigate before proceeding. -func readPersistedBoundary(db *pebble.DB) (uint64, error) { - raw, closer, err := db.Get(pruneBoundaryKey) - if errors.Is(err, pebble.ErrNotFound) { - return 0, nil - } - if err != nil { - return 0, fmt.Errorf("failed to read prune boundary: %w", err) - } - defer func() { _ = closer.Close() }() - - boundary, err := decodeBoundaryValue(raw) - if err != nil { - logger.Error("hashvault prune boundary is malformed; refusing rollback", - "rawHex", hex.EncodeToString(raw), "err", err) - return 0, err - } - return boundary, nil -} - -// wipeEntireStore drops every hash entry and the prune boundary record in a single atomic Pebble -// batch, leaving the store indistinguishable from a freshly-initialized vault. -func wipeEntireStore(db *pebble.DB, dataDir string, target, boundary uint64) error { - batch := db.NewBatch() - defer func() { _ = batch.Close() }() - if err := batch.DeleteRange(hashKey(0), hashKeyUpperBound(), nil); err != nil { - return fmt.Errorf("failed to stage wipe of hash range: %w", err) - } - if err := batch.Delete(pruneBoundaryKey, nil); err != nil { - return fmt.Errorf("failed to stage wipe of prune boundary: %w", err) - } - if err := batch.Commit(pebble.Sync); err != nil { - return fmt.Errorf("failed to wipe hashvault store: %w", err) - } - logger.Warn("hashvault rollback target is below prune boundary; wiped entire store", - "dataDir", dataDir, "rollbackTarget", target, "pruneBoundary", boundary) - return nil -} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go deleted file mode 100644 index e6e2d8e5b0..0000000000 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package hashvault - -import ( - "context" - "math" - "path/filepath" - "testing" - - "github.com/cockroachdb/pebble/v2" - "github.com/stretchr/testify/require" -) - -// TestHardRollbackPebbleHashVault covers the happy path: an existing commit at height 10 is -// removed by rolling back to height 5; a fresh commit at 10 with a different hash then succeeds -// and is itself locked in. The vault must be closed before invoking the static function (Pebble's -// directory lock would otherwise refuse). -func TestHardRollbackPebbleHashVault(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - - a := bytesOfLen(0xAA, 32) - b := bytesOfLen(0xBB, 32) - require.NoError(t, v.CommitToHash(ctx, 10, a)) - - cfg := v.config - require.NoError(t, v.Close(ctx)) - - require.NoError(t, HardRollbackPebbleHashVault(ctx, cfg, 5)) - - v2, err := NewUnsafePebbleHashVault(ctx, cfg) - require.NoError(t, err) - t.Cleanup(func() { _ = v2.Close(ctx) }) - - require.NoError(t, v2.CommitToHash(ctx, 10, b)) - require.ErrorIs(t, v2.CommitToHash(ctx, 10, a), ErrHashMismatch) -} - -func TestHardRollbackPebbleHashVaultBelowPruneBoundaryWipesStore(t *testing.T) { - // When the rollback target is strictly below the boundary, "partial rollback" is incoherent: - // every surviving hash has height >= boundary > target, so there is no consistent state to - // preserve. The function wipes everything (hashes + boundary record) so the next boot looks - // like a freshly-initialized vault. - ctx := context.Background() - v := newTestPebbleVault(t) - - for h := uint64(30); h <= 50; h++ { - require.NoError(t, v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32))) - } - require.NoError(t, v.Prune(ctx, 30)) - - cfg := v.config - require.NoError(t, v.Close(ctx)) - - require.NoError(t, HardRollbackPebbleHashVault(ctx, cfg, 10)) - - v2, err := NewUnsafePebbleHashVault(ctx, cfg) - require.NoError(t, err) - t.Cleanup(func() { _ = v2.Close(ctx) }) - - // Boundary is gone, so commits below the old boundary are now accepted. - require.NoError(t, v2.CommitToHash(ctx, 5, bytesOfLen(0xCC, 32))) - // Every previously-locked hash is also gone — height 50 used to be 0x32, but the wipe means a - // fresh hash there is allowed. - require.NoError(t, v2.CommitToHash(ctx, 50, bytesOfLen(0xEE, 32))) -} - -func TestHardRollbackPebbleHashVaultEqualToPruneBoundary(t *testing.T) { - // Rollback target == boundary: hashes above the target are removed, the boundary block is kept, - // and the prune boundary record is cleared so commits below the old boundary are allowed again. - ctx := context.Background() - v := newTestPebbleVault(t) - require.NoError(t, v.Prune(ctx, 100)) - cfg := v.config - require.NoError(t, v.Close(ctx)) - - require.NoError(t, HardRollbackPebbleHashVault(ctx, cfg, 100)) - - v2, err := NewUnsafePebbleHashVault(ctx, cfg) - require.NoError(t, err) - t.Cleanup(func() { _ = v2.Close(ctx) }) - require.NoError(t, v2.CommitToHash(ctx, 50, bytesOfLen(0xAA, 32))) -} - -func TestHardRollbackPebbleHashVaultRejectsLockedDir(t *testing.T) { - // Sanity check that the static function fails fast when a live vault still holds the Pebble - // directory lock — the whole point of being out-of-process is to make accidental concurrent - // use a clean error, not a silent corruption. - ctx := context.Background() - v := newTestPebbleVault(t) - cfg := v.config - - err := HardRollbackPebbleHashVault(ctx, cfg, 5) - require.Error(t, err, "must refuse to open while the live vault holds the lock") -} - -func TestHardRollbackPebbleHashVaultRefusesMalformedBoundary(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - cfg := v.config - require.NoError(t, v.Close(ctx)) - - // Plant a malformed boundary record so the function has to reject before performing any write. - directWritePebble(t, cfg.DataDir, func(db *pebble.DB) { - require.NoError(t, db.Set(pruneBoundaryKey, []byte{0x00, 0x01}, pebble.Sync)) - }) - - err := HardRollbackPebbleHashVault(ctx, cfg, 100) - require.ErrorIs(t, err, ErrCorruption) -} - -func TestHardRollbackPebbleHashVaultRejectsMaxUint64Height(t *testing.T) { - // blockHeight+1 must not be used for DeleteRange start keys: at MaxUint64 it wraps to 0 and - // would wipe the entire vault. Refuse rather than silently destroy data. - ctx := context.Background() - v := newTestPebbleVault(t) - require.NoError(t, v.CommitToHash(ctx, math.MaxUint64, bytesOfLen(0xFF, 32))) - - cfg := v.config - require.NoError(t, v.Close(ctx)) - - err := HardRollbackPebbleHashVault(ctx, cfg, math.MaxUint64) - require.ErrorIs(t, err, ErrRollbackHeightOverflow) - - v2, err := NewUnsafePebbleHashVault(ctx, cfg) - require.NoError(t, err) - t.Cleanup(func() { _ = v2.Close(ctx) }) - - require.ErrorIs(t, v2.CommitToHash(ctx, math.MaxUint64, bytesOfLen(0xEE, 32)), ErrHashMismatch) -} - -func TestHardRollbackPebbleHashVaultRejectsMissingDir(t *testing.T) { - ctx := context.Background() - cfg := DefaultHashVaultConfig() - // Point at a path that definitely doesn't exist; pebble.Open is the source of truth for the - // error here — we just want to verify the static function surfaces it rather than silently - // creating a fresh empty vault and pretending the rollback succeeded. - cfg.DataDir = filepath.Join(t.TempDir(), "does-not-exist", "vault") - - err := HardRollbackPebbleHashVault(ctx, cfg, 5) - require.Error(t, err) -} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_test.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_test.go deleted file mode 100644 index d19ef80d78..0000000000 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault_test.go +++ /dev/null @@ -1,338 +0,0 @@ -package hashvault - -import ( - "bytes" - "context" - "path/filepath" - "sync" - "testing" - - "github.com/cockroachdb/pebble/v2" - "github.com/stretchr/testify/require" -) - -// newTestPebbleVault constructs an unsafe Pebble vault rooted in t.TempDir() and arranges for it -// to be closed at end-of-test. -func newTestPebbleVault(t *testing.T, configMutators ...func(*HashVaultConfig)) *PebbleHashVault { - t.Helper() - cfg := DefaultHashVaultConfig() - cfg.DataDir = filepath.Join(t.TempDir(), "vault") - for _, m := range configMutators { - m(&cfg) - } - v, err := NewUnsafePebbleHashVault(context.Background(), cfg) - require.NoError(t, err) - t.Cleanup(func() { - _ = v.Close(context.Background()) - }) - return v -} - -// reopenTestPebbleVault closes v then reopens a fresh PebbleHashVault at the same DataDir. The -// returned vault is cleaned up at end-of-test. -func reopenTestPebbleVault(t *testing.T, v *PebbleHashVault) *PebbleHashVault { - t.Helper() - dir := v.config.DataDir - require.NoError(t, v.Close(context.Background())) - cfg := DefaultHashVaultConfig() - cfg.DataDir = dir - reopened, err := NewUnsafePebbleHashVault(context.Background(), cfg) - require.NoError(t, err) - t.Cleanup(func() { - _ = reopened.Close(context.Background()) - }) - return reopened -} - -// directWritePebble opens the Pebble dir at path, applies fn to the db, then closes. Used by -// corruption tests to poke at on-disk values without going through the HashVault. -func directWritePebble(t *testing.T, path string, fn func(*pebble.DB)) { - t.Helper() - db, err := pebble.Open(path, &pebble.Options{}) - require.NoError(t, err) - defer func() { require.NoError(t, db.Close()) }() - fn(db) -} - -func TestRestartRecoversPruneBoundary(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - - require.NoError(t, v.Prune(ctx, 50)) - v2 := reopenTestPebbleVault(t, v) - - // Strictly below the recovered boundary is rejected. - require.ErrorIs(t, v2.CommitToHash(ctx, 25, bytesOfLen(0xAA, 32)), ErrBelowPruneBoundary) - require.ErrorIs(t, v2.CommitToHash(ctx, 49, bytesOfLen(0xAA, 32)), ErrBelowPruneBoundary) - // At and above the boundary are allowed. - require.NoError(t, v2.CommitToHash(ctx, 50, bytesOfLen(0xAA, 32))) - require.NoError(t, v2.CommitToHash(ctx, 51, bytesOfLen(0xAA, 32))) -} - -func TestRestartRecoversHashes(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - - hash := bytesOfLen(0xCD, 32) - require.NoError(t, v.CommitToHash(ctx, 99, hash)) - - v2 := reopenTestPebbleVault(t, v) - - // Re-commit the same hash: succeeds. - require.NoError(t, v2.CommitToHash(ctx, 99, hash)) - // Different hash: locked out. - require.ErrorIs(t, - v2.CommitToHash(ctx, 99, bytesOfLen(0xFF, 32)), - ErrHashMismatch, - ) -} - -func TestPruneRemovesDataOnDisk(t *testing.T) { - // The shared suite already covers the externally-visible Prune contract; this test exists to - // pin the on-disk effect — i.e. that the deleted heights are actually gone from Pebble (and - // not merely shadowed by the in-memory boundary check). - ctx := context.Background() - const total = uint64(50) - - v := newTestPebbleVault(t) - for h := uint64(1); h <= total; h++ { - require.NoError(t, v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32))) - } - - require.NoError(t, v.Prune(ctx, total)) - - dir := v.config.DataDir - require.NoError(t, v.Close(ctx)) - - var remaining []uint64 - directWritePebble(t, dir, func(db *pebble.DB) { - iter, err := db.NewIter(&pebble.IterOptions{ - LowerBound: hashKey(0), - UpperBound: hashKeyUpperBound(), - }) - require.NoError(t, err) - defer func() { _ = iter.Close() }() - for iter.First(); iter.Valid(); iter.Next() { - h, err := decodeHashKey(iter.Key()) - require.NoError(t, err) - remaining = append(remaining, h) - } - }) - // Per the Prune contract the boundary block itself is kept, so only height==total should remain. - require.Equal(t, []uint64{total}, remaining, - "only the prune-boundary block should remain after Prune") -} - -func TestKeyEncodingRoundtrip(t *testing.T) { - cases := []uint64{0, 1, 7, 1 << 30, 1<<63 - 1, ^uint64(0)} - for _, h := range cases { - k := hashKey(h) - require.Len(t, k, len(hashKeyPrefix)+heightDigits) - require.True(t, bytes.HasPrefix(k, hashKeyPrefix)) - got, err := decodeHashKey(k) - require.NoError(t, err) - require.Equal(t, h, got) - } -} - -func TestKeyEncodingOrderingMatchesNumeric(t *testing.T) { - // Spot-check that lex order over hashKey() matches numeric order, including non-adjacent - // magnitudes. This is the whole reason for zero-padded fixed-width encoding. - heights := []uint64{0, 1, 9, 10, 99, 100, 1<<32 - 1, 1 << 32, 1<<63 - 1, ^uint64(0) - 1, ^uint64(0)} - for i := 0; i+1 < len(heights); i++ { - a, b := hashKey(heights[i]), hashKey(heights[i+1]) - require.Lessf(t, bytes.Compare(a, b), 0, - "hashKey(%d)=%q must sort before hashKey(%d)=%q", heights[i], a, heights[i+1], b) - } -} - -func TestKeyEncodingHumanReadable(t *testing.T) { - // Pin the on-disk layout so a future "let's switch back to binary BE for size" PR has to - // explicitly delete this test. - require.Equal(t, "h00000000000000000042", string(hashKey(42))) - require.Equal(t, "h18446744073709551615", string(hashKey(^uint64(0)))) -} - -func TestDecodeHashKeyRejectsMalformed(t *testing.T) { - _, err := decodeHashKey([]byte("h0000000000000000004")) - require.ErrorIs(t, err, ErrCorruption, "short length") - _, err = decodeHashKey([]byte("x00000000000000000042")) - require.ErrorIs(t, err, ErrCorruption, "wrong prefix") - _, err = decodeHashKey([]byte("h0000000000000000004x")) - require.ErrorIs(t, err, ErrCorruption, "non-digit byte") -} - -func TestValueCodecRoundtrip(t *testing.T) { - hash := bytesOfLen(0xAA, 32) - for _, h := range []uint64{0, 1, 7, 1 << 30, ^uint64(0)} { - raw := encodeHashValue(h, hash) - got, err := decodeHashValue(h, raw) - require.NoError(t, err) - require.Equal(t, hash, got) - } - - for _, b := range []uint64{0, 1, 1234567890, ^uint64(0)} { - raw := encodeBoundaryValue(b) - got, err := decodeBoundaryValue(raw) - require.NoError(t, err) - require.Equal(t, b, got) - } -} - -func TestBoundaryValueIsUnpaddedDecimal(t *testing.T) { - // Boundary encoding is variable-width by design (one row, no range scan to satisfy). Pin it - // so a future "let's pad for symmetry with keys" change has to explicitly delete this test. - require.Equal(t, "0", string(encodeBoundaryValue(0))) - require.Equal(t, "42", string(encodeBoundaryValue(42))) - require.Equal(t, "18446744073709551615", string(encodeBoundaryValue(^uint64(0)))) -} - -func TestDecodeBoundaryValueRejectsMalformed(t *testing.T) { - _, err := decodeBoundaryValue(nil) - require.ErrorIs(t, err, ErrCorruption, "empty") - _, err = decodeBoundaryValue([]byte{}) - require.ErrorIs(t, err, ErrCorruption, "zero length") - _, err = decodeBoundaryValue([]byte("123abc")) - require.ErrorIs(t, err, ErrCorruption, "non-digit byte") - // 21 digits cannot fit in uint64. - _, err = decodeBoundaryValue([]byte("184467440737095516150")) - require.ErrorIs(t, err, ErrCorruption, "too long") -} - -func TestValueCodecHeightTamper(t *testing.T) { - // The whole reason we feed the height into the SHA is to detect a value that was stored under - // a different key from the one we're now reading. Decoding under the wrong height MUST fail. - hash := bytesOfLen(0x77, 32) - raw := encodeHashValue(100, hash) - _, err := decodeHashValue(101, raw) - require.ErrorIs(t, err, ErrCorruption) -} - -func TestValueCodecBitFlip(t *testing.T) { - hash := bytesOfLen(0x77, 32) - raw := encodeHashValue(100, hash) - - // Flip one bit in every byte position and confirm each flip is caught. - for i := 0; i < len(raw); i++ { - corrupted := bytes.Clone(raw) - corrupted[i] ^= 0x01 - _, err := decodeHashValue(100, corrupted) - require.ErrorIsf(t, err, ErrCorruption, "expected ErrCorruption at byte %d", i) - } -} - -func TestValueCodecShortValue(t *testing.T) { - // A value shorter than the trailer can't carry a valid checksum. - for n := 0; n < checksumSize; n++ { - _, err := decodeHashValue(0, make([]byte, n)) - require.ErrorIs(t, err, ErrCorruption) - } -} - -func TestCommitDetectsDiskCorruption(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - hash := bytesOfLen(0xAA, 32) - require.NoError(t, v.CommitToHash(ctx, 42, hash)) - - dir := v.config.DataDir - require.NoError(t, v.Close(ctx)) - - // Flip one bit of the stored value via direct Pebble access. - directWritePebble(t, dir, func(db *pebble.DB) { - key := hashKey(42) - raw, closer, err := db.Get(key) - require.NoError(t, err) - corrupted := bytes.Clone(raw) - _ = closer.Close() - corrupted[0] ^= 0x01 - require.NoError(t, db.Set(key, corrupted, pebble.Sync)) - }) - - reopenCfg := DefaultHashVaultConfig() - reopenCfg.DataDir = dir - v2, err := NewUnsafePebbleHashVault(ctx, reopenCfg) - require.NoError(t, err) - t.Cleanup(func() { _ = v2.Close(ctx) }) - - err = v2.CommitToHash(ctx, 42, hash) - require.ErrorIs(t, err, ErrCorruption) -} - -func TestStartupRejectsMalformedBoundary(t *testing.T) { - // The boundary value has no checksum (it's just GC bookkeeping), so a flipped byte is - // indistinguishable from a legitimately-written boundary and is accepted silently. The only - // failure mode startup still catches is a length mismatch, which indicates the record was - // truncated/extended outside Pebble's normal write path. - ctx := context.Background() - v := newTestPebbleVault(t) - require.NoError(t, v.Prune(ctx, 7)) - - dir := v.config.DataDir - require.NoError(t, v.Close(ctx)) - - directWritePebble(t, dir, func(db *pebble.DB) { - require.NoError(t, db.Set(pruneBoundaryKey, []byte{0x00, 0x01, 0x02}, pebble.Sync)) - }) - - reopenCfg := DefaultHashVaultConfig() - reopenCfg.DataDir = dir - _, err := NewUnsafePebbleHashVault(ctx, reopenCfg) - require.Error(t, err) - require.ErrorIs(t, err, ErrCorruption) -} - -func TestProductionConstructorForcesFsync(t *testing.T) { - ctx := context.Background() - // Caller asks for no fsync, but the production constructor must override that. - cfg := DefaultHashVaultConfig() - cfg.DataDir = filepath.Join(t.TempDir(), "vault") - cfg.Fsync = false - v, err := NewPebbleHashVault(ctx, cfg) - require.NoError(t, err) - t.Cleanup(func() { _ = v.Close(ctx) }) - - require.True(t, v.config.Fsync, "NewPebbleHashVault must force Fsync=true") - require.Equal(t, pebble.Sync, v.writeOpts, "writeOpts must be pebble.Sync in production") -} - -func TestUnsafeConstructorHonorsFsync(t *testing.T) { - ctx := context.Background() - cfg := DefaultHashVaultConfig() - cfg.DataDir = filepath.Join(t.TempDir(), "vault") - cfg.Fsync = false - v, err := NewUnsafePebbleHashVault(ctx, cfg) - require.NoError(t, err) - t.Cleanup(func() { _ = v.Close(ctx) }) - - require.False(t, v.config.Fsync) - require.Equal(t, pebble.NoSync, v.writeOpts) -} - -func TestContextCancelledCommit(t *testing.T) { - // A pre-cancelled ctx must short-circuit before we touch any state. We don't otherwise check - // the ctx mid-operation (the work is all local, fast, and uninterruptible once started). - ctx, cancel := context.WithCancel(context.Background()) - cancel() - v := newTestPebbleVault(t) - err := v.CommitToHash(ctx, 1, bytesOfLen(0xAA, 32)) - require.ErrorIs(t, err, context.Canceled) -} - -// Cross-check: nothing in the production code accidentally panics on simultaneous Close+Commit. -func TestCloseConcurrentWithCommits(t *testing.T) { - ctx := context.Background() - v := newTestPebbleVault(t) - - var wg sync.WaitGroup - for i := 0; i < 32; i++ { - wg.Add(1) - go func(h uint64) { - defer wg.Done() - _ = v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32)) - }(uint64(i + 1)) - } - // Race Close against the commits. - _ = v.Close(ctx) - wg.Wait() -} diff --git a/sei-db/state_db/sc/hashvault/prune.go b/sei-db/state_db/sc/hashvault/prune.go new file mode 100644 index 0000000000..5a88cdb917 --- /dev/null +++ b/sei-db/state_db/sc/hashvault/prune.go @@ -0,0 +1,75 @@ +package hashvault + +import ( + "fmt" + "sync/atomic" + + "github.com/sei-protocol/sei-chain/sei-db/controller" +) + +var _ controller.PrunableStore = (*HashVault)(nil) + +// PruneBelow permits the hashes of blocks below blockNumber to be deleted, as far as the vault's owner is +// concerned. A hash is deleted only once PruneHistory() has permitted it too. +func (v *HashVault) PruneBelow(blockNumber uint64) { + raiseFloor(&v.outerFloor, blockNumber) +} + +// gcFilter reports whether LittDB may delete key: only a block below both floors may go. +func (v *HashVault) gcFilter(key []byte, _ bool) (bool, error) { + blockNumber, err := decodeKey(key) + if err != nil { + return false, fmt.Errorf("decode a hash vault key: %w", err) + } + return blockNumber < min(v.outerFloor.Load(), v.gcFloor.Load()), nil +} + +// raiseFloor raises floor to blockNumber, leaving it where it is when it is already higher. +func raiseFloor(floor *atomic.Uint64, blockNumber uint64) { + for { + current := floor.Load() + if blockNumber <= current || floor.CompareAndSwap(current, blockNumber) { + return + } + } +} + +// Name implements controller.PrunableStore. +func (v *HashVault) Name() string { + return "HashVault" +} + +// PruneHistory implements controller.PrunableStore. It permits the hashes of blocks below blockNumber to be +// deleted, as far as the storage garbage collector is concerned. A hash is deleted only once PruneBelow() +// has permitted it too. +func (v *HashVault) PruneHistory(blockNumber uint64) error { + raiseFloor(&v.gcFloor, blockNumber) + return nil +} + +// PruneSnapshots implements controller.PrunableStore. The vault keeps no snapshots. +func (v *HashVault) PruneSnapshots(uint64) error { + return nil +} + +// ExternalPruning implements controller.PrunableStore. The vault has no pruner of its own. +func (v *HashVault) ExternalPruning() bool { + return true +} + +// GetRollbackFloor implements controller.PrunableStore. Every recorded block is readable directly, so the +// floor is the newest recorded block less rollbackWindow, or 0 when the window is deeper than that. +func (v *HashVault) GetRollbackFloor(rollbackWindow uint64) uint64 { + head, ok := v.Head() + if !ok || head < rollbackWindow { + return 0 + } + return head - rollbackWindow +} + +// GetLatestBlock implements controller.PrunableStore. It returns the newest recorded block, or 0 when the +// vault holds no hashes. +func (v *HashVault) GetLatestBlock() (uint64, error) { + head, _ := v.Head() + return head, nil +} diff --git a/sei-db/state_db/sc/hashvault/record.go b/sei-db/state_db/sc/hashvault/record.go new file mode 100644 index 0000000000..f39692bc4a --- /dev/null +++ b/sei-db/state_db/sc/hashvault/record.go @@ -0,0 +1,55 @@ +package hashvault + +import ( + "encoding/binary" + "fmt" +) + +// recordFormatVersion is the format version every value this package writes starts with. +const recordFormatVersion byte = 1 + +// hashSize is the length of a recorded block hash. +const hashSize = 32 + +// keySize is the length of a key: a block number, big-endian. +const keySize = 8 + +// valueSize is the length of a value: the format version, then the hash. +const valueSize = 1 + hashSize + +// encodeKey returns the key a block's hash is recorded under. +func encodeKey(blockNumber uint64) []byte { + key := make([]byte, keySize) + binary.BigEndian.PutUint64(key, blockNumber) + return key +} + +// decodeKey returns the block number a key records. +func decodeKey(key []byte) (uint64, error) { + if len(key) != keySize { + return 0, fmt.Errorf("hash vault key is %d bytes, expected %d", len(key), keySize) + } + return binary.BigEndian.Uint64(key), nil +} + +// encodeValue returns the value a hash is recorded as. +func encodeValue(hash [32]byte) []byte { + value := make([]byte, valueSize) + value[0] = recordFormatVersion + copy(value[1:], hash[:]) + return value +} + +// decodeValue returns the hash a value records. +func decodeValue(value []byte) ([32]byte, error) { + var hash [32]byte + if len(value) != valueSize { + return hash, fmt.Errorf("hash vault value is %d bytes, expected %d", len(value), valueSize) + } + if value[0] != recordFormatVersion { + return hash, fmt.Errorf("hash vault value has format version %d, expected %d", + value[0], recordFormatVersion) + } + copy(hash[:], value[1:]) + return hash, nil +} diff --git a/sei-db/state_db/sc/memiavl/hashlog_test.go b/sei-db/state_db/sc/memiavl/hashlog_test.go index 8d33c9703c..d6465f58ef 100644 --- a/sei-db/state_db/sc/memiavl/hashlog_test.go +++ b/sei-db/state_db/sc/memiavl/hashlog_test.go @@ -39,7 +39,7 @@ func (c *captureLogger) ReportChangeset(uint64, []*proto.NamedChangeSet) {} // HashListener is unused here: memIAVL reports its hashes synchronously through RecordHashes, and a // listener is for the store that publishes hashes asynchronously. -func (c *captureLogger) HashListener(context.Context, int64, *lthash.BlockHash) error { return nil } +func (c *captureLogger) HashListener(context.Context, uint64, *lthash.BlockHash) error { return nil } func (c *captureLogger) Close() error { return nil } diff --git a/sei-tendermint/config/autobahn_toml_test.go b/sei-tendermint/config/autobahn_toml_test.go index a62256d941..84f3e58107 100644 --- a/sei-tendermint/config/autobahn_toml_test.go +++ b/sei-tendermint/config/autobahn_toml_test.go @@ -18,7 +18,7 @@ import ( // TestAutobahnKeysParseFromTopLevel guards against the trap where TOML keys // authored after a [section] header get silently nested under that section. -// AutobahnConfigFile and HashVaultDisabledUnsafe are top-level fields on +// AutobahnConfigFile and the hash vault settings are top-level fields on // Config, so they must appear before any [section] header in the on-disk file. func TestAutobahnKeysParseFromTopLevel(t *testing.T) { viper.Reset() @@ -26,7 +26,8 @@ func TestAutobahnKeysParseFromTopLevel(t *testing.T) { const content = ` autobahn-config-file = "/etc/sei/autobahn.json" -hash-vault-disabled-unsafe = true +hash-vault-halt-on-mismatch = false +hash-vault-empty-rollback-blocks = 7 [rpc] laddr = "tcp://127.0.0.1:26657" @@ -40,7 +41,8 @@ laddr = "tcp://127.0.0.1:26657" cfg, err := commands.ParseConfig(tmconfig.DefaultConfig()) require.NoError(t, err) require.Equal(t, "/etc/sei/autobahn.json", cfg.AutobahnConfigFile) - require.True(t, cfg.HashVaultDisabledUnsafe) + require.False(t, cfg.HashVaultHaltOnMismatch) + require.Equal(t, uint64(7), cfg.HashVaultEmptyRollbackBlocks) } // TestAutobahnKeysIgnoredUnderSectionHeader documents what breaks if the @@ -54,7 +56,8 @@ func TestAutobahnKeysIgnoredUnderSectionHeader(t *testing.T) { const content = ` [self-remediation] autobahn-config-file = "/etc/sei/autobahn.json" -hash-vault-disabled-unsafe = true +hash-vault-halt-on-mismatch = false +hash-vault-empty-rollback-blocks = 7 ` configPath := filepath.Join(t.TempDir(), "config.toml") require.NoError(t, os.WriteFile(configPath, []byte(content), 0600)) @@ -67,7 +70,8 @@ hash-vault-disabled-unsafe = true // The field ends up empty — viper saw self-remediation.autobahn-config-file // instead of the top-level key mapstructure was looking for. require.Empty(t, cfg.AutobahnConfigFile) - require.False(t, cfg.HashVaultDisabledUnsafe) + require.True(t, cfg.HashVaultHaltOnMismatch) + require.Equal(t, uint64(1000), cfg.HashVaultEmptyRollbackBlocks) } // TestRenderedTemplateAutobahnKeysAtTopLevel verifies that the freshly @@ -83,7 +87,9 @@ func TestRenderedTemplateAutobahnKeysAtTopLevel(t *testing.T) { require.NoError(t, err) rendered := string(data) - for _, key := range []string{"autobahn-config-file", "hash-vault-disabled-unsafe"} { + for _, key := range []string{ + "autobahn-config-file", "hash-vault-halt-on-mismatch", "hash-vault-empty-rollback-blocks", + } { keyIdx := strings.Index(rendered, key) require.NotEqual(t, -1, keyIdx, "key %q must appear in rendered template", key) // Find the nearest [section] header above keyIdx, if any. diff --git a/sei-tendermint/config/config.go b/sei-tendermint/config/config.go index 9b1557fc82..d589f77899 100644 --- a/sei-tendermint/config/config.go +++ b/sei-tendermint/config/config.go @@ -12,6 +12,7 @@ import ( "time" "github.com/sei-protocol/sei-chain/ratelimiter" + seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config" mempoolcfg "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" tmos "github.com/sei-protocol/sei-chain/sei-tendermint/libs/os" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" @@ -87,26 +88,31 @@ type Config struct { // if mode disagrees with address-book membership. AutobahnConfigFile string `mapstructure:"autobahn-config-file"` - // HashVaultDisabledUnsafe disables the app-hash equivocation guard (HashVault). The vault is - // on by default (false). Setting this to true is an explicit, last-resort operator decision to - // run WITHOUT equivocation protection; the node logs loudly that it is unsafe. - HashVaultDisabledUnsafe bool `mapstructure:"hash-vault-disabled-unsafe"` + // HashVaultHaltOnMismatch selects what an Autobahn node does when the hash vault sees a state hash + // differ from the one it recorded for the same block: halt when true, or log an error and replace the + // recorded hashes when false. + HashVaultHaltOnMismatch bool `mapstructure:"hash-vault-halt-on-mismatch"` + + // HashVaultEmptyRollbackBlocks is how many blocks an Autobahn node rewinds and replays when it starts + // over an empty hash vault, to refill it. + HashVaultEmptyRollbackBlocks uint64 `mapstructure:"hash-vault-empty-rollback-blocks"` } // DefaultConfig returns a default configuration for a Tendermint node func DefaultConfig() *Config { return &Config{ - BaseConfig: DefaultBaseConfig(), - RPC: DefaultRPCConfig(), - P2P: DefaultP2PConfig(), - Mempool: DefaultMempoolConfig(), - StateSync: DefaultStateSyncConfig(), - Consensus: DefaultConsensusConfig(), - TxIndex: DefaultTxIndexConfig(), - Instrumentation: DefaultInstrumentationConfig(), - PrivValidator: DefaultPrivValidatorConfig(), - SelfRemediation: DefaultSelfRemediationConfig(), - HashVaultDisabledUnsafe: false, + BaseConfig: DefaultBaseConfig(), + RPC: DefaultRPCConfig(), + P2P: DefaultP2PConfig(), + Mempool: DefaultMempoolConfig(), + StateSync: DefaultStateSyncConfig(), + Consensus: DefaultConsensusConfig(), + TxIndex: DefaultTxIndexConfig(), + Instrumentation: DefaultInstrumentationConfig(), + PrivValidator: DefaultPrivValidatorConfig(), + SelfRemediation: DefaultSelfRemediationConfig(), + HashVaultHaltOnMismatch: seidbconfig.DefaultHashVaultConfig().HaltOnMismatch, + HashVaultEmptyRollbackBlocks: seidbconfig.DefaultHashVaultConfig().EmptyVaultRollbackBlocks, } } diff --git a/sei-tendermint/config/config_fuzz_test.go b/sei-tendermint/config/config_fuzz_test.go index aedd63e499..3f111e07b8 100644 --- a/sei-tendermint/config/config_fuzz_test.go +++ b/sei-tendermint/config/config_fuzz_test.go @@ -118,10 +118,10 @@ func TestValidateBasicDistinguishesAnAbsentModeFromAnUnknownOne(t *testing.T) { } } -// FuzzRootScopeKeysRequireRootScope pins the placement trap on the two root-scope +// FuzzRootScopeKeysRequireRootScope pins the placement trap on the root-scope // keys. // -// autobahn-config-file and hash-vault-disabled-unsafe are declared at the top level +// autobahn-config-file and hash-vault-halt-on-mismatch are declared at the top level // of the Config struct, so in TOML they must appear before any [section] header. // Written after one they become that section's key — p2p.autobahn-config-file — // which nothing reads, and the node starts with the subsystem the operator meant to @@ -152,7 +152,7 @@ func FuzzRootScopeKeysRequireRootScope(f *testing.F) { doc.WriteString("[p2p]\n") } doc.WriteString("autobahn-config-file = \"" + path + "\"\n") - doc.WriteString("hash-vault-disabled-unsafe = true\n") + doc.WriteString("hash-vault-halt-on-mismatch = false\n") } conf, err := unmarshalConfigTOML(t, doc.String()) @@ -161,19 +161,19 @@ func FuzzRootScopeKeysRequireRootScope(f *testing.F) { } wantPath := "" - wantDisabled := false + wantHalt := true if present && !underSection { wantPath = path - wantDisabled = true + wantHalt = false } if conf.AutobahnConfigFile != wantPath { t.Fatalf("autobahn-config-file resolved to %q, want %q (present=%v underSection=%v); "+ "root-scope keys are only read before the first section header", conf.AutobahnConfigFile, wantPath, present, underSection) } - if conf.HashVaultDisabledUnsafe != wantDisabled { - t.Fatalf("hash-vault-disabled-unsafe resolved to %v, want %v (present=%v underSection=%v)", - conf.HashVaultDisabledUnsafe, wantDisabled, present, underSection) + if conf.HashVaultHaltOnMismatch != wantHalt { + t.Fatalf("hash-vault-halt-on-mismatch resolved to %v, want %v (present=%v underSection=%v)", + conf.HashVaultHaltOnMismatch, wantHalt, present, underSection) } }) } @@ -295,8 +295,8 @@ func TestAutobahnPointerAbsenceDisablesTheSubsystem(t *testing.T) { if conf.AutobahnConfigFile != "" { t.Fatalf("the default autobahn pointer must be empty, got %q", conf.AutobahnConfigFile) } - if conf.HashVaultDisabledUnsafe { - t.Fatal("the default must leave the app-hash equivocation guard enabled") + if !conf.HashVaultHaltOnMismatch { + t.Fatal("the default must leave a hash vault mismatch halting the node") } } diff --git a/sei-tendermint/config/toml.go b/sei-tendermint/config/toml.go index 1d054cd4ad..6285c68e0a 100644 --- a/sei-tendermint/config/toml.go +++ b/sei-tendermint/config/toml.go @@ -163,19 +163,20 @@ mock-app = {{ .BaseConfig.MockApp }} # would otherwise nest it under the immediately preceding section. autobahn-config-file = "{{ .AutobahnConfigFile }}" -# hash-vault-disabled-unsafe disables the app-hash equivocation guard (HashVault). -# DO NOT set this to true unless you are knowingly running an UNSAFE node as a last-resort -# recovery measure. A node with this enabled has NO protection against changing its mind about -# a committed block's app hash, and will log error-level warnings on every startup. +# hash-vault-halt-on-mismatch selects what an Autobahn node does when the hash vault sees a +# block's state hash differ from the one it recorded for that block. When true, the node halts; +# DO NOT RESTART WITHOUT HUMAN INVESTIGATION. When false, the node logs an error, discards the +# recorded hashes from that block up, and records the new hash in their place. # -# It is safer to leave HashVault enabled: if you hit a startup panic, first remove the HashVault -# files as instructed in the panic message and let the node run. Only disable HashVault if you are -# very sure the stored hashes are totally wrong and you keep hitting the same panic on new blocks. +# hash-vault-empty-rollback-blocks is how many blocks an Autobahn node rewinds and replays when it +# starts over an empty hash vault, so that the vault holds the hashes of recent blocks again. The +# rewind is shortened to what the node's snapshots and state WAL can reach. # -# Placed here (as a top-level key, before any [section] header) so the TOML parser sees it at -# root scope where mapstructure expects it — viper would otherwise nest it under the +# Placed here (as top-level keys, before any [section] header) so the TOML parser sees them at +# root scope where mapstructure expects them — viper would otherwise nest them under the # immediately preceding section. -hash-vault-disabled-unsafe = {{ .HashVaultDisabledUnsafe }} +hash-vault-halt-on-mismatch = {{ .HashVaultHaltOnMismatch }} +hash-vault-empty-rollback-blocks = {{ .HashVaultEmptyRollbackBlocks }} ####################################################################### ### Advanced Configuration Options ### diff --git a/sei-tendermint/internal/p2p/giga_router.go b/sei-tendermint/internal/p2p/giga_router.go index e88df6db67..2dea8a0938 100644 --- a/sei-tendermint/internal/p2p/giga_router.go +++ b/sei-tendermint/internal/p2p/giga_router.go @@ -34,7 +34,7 @@ type GigaRouterCommonConfig struct { ValidatorAddrs map[atypes.PublicKey]GigaNodeAddr GenDoc *types.GenesisDoc // PersistentStateDir is the absolute on-disk root for durable state - // (BlockDB, hashvault, epoch snapshots, and the validator's consensus + // (BlockDB, epoch snapshots, and the validator's consensus // persister in sibling subdirs). Required and must already exist. PersistentStateDir string // App is the ABCI proxy executeBlock drives. NewGigaValidatorRouter @@ -45,11 +45,6 @@ type GigaRouterCommonConfig struct { // peers. 0 rejects all; positive caps at n, up to maxInboundFullnodePeers. MaxInboundFullnodePeers int - // HashVaultDisabledUnsafe disables the app-hash equivocation guard (HashVault). The guard is - // on by default (false); the GigaRouter builds and owns it (see runExecute). Setting this to true - // is an explicit, last-resort operator decision to run WITHOUT equivocation protection. - HashVaultDisabledUnsafe bool - // Whether validator should proxy txs which do not belong to the local node. EnableEvmProxy bool } diff --git a/sei-tendermint/internal/p2p/giga_router_common.go b/sei-tendermint/internal/p2p/giga_router_common.go index db0330c53c..a9f38779c0 100644 --- a/sei-tendermint/internal/p2p/giga_router_common.go +++ b/sei-tendermint/internal/p2p/giga_router_common.go @@ -5,13 +5,11 @@ import ( "errors" "fmt" "net/url" - "path/filepath" "slices" "sort" "sync/atomic" ethrpc "github.com/ethereum/go-ethereum/rpc" - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashvault" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/crypto" @@ -216,7 +214,7 @@ func (r *gigaRouterCommon) translateGlobalBlock(gb *atypes.GlobalBlock) *coretyp } } -func (r *gigaRouterCommon) executeBlock(ctx context.Context, b *atypes.GlobalBlock, hashVault hashvault.HashVault) (*abci.ResponseCommit, error) { +func (r *gigaRouterCommon) executeBlock(ctx context.Context, b *atypes.GlobalBlock) (*abci.ResponseCommit, error) { app := r.app hash := b.Header.Hash() var proposerAddress types.Address @@ -253,15 +251,6 @@ func (r *gigaRouterCommon) executeBlock(ctx context.Context, b *atypes.GlobalBlo return nil, fmt.Errorf("app.FinalizeBlock(): %w", err) } - // Commit this height's app hash to the equivocation guard before persisting app state, so the - // vault always records our commitment to a height before the state it implies is committed (and - // before the hash is proposed for AppQC voting via PushAppHash below). On restart the block is - // re-executed and the identical hash is re-committed idempotently. A returned error is a benign - // shutdown cancellation; genuine faults panic inside the call. See commitAppHashToVault. - if err := commitAppHashToVault(ctx, hashVault, b.GlobalNumber, resp.AppHash); err != nil { - return nil, err - } - commitResp, err := app.Commit(ctx) if err != nil { return nil, fmt.Errorf("app.Commit(): %w", err) @@ -313,75 +302,7 @@ func finalizeBlockGasUsed(resp *abci.ResponseFinalizeBlock) int64 { return total } -// buildHashVault constructs the app-hash equivocation guard runExecute owns. By default it -// returns a durable Pebble-backed vault rooted at /hashvault, alongside the -// other Autobahn on-disk state. It returns a no-op vault (no protection) when the operator -// explicitly sets HashVaultDisabledUnsafe, logged loudly. -func buildHashVault(ctx context.Context, cfg *GigaRouterCommonConfig) (hashvault.HashVault, error) { - if cfg.HashVaultDisabledUnsafe { - logger.Error("################################################################") - logger.Error("# HASHVAULT DISABLED (hash-vault-disabled-unsafe=true). #") - logger.Error("# This node has NO app-hash equivocation protection and is #") - logger.Error("# running in an UNSAFE configuration. Re-enable as soon as the #") - logger.Error("# underlying issue is resolved. #") - logger.Error("################################################################") - return hashvault.NewNoopHashVault(), nil - } - hvCfg := hashvault.DefaultHashVaultConfig() - hvCfg.DataDir = filepath.Join(cfg.PersistentStateDir, "hashvault") - return hashvault.NewPebbleHashVault(ctx, hvCfg) -} - -// commitAppHashToVault records the app hash for the given height in the equivocation guard and halts -// the node on any error. Every executed height is guarded, so a node can never commit to two -// different app hashes for the same height without deliberate human intervention. -func commitAppHashToVault( - ctx context.Context, - vault hashvault.HashVault, - height atypes.GlobalBlockNumber, - hash []byte, -) error { - err := vault.CommitToHash(ctx, uint64(height), hash) - if err == nil { - return nil - } - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - logger.Info("HashVault commit aborted by context cancellation during shutdown; not recording hash", - "height", height, "err", err) - return fmt.Errorf("hashvault CommitToHash aborted at height %d: %w", height, err) - } - // Build the fatal message once and use it for both the log and the panic. The logger writes - // directly (no in-process buffer), but a hard crash could still drop the final line, so the - // panic string carries the full guidance too — panic output is what an operator sees first. - var msg string - if errors.Is(err, hashvault.ErrHashMismatch) { - // The HashVault has already logged the conflicting hashes, its data directory, and the - // bypass/slashing guidance immediately before returning this error; don't duplicate it. - msg = fmt.Sprintf("FATAL: HashVault detected an app-hash equivocation at height %d; halting. "+ - "See the preceding HashVault error for the conflicting hashes and recovery steps. "+ - "DO NOT RESTART WITHOUT HUMAN INTERVENTION.", height) - } else { - msg = fmt.Sprintf("FATAL: HashVault could not commit the app hash at height %d (operational "+ - "error, not a confirmed equivocation): %v. hashHex=%x. Halting.", height, err, hash) - } - logger.Error(msg) - panic(msg) -} - func (r *gigaRouterCommon) runExecute(ctx context.Context) error { - // runExecute is the single block-execution loop spawned by both the validator and fullnode Run - // methods, so it owns the equivocation guard for both roles: build it here (set before the first - // executeBlock, the only other reader) and close it on exit. - hashVault, err := buildHashVault(ctx, r.cfg) - if err != nil { - return fmt.Errorf("buildHashVault(): %w", err) - } - defer func() { - if err := hashVault.Close(context.Background()); err != nil { - logger.Error("failed to close hashvault", "err", err) - } - }() - app := r.app info := app.Info() @@ -427,15 +348,6 @@ func (r *gigaRouterCommon) runExecute(ctx context.Context) error { // TODO: for consistency we should also set proposerAddress here, // but this is a placeholder solution so maybe we don't care. }).ToProto()) - // Re-commit the last finalized block's app hash to the equivocation guard before re-proposing it - // for AppQC voting (PushAppHash below), mirroring executeBlock's commit-before-PushAppHash - // ordering. On a normal restart this idempotently matches the hash recorded when `last` was - // first executed; if the committed app state has diverged from what the vault recorded (e.g. an - // out-of-band rollback/restore), this halts the node instead of externalizing a conflicting - // hash. A returned error is a benign shutdown cancellation; genuine faults panic inside. - if err := commitAppHashToVault(ctx, hashVault, last, info.LastBlockAppHash); err != nil { - return err - } // Losing a prefix of appHashes on crash is fine: AppQC is reached // once everyone votes on apphashes of a suffix of finalized blocks. weights, err := committeeWeights(app.GetValidators()) @@ -452,7 +364,7 @@ func (r *gigaRouterCommon) runExecute(ctx context.Context) error { if err != nil { return fmt.Errorf("r.data.GlobalBlock(%v): %w", n, err) } - commitResp, err := r.executeBlock(ctx, b, hashVault) + commitResp, err := r.executeBlock(ctx, b) if err != nil { return fmt.Errorf("r.executeBlock(%v): %w", n, err) } @@ -463,17 +375,6 @@ func (r *gigaRouterCommon) runExecute(ctx context.Context) error { if err := r.data.PruneBefore(pruneBefore); err != nil { return fmt.Errorf("r.data.PruneBefore(%v): %w", pruneBefore, err) } - // Align the vault's retention with the data layer's prune boundary. - if err := hashVault.Prune(ctx, uint64(pruneBefore)); err != nil { - // A canceled context just means we're shutting down between a successful executeBlock - // and this prune; that's benign, not a prune failure, so don't alarm operators. - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - logger.Info("hashvault prune aborted by context cancellation during shutdown", - "prune_before", pruneBefore, "err", err) - } else { - logger.Error("failed to prune hashvault", "prune_before", pruneBefore, "err", err) - } - } } } diff --git a/sei-tendermint/internal/p2p/giga_router_common_test.go b/sei-tendermint/internal/p2p/giga_router_common_test.go index afd48abeab..aadc4926dd 100644 --- a/sei-tendermint/internal/p2p/giga_router_common_test.go +++ b/sei-tendermint/internal/p2p/giga_router_common_test.go @@ -11,7 +11,6 @@ import ( ethrpc "github.com/ethereum/go-ethereum/rpc" "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/memblock" - "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashvault" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/blockstore" atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" @@ -52,54 +51,6 @@ func (a *fixedHeightApp) Info() *abci.ResponseInfo { return &abci.ResponseInfo{LastBlockHeight: a.height} } -// newSeededVault returns a durable Pebble vault rooted in a temp dir with hash committed at height. -func newSeededVault(t *testing.T, height atypes.GlobalBlockNumber, hash []byte) hashvault.HashVault { - t.Helper() - cfg := hashvault.DefaultHashVaultConfig() - cfg.DataDir = t.TempDir() - v, err := hashvault.NewUnsafePebbleHashVault(context.Background(), cfg) - require.NoError(t, err) - t.Cleanup(func() { _ = v.Close(context.Background()) }) - require.NoError(t, v.CommitToHash(context.Background(), uint64(height), hash)) - return v -} - -// TestCommitHashToVault covers the safety contract the restart path in runExecute relies on: -// an idempotent match returns nil, a divergent hash halts the node (panic), and a canceled -// context returns an error without halting. -func TestCommitHashToVault(t *testing.T) { - const height atypes.GlobalBlockNumber = 42 - h1 := make([]byte, hashvault.BlockHashSize) - for i := range h1 { - h1[i] = 0xAA - } - h2 := make([]byte, hashvault.BlockHashSize) - for i := range h2 { - h2[i] = 0xBB - } - - t.Run("matching hash is idempotent", func(t *testing.T) { - vault := newSeededVault(t, height, h1) - require.NoError(t, commitAppHashToVault(context.Background(), vault, height, h1)) - }) - - t.Run("divergent hash halts the node", func(t *testing.T) { - vault := newSeededVault(t, height, h1) - require.Panics(t, func() { - _ = commitAppHashToVault(context.Background(), vault, height, h2) - }) - }) - - t.Run("canceled context returns error without halting", func(t *testing.T) { - vault := newSeededVault(t, height, h1) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - // Must not panic: a canceled context is a benign shutdown, not an equivocation. - err := commitAppHashToVault(ctx, vault, height, h2) - require.Error(t, err) - }) -} - func TestFinalizeBlockGasUsed(t *testing.T) { resp := &abci.ResponseFinalizeBlock{ TxResults: []*abci.ExecTxResult{ diff --git a/sei-tendermint/node/public.go b/sei-tendermint/node/public.go index 58289a1065..31d6e6ea31 100644 --- a/sei-tendermint/node/public.go +++ b/sei-tendermint/node/public.go @@ -162,7 +162,7 @@ func prepareApplication( if err != nil { return nil, noStorage, fmt.Errorf("load Autobahn committee: %w", err) } - manager, err := openAutobahnStorageManager(ctx, conf.RootDir, fc) + manager, err := openAutobahnStorageManager(ctx, conf, fc) if err != nil { return nil, noStorage, fmt.Errorf("open Autobahn storage: %w", err) } diff --git a/sei-tendermint/node/setup.go b/sei-tendermint/node/setup.go index fdfe2116c4..9ca2d4ae5d 100644 --- a/sei-tendermint/node/setup.go +++ b/sei-tendermint/node/setup.go @@ -323,9 +323,6 @@ func buildGigaRouter( return nil, err } valCfg.PersistentStateDir = stateDir - // The GigaRouter builds and owns the equivocation guard itself; just pass the operator's - // enable/disable decision through as plain config. - valCfg.HashVaultDisabledUnsafe = cfg.HashVaultDisabledUnsafe logger.Info("Autobahn: starting as validator", "validators", len(valCfg.ValidatorAddrs)) dataState, err := p2p.BuildDataState(&valCfg.GigaRouterCommonConfig, blockStore) if err != nil { @@ -346,9 +343,6 @@ func buildGigaRouter( return nil, err } fnCfg.PersistentStateDir = stateDir - // The GigaRouter builds and owns the equivocation guard itself; just pass the operator's - // enable/disable decision through as plain config. - fnCfg.HashVaultDisabledUnsafe = cfg.HashVaultDisabledUnsafe logger.Info("Autobahn: starting as fullnode", "mode", cfg.Mode, "validators", len(validatorAddrs)) dataState, err := p2p.BuildDataState(fnCfg, blockStore) if err != nil { @@ -381,10 +375,10 @@ func resolvePersistentStateDir(rootDir, dir string) (string, error) { // persistent-state directory. func openAutobahnStorageManager( ctx context.Context, - rootDir string, + conf *config.Config, fc *config.AutobahnFileConfig, ) (*bootstrap.GigaStorageManager, error) { - directory, err := resolvePersistentStateDir(rootDir, fc.PersistentStateDir) + directory, err := resolvePersistentStateDir(conf.RootDir, fc.PersistentStateDir) if err != nil { return nil, err } @@ -397,6 +391,10 @@ func openAutobahnStorageManager( return nil, fmt.Errorf("build Autobahn block DB config: %w", err) } storageConfig.BlockDBConfig = &blockConfig + storageConfig.HashVaultConfig.HaltOnMismatch = conf.HashVaultHaltOnMismatch + storageConfig.HashVaultConfig.EmptyVaultRollbackBlocks = conf.HashVaultEmptyRollbackBlocks + // The Pebble-backed vault the giga router kept here before the vault moved into the state DB. + storageConfig.HashVaultConfig.LegacyPebbleDir = filepath.Join(directory, "hashvault") return bootstrap.NewGigaStorageManager(ctx, storageConfig) } From 0120d78897fda5d618777be38dd94ac8ad1bfe94 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 25 Sep 2026 08:27:57 -0500 Subject: [PATCH 2/5] revert hashvault changes --- sei-db/bench/cryptosim/cryptosim.go | 4 +- sei-db/config/giga_config.go | 5 +- sei-db/config/hashvault_config.go | 44 -- sei-db/state_db/giga/state_db.go | 13 +- sei-db/state_db/giga/state_db_hash_vault.go | 9 +- .../state_db/giga/state_db_hash_vault_test.go | 13 +- sei-db/state_db/giga/state_db_recovery.go | 4 +- sei-db/state_db/sc/hashvault/hashvault.go | 414 ++------------- .../state_db/sc/hashvault/hashvault_config.go | 56 ++ .../state_db/sc/hashvault/hashvault_test.go | 432 +++++++-------- .../state_db/sc/hashvault/noop_hashvault.go | 30 ++ sei-db/state_db/sc/hashvault/offline.go | 119 ----- sei-db/state_db/sc/hashvault/offline_test.go | 77 --- .../state_db/sc/hashvault/pebble_hashvault.go | 493 ++++++++++++++++++ .../hashvault/pebble_hashvault_branch_test.go | 194 +++++++ .../sc/hashvault/pebble_hashvault_codec.go | 160 ++++++ .../sc/hashvault/pebble_hashvault_rollback.go | 159 ++++++ .../pebble_hashvault_rollback_test.go | 144 +++++ .../sc/hashvault/pebble_hashvault_test.go | 338 ++++++++++++ sei-db/state_db/sc/hashvault/prune.go | 75 --- sei-db/state_db/sc/hashvault/record.go | 55 -- sei-tendermint/config/config.go | 6 +- 22 files changed, 1845 insertions(+), 999 deletions(-) delete mode 100644 sei-db/config/hashvault_config.go create mode 100644 sei-db/state_db/sc/hashvault/hashvault_config.go create mode 100644 sei-db/state_db/sc/hashvault/noop_hashvault.go delete mode 100644 sei-db/state_db/sc/hashvault/offline.go delete mode 100644 sei-db/state_db/sc/hashvault/offline_test.go create mode 100644 sei-db/state_db/sc/hashvault/pebble_hashvault.go create mode 100644 sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go create mode 100644 sei-db/state_db/sc/hashvault/pebble_hashvault_codec.go create mode 100644 sei-db/state_db/sc/hashvault/pebble_hashvault_rollback.go create mode 100644 sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go create mode 100644 sei-db/state_db/sc/hashvault/pebble_hashvault_test.go delete mode 100644 sei-db/state_db/sc/hashvault/prune.go delete mode 100644 sei-db/state_db/sc/hashvault/record.go diff --git a/sei-db/bench/cryptosim/cryptosim.go b/sei-db/bench/cryptosim/cryptosim.go index 6060527f3a..5cc95d28c4 100644 --- a/sei-db/bench/cryptosim/cryptosim.go +++ b/sei-db/bench/cryptosim/cryptosim.go @@ -12,9 +12,9 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/common/keys" crand "github.com/sei-protocol/sei-chain/sei-db/common/rand" "github.com/sei-protocol/sei-chain/sei-db/common/utils" - dbconfig "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/controller" "github.com/sei-protocol/sei-chain/sei-db/state_db/giga" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashvault" ) const ( @@ -147,7 +147,7 @@ func NewCryptoSim( // giga.NewStateDB is the node's own entry point, and the only one that leaves the state WAL // outside the live state DB: it opens the WAL itself and writes each block to it ahead of the // commit. A live state DB opened directly would own its WAL and write it inline instead. - hashVaultConfig := dbconfig.DefaultHashVaultConfig() + hashVaultConfig := hashvault.DefaultHashVaultConfig() hashVaultConfig.DataDir = filepath.Join(config.DataDir, "hashvault") db, err := giga.NewStateDB( ctx, config.FlatKVConfig, config.StateStoreConfig, config.CheckpointConfig, hashVaultConfig, 0) diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index 20b0ca9306..716696a9f9 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -6,6 +6,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" flatkvConfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashvault" ) // GigaStorageConfig composes the store configs a Giga node opens. It is not read from @@ -18,7 +19,7 @@ type GigaStorageConfig struct { BlockDBConfig *littblock.BlockDBConfig // required PruningConfig *StorageGarbageCollectorConfig // required CheckpointConfig CheckpointConfig - HashVaultConfig HashVaultConfig + HashVaultConfig hashvault.HashVaultConfig } // gigaReceiptBackend is the receipt backend Giga opens (littidx). @@ -48,7 +49,7 @@ func DefaultGigaStorageConfig(homePath string) (*GigaStorageConfig, error) { ssConfig.ExternalPruning = true ssConfig.DisableInternalWAL = true - hashVaultConfig := DefaultHashVaultConfig() + hashVaultConfig := hashvault.DefaultHashVaultConfig() hashVaultConfig.DataDir = utils.GetHashVaultPath(homePath) receiptConfig := DefaultReceiptStoreConfig() diff --git a/sei-db/config/hashvault_config.go b/sei-db/config/hashvault_config.go deleted file mode 100644 index c30018b9cb..0000000000 --- a/sei-db/config/hashvault_config.go +++ /dev/null @@ -1,44 +0,0 @@ -package config - -import "fmt" - -// HashVaultConfig configures the hash vault, the equivocation guard over the live state DB's block hashes. -type HashVaultConfig struct { - // DataDir is the directory the vault keeps its hashes in. - DataDir string - - // HaltOnMismatch selects what a hash that differs from the recorded one does. When true, the state DB - // fails and the node halts. When false, the mismatch is logged as an error, the vault discards the - // recorded hashes from that block up, and the new hash is recorded in their place. - HaltOnMismatch bool - - // EmptyVaultRollbackBlocks is how many blocks the state DB rewinds and replays when it opens over an - // empty vault, so that the vault holds the hashes of recent blocks and not just the loaded one. The - // rewind is limited to what the state commit store's snapshots and the state WAL can reach. 0 records - // only the loaded block's hash. - EmptyVaultRollbackBlocks uint64 - - // LegacyPebbleDir is the directory of the Pebble-backed vault this one replaces, deleted when the vault - // opens. Empty when there is none to delete. - LegacyPebbleDir string - - // Fsync controls whether each recorded hash is fsynced before the vault reports it recorded. - Fsync bool -} - -// DefaultHashVaultConfig returns the default hash vault config. DataDir is left for the caller to set. -func DefaultHashVaultConfig() HashVaultConfig { - return HashVaultConfig{ - HaltOnMismatch: true, - EmptyVaultRollbackBlocks: 1000, - Fsync: true, - } -} - -// Validate returns an error if the config cannot open a vault. -func (c *HashVaultConfig) Validate() error { - if c.DataDir == "" { - return fmt.Errorf("hash vault data dir is required") - } - return nil -} diff --git a/sei-db/state_db/giga/state_db.go b/sei-db/state_db/giga/state_db.go index 6ecabd4822..767b17ef65 100644 --- a/sei-db/state_db/giga/state_db.go +++ b/sei-db/state_db/giga/state_db.go @@ -42,7 +42,7 @@ type StateDB struct { ss *evm.EVMStateStore // The hash vault SC's block hashes are recorded in. - vault *hashvault.HashVault + vault *hashvault.PebbleHashVault // The checkpoint schedule SC and SS take their snapshot boundaries from. checkpointer *controller.CheckpointScheduler @@ -66,7 +66,7 @@ func NewStateDB( flatkvCfg *flatkvconfig.Config, ssCfg config.StateStoreConfig, checkpointCfg config.CheckpointConfig, - hashVaultCfg config.HashVaultConfig, + hashVaultCfg hashvault.HashVaultConfig, // The height to roll back to, or 0 to load the latest block possible. Data after rollbackTo target // may be permanently deleted. Returns an error if not possible to roll back to requested block height. rollbackTo uint64, @@ -80,7 +80,7 @@ func NewStateDB( var err error var ss *evm.EVMStateStore var sc *flatkv.CommitStore - var vault *hashvault.HashVault + var vault *hashvault.PebbleHashVault var wal statewal.StateWAL defer func() { if retErr == nil { @@ -99,7 +99,7 @@ func NewStateDB( return nil, fmt.Errorf("open the state DB: %w", err) } - if vault, err = hashvault.Open(hashVaultCfg); err != nil { + if vault, err = hashvault.NewPebbleHashVault(ctx, hashVaultCfg); err != nil { return nil, fmt.Errorf("open the state DB: %w", err) } // The vault must be SC's first listener, and registering it before SC is reachable from outside this @@ -208,7 +208,7 @@ func (s *StateDB) Close() error { func closeStores( ss *evm.EVMStateStore, sc *flatkv.CommitStore, - vault *hashvault.HashVault, + vault *hashvault.PebbleHashVault, wal statewal.StateWAL, ) error { var errs error @@ -224,7 +224,8 @@ func closeStores( } } if vault != nil { - if err := timer.Close("hashvault", vault.Close); err != nil { + closeVault := func() error { return vault.Close(context.Background()) } + if err := timer.Close("hashvault", closeVault); err != nil { errs = errors.Join(errs, fmt.Errorf("close hash vault: %w", err)) } } diff --git a/sei-db/state_db/giga/state_db_hash_vault.go b/sei-db/state_db/giga/state_db_hash_vault.go index c9aca724e8..2af783ecd0 100644 --- a/sei-db/state_db/giga/state_db_hash_vault.go +++ b/sei-db/state_db/giga/state_db_hash_vault.go @@ -11,12 +11,13 @@ import ( ) // hashVaultListener returns the listener that records each block's hash in vault. -func hashVaultListener(vault *hashvault.HashVault) gigatypes.HashListener { - return func(_ context.Context, blockNumber uint64, hash *lthash.BlockHash) error { +func hashVaultListener(vault *hashvault.PebbleHashVault) gigatypes.HashListener { + return func(ctx context.Context, blockNumber uint64, hash *lthash.BlockHash) error { if hash.Global == nil { return fmt.Errorf("record the hash of block %d: it carries no global hash", blockNumber) } - if err := vault.Commit(blockNumber, hash.Global.Checksum()); err != nil { + checksum := hash.Global.Checksum() + if err := vault.CommitToHash(ctx, blockNumber, checksum[:]); err != nil { return fmt.Errorf("record the hash of block %d in the hash vault: %w", blockNumber, err) } return nil @@ -26,7 +27,7 @@ func hashVaultListener(vault *hashvault.HashVault) gigatypes.HashListener { // recordLoadedBlockHash records, or checks, the hash of the block SC opened on, so that the vault holds // it once the open returns. SC dispatches that hash while it loads, before the vault is registered, and // no replay re-dispatches it when SC was already on the WAL's head. -func recordLoadedBlockHash(sc *flatkv.CommitStore, vault *hashvault.HashVault) error { +func recordLoadedBlockHash(sc *flatkv.CommitStore, vault *hashvault.PebbleHashVault) error { if err := sc.FlushHashes(); err != nil { return fmt.Errorf("wait for the replayed blocks to reach the hash vault: %w", err) } diff --git a/sei-db/state_db/giga/state_db_hash_vault_test.go b/sei-db/state_db/giga/state_db_hash_vault_test.go index 1f94d8d9c5..b54f15288d 100644 --- a/sei-db/state_db/giga/state_db_hash_vault_test.go +++ b/sei-db/state_db/giga/state_db_hash_vault_test.go @@ -31,7 +31,7 @@ type vaultTestStores struct { checkpointCfg config.CheckpointConfig // Where the hash vault lives, and what it does on a mismatch. - hashVaultCfg config.HashVaultConfig + hashVaultCfg hashvault.HashVaultConfig } // newVaultTestStores returns configs for a fresh StateDB whose vault halts on a mismatch. @@ -40,7 +40,7 @@ func newVaultTestStores(t *testing.T) *vaultTestStores { flatkvCfg := flatkvconfig.DefaultTestConfig(t) // The snapshots the rewinds land on are kept, since no collector runs here to prune them. flatkvCfg.ExternalPruning = true - hashVaultCfg := config.DefaultHashVaultConfig() + hashVaultCfg := hashvault.DefaultHashVaultConfig() hashVaultCfg.DataDir = filepath.Join(t.TempDir(), "hashvault") hashVaultCfg.Fsync = false return &vaultTestStores{ @@ -148,7 +148,7 @@ func TestAVaultBehindSCIsRefilledByReplay(t *testing.T) { before := recordedHashes(t, db, 1, 6) require.NoError(t, db.Close()) - require.NoError(t, hashvault.PruneAfter(stores.hashVaultCfg, 3)) + require.NoError(t, hashvault.HardRollbackPebbleHashVault(context.Background(), stores.hashVaultCfg, 3)) reopened := stores.open(t) defer func() { require.NoError(t, reopened.Close()) }() @@ -218,10 +218,11 @@ func tamperLoadedBlock(t *testing.T, stores *vaultTestStores, blockNumber uint64 t.Helper() cfg := stores.hashVaultCfg cfg.HaltOnMismatch = false - vault, err := hashvault.Open(cfg) + vault, err := hashvault.NewUnsafePebbleHashVault(context.Background(), cfg) require.NoError(t, err) - require.NoError(t, vault.Commit(blockNumber, [32]byte{0xEE})) - require.NoError(t, vault.Close()) + tampered := [32]byte{0xEE} + require.NoError(t, vault.CommitToHash(context.Background(), blockNumber, tampered[:])) + require.NoError(t, vault.Close(context.Background())) } // The loaded block's hash is checked against the vault even when nothing replays, so a vault that diff --git a/sei-db/state_db/giga/state_db_recovery.go b/sei-db/state_db/giga/state_db_recovery.go index 43558e7cc2..3a11e29e8c 100644 --- a/sei-db/state_db/giga/state_db_recovery.go +++ b/sei-db/state_db/giga/state_db_recovery.go @@ -57,7 +57,7 @@ type recoveryPlan struct { func recoverStores( flatkvCfg *flatkvconfig.Config, ssCfg config.StateStoreConfig, - hashVaultCfg config.HashVaultConfig, + hashVaultCfg hashvault.HashVaultConfig, rollbackTo uint64, ) error { survey, err := surveyStores(flatkvCfg, hashVaultCfg) @@ -77,7 +77,7 @@ func recoverStores( // Reads what the state WAL, the hash vault and the live state DB's snapshots hold. Every store must be // closed. -func surveyStores(flatkvCfg *flatkvconfig.Config, hashVaultCfg config.HashVaultConfig) (recoverySurvey, error) { +func surveyStores(flatkvCfg *flatkvconfig.Config, hashVaultCfg hashvault.HashVaultConfig) (recoverySurvey, error) { // This takes the WAL directory's exclusive lock, so it only works before the WAL opens. walStored, walFirst, walLast, err := statewal.GetRange(flatkv.StateWALConfig(flatkvCfg.DataDir)) if err != nil { diff --git a/sei-db/state_db/sc/hashvault/hashvault.go b/sei-db/state_db/sc/hashvault/hashvault.go index 5e3a807060..3a40600039 100644 --- a/sei-db/state_db/sc/hashvault/hashvault.go +++ b/sei-db/state_db/sc/hashvault/hashvault.go @@ -1,366 +1,58 @@ -// Package hashvault records the live state DB's block hashes and refuses to let a recorded hash change, -// so that a node cannot commit to two different states for the same block without human intervention. package hashvault import ( - "encoding/hex" - "fmt" - "os" - "sync" - "sync/atomic" - "time" - - "github.com/sei-protocol/seilog" - - "github.com/sei-protocol/sei-chain/sei-db/config" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/disktable/keymap" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/littbuilder" - gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" + "context" + "errors" ) -var logger = seilog.NewLogger("db", "state-db", "sc", "hashvault") - -// tableName is the LittDB table the hashes are recorded in. -const tableName = "hashes" - -// HashVault records one hash per block for a contiguous range of blocks, and holds each hash fixed once -// it is recorded. -// -// Every method is safe to call from any goroutine. -type HashVault struct { - // The config the vault was opened with. - config config.HashVaultConfig - - // Guards db, table, empty, head and closed, and so the table against being replaced while it is read. - // Commit, Reset and Close take it exclusively; lookups share it. - mu sync.RWMutex - - // The database the table lives in. Replaced when a mismatch or Reset rewrites the files. - db litt.DB - - // The table the hashes are recorded in. - table litt.Table - - // True when the vault holds no hashes. - empty bool - - // The newest recorded block. Meaningless when empty is true. - head uint64 - - // True once Close has run. - closed bool - - // The floor PruneBlockHashesBelow() has raised. Only ever rises. - outerFloor atomic.Uint64 - - // The floor the storage garbage collector has raised through PruneHistory(). Only ever rises. - gcFloor atomic.Uint64 -} - -// Open opens the vault under cfg.DataDir, creating it if it does not exist, and deletes -// cfg.LegacyPebbleDir if it is present. -func Open(cfg config.HashVaultConfig) (*HashVault, error) { - if err := cfg.Validate(); err != nil { - return nil, fmt.Errorf("invalid hash vault config: %w", err) - } - if err := deleteLegacyPebbleVault(cfg.LegacyPebbleDir); err != nil { - return nil, fmt.Errorf("open the hash vault: %w", err) - } - v := &HashVault{config: cfg} - if err := v.openTable(); err != nil { - return nil, fmt.Errorf("open the hash vault: %w", err) - } - return v, nil -} - -// deleteLegacyPebbleVault deletes the Pebble-backed vault this one replaced, if it is present. Its hashes -// are app hashes rather than state hashes, so none of them can be carried over. -// -// This can be deleted once every node that ran the Pebble-backed vault has started on this one. -func deleteLegacyPebbleVault(dir string) error { - if dir == "" { - return nil - } - if _, err := os.Stat(dir); err != nil { - if os.IsNotExist(err) { - return nil - } - return fmt.Errorf("stat legacy hash vault dir %q: %w", dir, err) - } - if err := os.RemoveAll(dir); err != nil { - return fmt.Errorf("delete legacy hash vault dir %q: %w", dir, err) - } - logger.Info("Deleted the legacy Pebble hash vault; its app hashes cannot be compared with state hashes", - "dir", dir) - return nil -} - -// littConfig returns the config a vault's LittDB is opened, surveyed and pruned with. -func littConfig(vaultCfg config.HashVaultConfig) (*litt.Config, error) { - cfg, err := litt.DefaultConfig(vaultCfg.DataDir) - if err != nil { - return nil, fmt.Errorf("build hash vault littdb config: %w", err) - } - cfg.Fsync = vaultCfg.Fsync - // The table holds a few thousand small keys, so an in-memory keymap rebuilt at open is cheap, and it - // spares every flush a second database to sync. - cfg.KeymapType = keymap.MemKeymapType - cfg.DoubleWriteProtection = true - return cfg, nil -} - -// openTable opens the database and its table, and loads the newest recorded block. -func (v *HashVault) openTable() error { - cfg, err := littConfig(v.config) - if err != nil { - return fmt.Errorf("open the hash vault table: %w", err) - } - db, err := littbuilder.NewDB(cfg) - if err != nil { - return fmt.Errorf("open hash vault littdb at %q: %w", v.config.DataDir, err) - } - tableConfig := litt.DefaultTableConfig(tableName) - // A single write shard is what makes the writes that survive a crash a prefix of the ones issued, so - // a crash can shorten the recorded range but never leave a gap in it. - tableConfig.ShardingFactor = 1 - // A TTL is required for LittDB to collect at all. This one is shorter than any block, so the GC filter - // alone decides what is deleted. - tableConfig.TTL = time.Nanosecond - tableConfig.GCFilter = v.gcFilter - table, err := db.BuildTable(tableConfig) - if err != nil { - _ = db.Close() - return fmt.Errorf("open hash vault table: %w", err) - } - v.db = db - v.table = table - if err := v.loadRange(); err != nil { - _ = db.Close() - return fmt.Errorf("load the hash vault's range: %w", err) - } - return nil -} - -// loadRange reads the newest recorded block and checks that the recorded blocks are contiguous. -func (v *HashVault) loadRange() error { - newestKey, found, err := v.table.GetNewestKey() - if err != nil { - return fmt.Errorf("read the newest hash vault key: %w", err) - } - if !found { - v.empty = true - v.head = 0 - return nil - } - newest, err := decodeKey(newestKey) - if err != nil { - return fmt.Errorf("decode the newest hash vault key: %w", err) - } - oldestKey, found, err := v.table.GetOldestKey() - if err != nil { - return fmt.Errorf("read the oldest hash vault key: %w", err) - } - if !found { - return fmt.Errorf("hash vault has a newest key but no oldest key") - } - oldest, err := decodeKey(oldestKey) - if err != nil { - return fmt.Errorf("decode the oldest hash vault key: %w", err) - } - if count := v.table.KeyCount(); oldest > newest || newest-oldest+1 != count { - return fmt.Errorf("hash vault at %q is corrupt: it holds %d hashes for blocks %d to %d, which is not "+ - "one per block", v.config.DataDir, count, oldest, newest) - } - v.empty = false - v.head = newest - return nil -} - -// Head returns the newest recorded block, and false when the vault holds no hashes. -func (v *HashVault) Head() (uint64, bool) { - v.mu.RLock() - defer v.mu.RUnlock() - return v.head, !v.empty -} - -// Commit records hash as blockNumber's hash, or checks it against the hash already recorded for that -// block. It returns once the hash is recorded, and the recording is crash durable when cfg.Fsync is set. -// -// An empty vault takes any block, and a vault that is not empty takes a block at or below its newest -// recorded block, or the one after it. A block further ahead is an error. A hash that differs from the -// recorded one, or a block below the oldest recorded one, is an error when cfg.HaltOnMismatch is set; -// otherwise the vault discards its hashes from that block up and records this one in their place. -func (v *HashVault) Commit(blockNumber uint64, hash [32]byte) error { - v.mu.Lock() - defer v.mu.Unlock() - if v.closed { - return fmt.Errorf("commit the hash of block %d: the hash vault is closed", blockNumber) - } - - if v.empty || blockNumber == v.head+1 { - if err := v.append(blockNumber, hash); err != nil { - return fmt.Errorf("commit the hash of block %d: %w", blockNumber, err) - } - return nil - } - if blockNumber > v.head { - return fmt.Errorf("commit the hash of block %d: the hash vault's newest block is %d, so block %d "+ - "would leave a gap", blockNumber, v.head, blockNumber) - } - - recorded, found, err := v.read(blockNumber) - if err != nil { - return fmt.Errorf("commit the hash of block %d: %w", blockNumber, err) - } - if found && recorded == hash { - return nil - } - if err := v.resolveMismatch(blockNumber, hash, recorded, found); err != nil { - return fmt.Errorf("commit the hash of block %d: %w", blockNumber, err) - } - return nil -} - -// resolveMismatch handles a hash for blockNumber that differs from the recorded one, or a block below the -// oldest recorded one, which found reports. It halts or replaces the recorded hashes, as -// cfg.HaltOnMismatch selects. -func (v *HashVault) resolveMismatch(blockNumber uint64, hash [32]byte, recorded [32]byte, found bool) error { - recordedHex := "" - if found { - recordedHex = hex.EncodeToString(recorded[:]) - } - fields := []any{ - "blockNumber", blockNumber, - "recordedHex", recordedHex, - "incomingHex", hex.EncodeToString(hash[:]), - "newestRecordedBlock", v.head, - "hashVaultDir", v.config.DataDir, - } - - if v.config.HaltOnMismatch { - logger.Error("HASH VAULT MISMATCH: the node computed a different state hash for a block it already "+ - "recorded, or a block older than any it keeps. Halting. DO NOT RESTART WITHOUT HUMAN "+ - "INVESTIGATION. To continue past this instead, set hash-vault-halt-on-mismatch = false.", - fields...) - return fmt.Errorf("hash vault mismatch at block %d: recorded %s, computed %x", - blockNumber, recordedHex, hash) - } - - logger.Error("HASH VAULT MISMATCH: the node computed a different state hash for a block it already "+ - "recorded, or a block older than any it keeps. hash-vault-halt-on-mismatch is false, so the "+ - "recorded hashes from this block up are discarded and the new hash replaces them.", fields...) - if err := v.discardFrom(blockNumber); err != nil { - return fmt.Errorf("discard the hash vault from block %d after a mismatch: %w", blockNumber, err) - } - if err := v.append(blockNumber, hash); err != nil { - return fmt.Errorf("record the replacing hash of block %d: %w", blockNumber, err) - } - return nil -} - -// discardFrom closes the database, deletes every hash from blockNumber up, and reopens it. When no hash -// below blockNumber is recorded, the vault is left empty. -func (v *HashVault) discardFrom(blockNumber uint64) error { - if err := v.db.Close(); err != nil { - return fmt.Errorf("close the hash vault before pruning it: %w", err) - } - if err := pruneFrom(v.config, blockNumber); err != nil { - return fmt.Errorf("prune the hash vault from block %d: %w", blockNumber, err) - } - if err := v.openTable(); err != nil { - return fmt.Errorf("reopen the hash vault after pruning it: %w", err) - } - return nil -} - -// Reset deletes every recorded hash and records hash as blockNumber's, leaving it the only one. -func (v *HashVault) Reset(blockNumber uint64, hash [32]byte) error { - v.mu.Lock() - defer v.mu.Unlock() - if v.closed { - return fmt.Errorf("reset the hash vault to block %d: the hash vault is closed", blockNumber) - } - if err := v.db.Close(); err != nil { - return fmt.Errorf("close the hash vault before resetting it: %w", err) - } - if err := os.RemoveAll(v.config.DataDir); err != nil { - return fmt.Errorf("delete the hash vault at %q: %w", v.config.DataDir, err) - } - if err := v.openTable(); err != nil { - return fmt.Errorf("reopen the hash vault after deleting it: %w", err) - } - logger.Info("Reset the hash vault", "blockNumber", blockNumber, "hashVaultDir", v.config.DataDir) - if err := v.append(blockNumber, hash); err != nil { - return fmt.Errorf("record the hash of block %d after a reset: %w", blockNumber, err) - } - return nil -} - -// append records hash as blockNumber's hash and flushes it. blockNumber must be the block after the -// newest recorded one, or any block when the vault is empty. -func (v *HashVault) append(blockNumber uint64, hash [32]byte) error { - if err := v.table.Put(encodeKey(blockNumber), encodeValue(hash)); err != nil { - return fmt.Errorf("record the hash of block %d: %w", blockNumber, err) - } - if err := v.table.Flush(); err != nil { - return fmt.Errorf("flush the hash of block %d: %w", blockNumber, err) - } - v.empty = false - v.head = blockNumber - return nil -} - -// read returns the hash recorded for blockNumber, and false when none is. -func (v *HashVault) read(blockNumber uint64) ([32]byte, bool, error) { - value, found, err := v.table.Get(encodeKey(blockNumber)) - if err != nil { - return [32]byte{}, false, fmt.Errorf("read the hash of block %d: %w", blockNumber, err) - } - if !found { - return [32]byte{}, false, nil - } - hash, err := decodeValue(value) - if err != nil { - return [32]byte{}, false, fmt.Errorf("decode the hash of block %d: %w", blockNumber, err) - } - return hash, true, nil -} - -// Get returns the hash recorded for blockNumber, without blocking. -func (v *HashVault) Get(blockNumber uint64) ([32]byte, gigatypes.BlockHashStatus, error) { - v.mu.RLock() - defer v.mu.RUnlock() - if v.closed { - return [32]byte{}, gigatypes.BlockHashStatusError, - fmt.Errorf("get the hash of block %d: the hash vault is closed", blockNumber) - } - if v.empty || blockNumber > v.head { - return [32]byte{}, gigatypes.BlockHashStatusNotReady, nil - } - hash, found, err := v.read(blockNumber) - if err != nil { - return [32]byte{}, gigatypes.BlockHashStatusError, - fmt.Errorf("get the hash of block %d: %w", blockNumber, err) - } - if !found { - // The recorded blocks are contiguous up to head, so a block at or below it that is missing has - // been pruned. - return [32]byte{}, gigatypes.BlockHashStatusTooOld, nil - } - return hash, gigatypes.BlockHashStatusFound, nil -} - -// Close closes the vault. Every later call fails. -func (v *HashVault) Close() error { - v.mu.Lock() - defer v.mu.Unlock() - if v.closed { - return nil - } - v.closed = true - if err := v.db.Close(); err != nil { - return fmt.Errorf("close the hash vault: %w", err) - } - return nil -} +// HashVault is a safety mechanism to prevent a validator from "changing its mind" about the hash of a block +// without human intervention. +type HashVault interface { + + // CommitToHash takes a provided hash for a block and writes it to disk. This method blocks until the hash is + // crash durable. + // + // This utility may be passed the hash for a block multiple times, but it will refuse to allow the hash to change + // for a particular block height. If this method returns nil, then it means that the hash is either the first + // one observed by the HashVault, or that the hash is the same as one for this block that was previously reported. + // + // If this method returns an error, DO NOT ATTEMPT TO RECOVER WITHOUT HUMAN INTERVENTION! + CommitToHash(ctx context.Context, blockHeight uint64, hash []byte) error + + // Prune deletes all data for blocks below the specified height. Keeps data for the specified block height. + // Note that reporting the hash for a block below the pruning boundary will result in an error + // (as it is impossible to validate the correctness of the hash for a block below the pruning boundary). + Prune(ctx context.Context, blockHeight uint64) error + + // Close shuts the HashVault down and frees all resources (but does not delete the data from disk). + Close(ctx context.Context) error +} + +// BlockHashSize is the required byte length for hashes passed to CommitToHash (CometBFT block ID / header hash). +const BlockHashSize = 32 + +// ErrInvalidHashLength is returned when CommitToHash is called with a hash whose length is not BlockHashSize. +var ErrInvalidHashLength = errors.New("block hash must be 32 bytes") + +// ErrHashMismatch is returned by CommitToHash when the caller provides a hash that differs from the +// hash previously committed for the same block height. This is the primary "node changed its mind" +// signal and MUST cause the calling node to halt. +var ErrHashMismatch = errors.New("block hash mismatch") + +// ErrBelowPruneBoundary is returned when an operation targets a block height that has already been +// pruned. Reporting or rolling back through pruned heights is impossible to validate and so is rejected. +var ErrBelowPruneBoundary = errors.New("block height below prune boundary") + +// ErrClosed is returned when a method is called after Close. +var ErrClosed = errors.New("hashvault is closed") + +// ErrCorruption is returned when the on-disk integrity check (SHA-256 trailer bound to (height, hash)) +// fails. This indicates either disk corruption or a bug in the encoding layer. Callers MUST treat +// this as fatal and require human intervention. +var ErrCorruption = errors.New("hashvault on-disk integrity check failed") + +// ErrRollbackHeightOverflow is returned by HardRollbackPebbleHashVault when blockHeight is +// math.MaxUint64. The partial-rollback path deletes hashes strictly above blockHeight via +// DeleteRange(hashKey(blockHeight+1), ...); at MaxUint64 that addition wraps to zero and would +// delete the entire vault instead of none. +var ErrRollbackHeightOverflow = errors.New("rollback block height overflows uint64") diff --git a/sei-db/state_db/sc/hashvault/hashvault_config.go b/sei-db/state_db/sc/hashvault/hashvault_config.go new file mode 100644 index 0000000000..25432211f7 --- /dev/null +++ b/sei-db/state_db/sc/hashvault/hashvault_config.go @@ -0,0 +1,56 @@ +package hashvault + +import ( + "fmt" +) + +// HashVaultConfig is the configuration for a HashVault. +type HashVaultConfig struct { + // DataDir is the directory in which the PebbleDB-backed HashVault stores its data. + DataDir string + + // Fsync controls whether the underlying Pebble writes are fsynced. + // + // This field is test-only. Production callers should construct via NewPebbleHashVault, which forces + // fsync on regardless of this value. NewUnsafePebbleHashVault honors this flag and is intended for + // tests that exercise enough writes that fsync would dominate runtime. + Fsync bool + + // CacheSize is the number of recent (height -> verified hash) entries in the in-process LRU cache. + CacheSize int + + // HaltOnMismatch selects what a hash that differs from the recorded one does. When true, CommitToHash + // returns ErrHashMismatch. When false, the mismatch is logged, the recorded hashes from that block up + // are discarded, and the new hash is recorded in their place. + HaltOnMismatch bool + + // EmptyVaultRollbackBlocks is how many blocks the state DB rewinds and replays when it opens over an + // empty vault, so that the vault holds the hashes of recent blocks and not just the loaded one. + EmptyVaultRollbackBlocks uint64 + + // LegacyPebbleDir is the directory of the app-hash vault this one replaces, deleted when the vault + // opens. Empty when there is none to delete. + LegacyPebbleDir string +} + +// DefaultHashVaultConfig returns a HashVaultConfig with production defaults. +func DefaultHashVaultConfig() HashVaultConfig { + return HashVaultConfig{ + Fsync: false, + CacheSize: 1024, + HaltOnMismatch: true, + EmptyVaultRollbackBlocks: 1000, + } +} + +// Validate returns a non-nil error if the configuration is missing required fields or has values +// that the HashVault cannot accept. +func (c *HashVaultConfig) Validate() error { + if c.DataDir == "" { + return fmt.Errorf("data directory is required") + } + if c.CacheSize <= 0 { + return fmt.Errorf("cache size must be greater than zero") + } + return nil +} diff --git a/sei-db/state_db/sc/hashvault/hashvault_test.go b/sei-db/state_db/sc/hashvault/hashvault_test.go index 2462864271..a3269254a3 100644 --- a/sei-db/state_db/sc/hashvault/hashvault_test.go +++ b/sei-db/state_db/sc/hashvault/hashvault_test.go @@ -1,274 +1,220 @@ package hashvault import ( - "os" - "path/filepath" + "context" + "errors" + "fmt" + "sync" "testing" "github.com/stretchr/testify/require" - - "github.com/sei-protocol/sei-chain/sei-db/config" - gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" ) -// testConfig returns a config for a vault in a fresh directory. Fsync is off, since the tests flush after -// every hash and the durability is LittDB's to prove, not this package's. -func testConfig(t *testing.T, haltOnMismatch bool) config.HashVaultConfig { - t.Helper() - cfg := config.DefaultHashVaultConfig() - cfg.DataDir = filepath.Join(t.TempDir(), "hashvault") - cfg.HaltOnMismatch = haltOnMismatch - cfg.Fsync = false - return cfg -} - -// openVault opens a vault from cfg, closed when the test ends. -func openVault(t *testing.T, cfg config.HashVaultConfig) *HashVault { - t.Helper() - v, err := Open(cfg) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, v.Close()) }) - return v -} +// Contract-level tests for HashVault. These exercise the externally-visible behavior promised by +// the HashVault interface against the PebbleHashVault implementation. Pebble-specific surface +// (encoding, restart recovery, on-disk inspection, the static rollback function, etc.) is tested +// per-implementation in pebble_hashvault_test.go and pebble_hashvault_rollback_test.go. -// hashOf returns a hash that differs for every distinct seed. -func hashOf(seed byte) [32]byte { - var hash [32]byte - for i := range hash { - hash[i] = seed +func bytesOfLen(b byte, n int) []byte { + out := make([]byte, n) + for i := range out { + out[i] = b } - return hash + return out } -// commitRange commits the hashes of blocks first to last, each seeded with its own block number. -func commitRange(t *testing.T, v *HashVault, first uint64, last uint64) { - t.Helper() - for block := first; block <= last; block++ { - require.NoError(t, v.Commit(block, hashOf(byte(block)))) - } -} +func TestCommitRejectsInvalidHashLength(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) -// requireHash asserts the vault holds want for blockNumber. -func requireHash(t *testing.T, v *HashVault, blockNumber uint64, want [32]byte) { - t.Helper() - got, status, err := v.Get(blockNumber) - require.NoError(t, err) - require.Equal(t, gigatypes.BlockHashStatusFound, status, "block %d", blockNumber) - require.Equal(t, want, got, "block %d", blockNumber) + require.ErrorIs(t, v.CommitToHash(ctx, 1, nil), ErrInvalidHashLength) + require.ErrorIs(t, v.CommitToHash(ctx, 1, []byte{}), ErrInvalidHashLength) + require.ErrorIs(t, v.CommitToHash(ctx, 1, bytesOfLen(0xAA, 31)), ErrInvalidHashLength) + require.ErrorIs(t, v.CommitToHash(ctx, 1, bytesOfLen(0xAA, 33)), ErrInvalidHashLength) } -// An empty vault has no range to hold a block to, so the first block may be any height, and nothing is -// ready to be read until it is recorded. -func TestAnEmptyVaultTakesAnyFirstBlock(t *testing.T) { - v := openVault(t, testConfig(t, true)) - - _, recorded := v.Head() - require.False(t, recorded) - _, status, err := v.Get(5) - require.NoError(t, err) - require.Equal(t, gigatypes.BlockHashStatusNotReady, status) - - require.NoError(t, v.Commit(100, hashOf(1))) - head, recorded := v.Head() - require.True(t, recorded) - require.Equal(t, uint64(100), head) - requireHash(t, v, 100, hashOf(1)) +func TestCommitFirstTime(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + hash := bytesOfLen(0xAA, 32) + require.NoError(t, v.CommitToHash(ctx, 7, hash)) } -// Blocks above the newest recorded one are not ready, whether or not they have been committed yet. -func TestABlockAboveTheHeadIsNotReady(t *testing.T) { - v := openVault(t, testConfig(t, true)) - commitRange(t, v, 1, 3) - - _, status, err := v.Get(4) - require.NoError(t, err) - require.Equal(t, gigatypes.BlockHashStatusNotReady, status) +func TestCommitIdempotent(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + hash := bytesOfLen(0xAB, 32) + require.NoError(t, v.CommitToHash(ctx, 7, hash)) + require.NoError(t, v.CommitToHash(ctx, 7, hash)) + require.NoError(t, v.CommitToHash(ctx, 7, hash)) } -// The recorded range is contiguous, so a block that would leave a gap is refused whatever the mismatch -// policy is. -func TestAGapIsRefused(t *testing.T) { - for _, halt := range []bool{true, false} { - v := openVault(t, testConfig(t, halt)) - commitRange(t, v, 1, 3) +func TestCommitMismatch(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + a := bytesOfLen(0x01, 32) + b := bytesOfLen(0x02, 32) + require.NoError(t, v.CommitToHash(ctx, 42, a)) - require.ErrorContains(t, v.Commit(5, hashOf(5)), "gap") - head, _ := v.Head() - require.Equal(t, uint64(3), head, "a refused block must not be recorded") + err := v.CommitToHash(ctx, 42, b) + require.Error(t, err) + require.ErrorIs(t, err, ErrHashMismatch) +} + +func TestCommitMismatchAfterRepeatedCommitIsSticky(t *testing.T) { + // Even after re-committing the same hash many times, a single mismatch still surfaces. This + // is essentially a regression check that the cache fast path also enforces the mismatch. + ctx := context.Background() + v := newTestPebbleVault(t) + a := bytesOfLen(0x55, 32) + b := bytesOfLen(0x66, 32) + for i := 0; i < 10; i++ { + require.NoError(t, v.CommitToHash(ctx, 5, a)) } + err := v.CommitToHash(ctx, 5, b) + require.ErrorIs(t, err, ErrHashMismatch) } -// Re-execution reproduces the hashes it recorded before, so committing the same hash again is a check -// that passes, not a write. -func TestRecommittingTheSameHashPasses(t *testing.T) { - v := openVault(t, testConfig(t, true)) - commitRange(t, v, 1, 5) - - commitRange(t, v, 2, 5) - head, _ := v.Head() - require.Equal(t, uint64(5), head) - requireHash(t, v, 3, hashOf(3)) -} - -// With halting selected, a different hash for a recorded block fails and leaves the record as it was. -func TestAMismatchHaltsWhenHaltingIsSelected(t *testing.T) { - v := openVault(t, testConfig(t, true)) - commitRange(t, v, 1, 5) - - require.ErrorContains(t, v.Commit(3, hashOf(0xEE)), "mismatch") - requireHash(t, v, 3, hashOf(3)) - head, _ := v.Head() - require.Equal(t, uint64(5), head) -} - -// With halting off, a different hash replaces the recorded one, and the hashes above it go with it: they -// were derived from the state the new hash disowns. -func TestAMismatchReplacesTheRecordWhenHaltingIsOff(t *testing.T) { - v := openVault(t, testConfig(t, false)) - commitRange(t, v, 1, 5) - - require.NoError(t, v.Commit(3, hashOf(0xEE))) - requireHash(t, v, 2, hashOf(2)) - requireHash(t, v, 3, hashOf(0xEE)) - head, _ := v.Head() - require.Equal(t, uint64(3), head) - _, status, err := v.Get(4) - require.NoError(t, err) - require.Equal(t, gigatypes.BlockHashStatusNotReady, status) - - require.NoError(t, v.Commit(4, hashOf(0xEF)), "commits carry on from the replaced block") - requireHash(t, v, 4, hashOf(0xEF)) -} - -// A mismatch at the oldest recorded block discards every hash, which the vault survives as an empty one. -func TestAMismatchAtTheOldestBlockLeavesOnlyTheNewHash(t *testing.T) { - v := openVault(t, testConfig(t, false)) - commitRange(t, v, 10, 12) - - require.NoError(t, v.Commit(10, hashOf(0xEE))) - requireHash(t, v, 10, hashOf(0xEE)) - head, _ := v.Head() - require.Equal(t, uint64(10), head) -} - -// What the vault records survives a restart, including a record a mismatch rewrote. -func TestTheRecordSurvivesAReopen(t *testing.T) { - cfg := testConfig(t, false) - v, err := Open(cfg) - require.NoError(t, err) - commitRange(t, v, 1, 5) - require.NoError(t, v.Commit(4, hashOf(0xEE))) - require.NoError(t, v.Close()) - - reopened := openVault(t, cfg) - head, recorded := reopened.Head() - require.True(t, recorded) - require.Equal(t, uint64(4), head) - requireHash(t, reopened, 3, hashOf(3)) - requireHash(t, reopened, 4, hashOf(0xEE)) - require.ErrorContains(t, reopened.Commit(6, hashOf(6)), "gap", "the reopened vault still refuses gaps") -} - -// Reset leaves the block it is given as the only one recorded, and commits carry on from it. -func TestResetLeavesOnlyTheGivenBlock(t *testing.T) { - v := openVault(t, testConfig(t, true)) - commitRange(t, v, 1, 5) - - require.NoError(t, v.Reset(1000, hashOf(0xAB))) - head, recorded := v.Head() - require.True(t, recorded) - require.Equal(t, uint64(1000), head) - requireHash(t, v, 1000, hashOf(0xAB)) - require.Equal(t, uint64(1), v.table.KeyCount()) - - require.NoError(t, v.Commit(1001, hashOf(0xAC))) - requireHash(t, v, 1001, hashOf(0xAC)) -} - -// The Pebble vault this one replaced holds app hashes nothing can use, so opening deletes it. -func TestOpenDeletesTheLegacyPebbleVault(t *testing.T) { - cfg := testConfig(t, true) - cfg.LegacyPebbleDir = filepath.Join(t.TempDir(), "hashvault") - require.NoError(t, os.MkdirAll(cfg.LegacyPebbleDir, 0o750)) - require.NoError(t, os.WriteFile(filepath.Join(cfg.LegacyPebbleDir, "000001.log"), []byte("x"), 0o600)) - - openVault(t, cfg) - _, err := os.Stat(cfg.LegacyPebbleDir) - require.True(t, os.IsNotExist(err), "the legacy vault must be gone, got %v", err) -} +func TestPruneRemovesData(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) -// A legacy dir that is not there is the common case once the testnet has run this build, not an error. -func TestOpenWithoutALegacyPebbleVault(t *testing.T) { - cfg := testConfig(t, true) - cfg.LegacyPebbleDir = filepath.Join(t.TempDir(), "absent") - openVault(t, cfg) -} + // Commit a handful of heights, prune below 5, then probe around the boundary. + for h := uint64(1); h <= 10; h++ { + require.NoError(t, v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32))) + } + require.NoError(t, v.Prune(ctx, 5)) + + // Below the boundary is rejected. + require.ErrorIs(t, + v.CommitToHash(ctx, 3, bytesOfLen(0x03, 32)), + ErrBelowPruneBoundary, + ) + // At the boundary is allowed (and the previously-committed hash is still locked in). + require.NoError(t, v.CommitToHash(ctx, 5, bytesOfLen(0x05, 32))) + require.ErrorIs(t, + v.CommitToHash(ctx, 5, bytesOfLen(0x55, 32)), + ErrHashMismatch, + ) + // Above the boundary is allowed and still locked. + require.NoError(t, v.CommitToHash(ctx, 7, bytesOfLen(0x07, 32))) + require.ErrorIs(t, + v.CommitToHash(ctx, 7, bytesOfLen(0x77, 32)), + ErrHashMismatch, + ) +} + +func TestCommitBelowPruneBoundary(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + + require.NoError(t, v.Prune(ctx, 100)) + // Strictly below the boundary is rejected. + require.ErrorIs(t, + v.CommitToHash(ctx, 99, bytesOfLen(0xAA, 32)), + ErrBelowPruneBoundary, + ) + require.ErrorIs(t, + v.CommitToHash(ctx, 50, bytesOfLen(0xAA, 32)), + ErrBelowPruneBoundary, + ) + // At the boundary is allowed: Prune keeps the boundary block per the godoc. + require.NoError(t, v.CommitToHash(ctx, 100, bytesOfLen(0xAA, 32))) + // Above is also obviously fine. + require.NoError(t, v.CommitToHash(ctx, 101, bytesOfLen(0xAA, 32))) +} + +func TestPruneMonotonic(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + + require.NoError(t, v.Prune(ctx, 50)) + require.NoError(t, v.Prune(ctx, 25)) // no-op + // Committing at 30 still errors: the effective boundary is still 50. + require.ErrorIs(t, + v.CommitToHash(ctx, 30, bytesOfLen(0xAA, 32)), + ErrBelowPruneBoundary, + ) + // Just below the boundary still errors. + require.ErrorIs(t, + v.CommitToHash(ctx, 49, bytesOfLen(0xAA, 32)), + ErrBelowPruneBoundary, + ) + // At and above the boundary succeed. + require.NoError(t, v.CommitToHash(ctx, 50, bytesOfLen(0x50, 32))) + require.NoError(t, v.CommitToHash(ctx, 51, bytesOfLen(0xAA, 32))) +} + +func TestCloseIsIdempotent(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + require.NoError(t, v.Close(ctx)) + require.NoError(t, v.Close(ctx)) + require.NoError(t, v.Close(ctx)) +} + +func TestCallsAfterCloseError(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + require.NoError(t, v.Close(ctx)) + require.ErrorIs(t, v.CommitToHash(ctx, 1, bytesOfLen(0xAA, 32)), ErrClosed) + require.ErrorIs(t, v.Prune(ctx, 1), ErrClosed) +} + +func TestConcurrentCommits(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + + // Commit heights 1..N in order, since the vault refuses gaps. + var wg sync.WaitGroup + const N = 100 + for i := 0; i < N; i++ { + h := uint64(i + 1) + require.NoError(t, v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32))) + } -// A hash may be deleted only once both the owner and the storage garbage collector permit it, and neither -// permission is taken back by a later, lower one. -func TestGCDeletesOnlyBelowBothFloors(t *testing.T) { - v := openVault(t, testConfig(t, true)) - deletable := func(blockNumber uint64) bool { - t.Helper() - ok, err := v.gcFilter(encodeKey(blockNumber), true) + // Re-committing the same (height, hash) from many goroutines should also all succeed. + errs2 := make(chan error, N) + for i := 0; i < N; i++ { + wg.Add(1) + go func(h uint64) { + defer wg.Done() + errs2 <- v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32)) + }(uint64(i + 1)) + } + wg.Wait() + close(errs2) + for err := range errs2 { require.NoError(t, err) - return ok } - require.False(t, deletable(0), "nothing is deletable before either floor is raised") - - v.PruneBelow(100) - require.False(t, deletable(50), "the owner alone cannot delete a hash") - - require.NoError(t, v.PruneHistory(60)) - require.True(t, deletable(59)) - require.False(t, deletable(60), "the lower floor bounds what is deleted") - require.False(t, deletable(99)) - - require.NoError(t, v.PruneHistory(200)) - require.True(t, deletable(99)) - require.False(t, deletable(100), "the owner's floor now bounds what is deleted") - - v.PruneBelow(10) - require.NoError(t, v.PruneHistory(10)) - require.True(t, deletable(99), "a lower floor must not take back a permission already given") -} - -// The vault restores nothing from snapshots, so its rollback floor is its newest block less the window. -func TestRollbackFloorIsTheHeadLessTheWindow(t *testing.T) { - v := openVault(t, testConfig(t, true)) - require.Equal(t, uint64(0), v.GetRollbackFloor(10), "an empty vault constrains nothing") - - commitRange(t, v, 1, 30) - require.Equal(t, uint64(20), v.GetRollbackFloor(10)) - require.Equal(t, uint64(0), v.GetRollbackFloor(40), "a window deeper than the history floors at 0") - latest, err := v.GetLatestBlock() - require.NoError(t, err) - require.Equal(t, uint64(30), latest) -} - -// A record written in a format this build does not know is refused rather than read as a hash. -func TestAnUnknownRecordFormatIsRefused(t *testing.T) { - value := encodeValue(hashOf(1)) - value[0] = recordFormatVersion + 1 - _, err := decodeValue(value) - require.ErrorContains(t, err, "format version") - - _, err = decodeValue(value[:10]) - require.ErrorContains(t, err, "bytes") + // Committing a *different* hash at any of those heights from many goroutines should yield + // at least one mismatch error and never a hidden success. + errs3 := make(chan error, N) + for i := 0; i < N; i++ { + wg.Add(1) + go func(h uint64) { + defer wg.Done() + errs3 <- v.CommitToHash(ctx, h, bytesOfLen(0xFF, 32)) + }(uint64(i + 1)) + } + wg.Wait() + close(errs3) + mismatches := 0 + for err := range errs3 { + require.Error(t, err) + if errors.Is(err, ErrHashMismatch) { + mismatches++ + } + } + require.Equal(t, N, mismatches, "every concurrent different-hash commit must return ErrHashMismatch") } -// Every method fails once the vault is closed, rather than reading a table that is gone. -func TestAClosedVaultRefusesEverything(t *testing.T) { - v, err := Open(testConfig(t, true)) - require.NoError(t, err) - commitRange(t, v, 1, 2) - require.NoError(t, v.Close()) - require.NoError(t, v.Close(), "closing twice is harmless") - - require.Error(t, v.Commit(3, hashOf(3))) - require.Error(t, v.Reset(3, hashOf(3))) - _, status, err := v.Get(1) - require.Error(t, err) - require.Equal(t, gigatypes.BlockHashStatusError, status) +// Sanity check that fmt.Errorf wrapping of our sentinels via %w stays Is-compatible. Defends +// against accidental future refactors of the codec or handlers that lose the sentinel. +func TestErrorWrappingIsCompatible(t *testing.T) { + wrapped := fmt.Errorf("outer: %w", ErrCorruption) + require.ErrorIs(t, wrapped, ErrCorruption) + wrappedLen := fmt.Errorf("outer: %w", ErrInvalidHashLength) + require.ErrorIs(t, wrappedLen, ErrInvalidHashLength) } diff --git a/sei-db/state_db/sc/hashvault/noop_hashvault.go b/sei-db/state_db/sc/hashvault/noop_hashvault.go new file mode 100644 index 0000000000..65df820cd8 --- /dev/null +++ b/sei-db/state_db/sc/hashvault/noop_hashvault.go @@ -0,0 +1,30 @@ +package hashvault + +import "context" + +var _ HashVault = (*NoopHashVault)(nil) + +// NoopHashVault is a HashVault implementation that does nothing. It provides no equivocation +// protection whatsoever. It exists for two purposes: +// - tests that construct a BlockExecutor but do not exercise the vault, and +// - the explicit, operator-opted-in "hash-vault-disabled-unsafe" escape hatch. +// +// Production code must never substitute this for a real vault without a deliberate human decision. +type NoopHashVault struct{} + +// NewNoopHashVault returns a HashVault whose methods are all no-ops. +func NewNoopHashVault() *NoopHashVault { + return &NoopHashVault{} +} + +func (n *NoopHashVault) CommitToHash(_ context.Context, _ uint64, _ []byte) error { + return nil +} + +func (n *NoopHashVault) Prune(_ context.Context, _ uint64) error { + return nil +} + +func (n *NoopHashVault) Close(_ context.Context) error { + return nil +} diff --git a/sei-db/state_db/sc/hashvault/offline.go b/sei-db/state_db/sc/hashvault/offline.go deleted file mode 100644 index d0f63afbd0..0000000000 --- a/sei-db/state_db/sc/hashvault/offline.go +++ /dev/null @@ -1,119 +0,0 @@ -package hashvault - -import ( - "errors" - "fmt" - "math" - "os" - - "github.com/sei-protocol/sei-chain/sei-db/config" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/offline" -) - -// StoredRange returns the oldest and newest blocks the closed vault under cfg.DataDir records, and false -// when it records none. A vault that has never been created records none. -func StoredRange(cfg config.HashVaultConfig) (oldest uint64, newest uint64, recorded bool, err error) { - exists, err := vaultExists(cfg) - if err != nil { - return 0, 0, false, fmt.Errorf("read the hash vault's range: %w", err) - } - if !exists { - return 0, 0, false, nil - } - oldest, recorded, err = firstStoredBlock(cfg, false) - if err != nil { - return 0, 0, false, fmt.Errorf("read the oldest hash vault block: %w", err) - } - if !recorded { - return 0, 0, false, nil - } - newest, _, err = firstStoredBlock(cfg, true) - if err != nil { - return 0, 0, false, fmt.Errorf("read the newest hash vault block: %w", err) - } - return oldest, newest, true, nil -} - -// PruneAfter deletes every hash the closed vault under cfg.DataDir records above blockNumber. -func PruneAfter(cfg config.HashVaultConfig, blockNumber uint64) error { - if blockNumber == math.MaxUint64 { - return nil - } - if err := pruneFrom(cfg, blockNumber+1); err != nil { - return fmt.Errorf("prune the hash vault after block %d: %w", blockNumber, err) - } - return nil -} - -// pruneFrom deletes every hash the closed vault under cfg.DataDir records at or above blockNumber. A -// vault left with no hash is left empty. -func pruneFrom(cfg config.HashVaultConfig, blockNumber uint64) error { - exists, err := vaultExists(cfg) - if err != nil { - return fmt.Errorf("prune the hash vault: %w", err) - } - if !exists { - return nil - } - littCfg, err := littConfig(cfg) - if err != nil { - return fmt.Errorf("prune the hash vault: %w", err) - } - // The rollback walks from the newest hash down and keeps everything from the first one this accepts. - keep := func(_ string, key []byte, _ bool) (bool, error) { - recordedBlock, err := decodeKey(key) - if err != nil { - return false, fmt.Errorf("decode a hash vault key: %w", err) - } - return recordedBlock < blockNumber, nil - } - if err := offline.RollbackLittDB(littCfg, keep); err != nil { - return fmt.Errorf("prune the hash vault from block %d: %w", blockNumber, err) - } - return nil -} - -// firstStoredBlock returns the first block the closed vault yields in the direction reverse selects: the -// newest when reverse is true, the oldest otherwise. It returns false when the vault records none. -func firstStoredBlock(cfg config.HashVaultConfig, reverse bool) (blockNumber uint64, found bool, err error) { - littCfg, err := littConfig(cfg) - if err != nil { - return 0, false, fmt.Errorf("iterate the hash vault offline: %w", err) - } - iterator, err := offline.NewIterator(littCfg, tableName, reverse) - if err != nil { - return 0, false, fmt.Errorf("open an offline iterator over the hash vault: %w", err) - } - defer func() { - if closeErr := iterator.Close(); closeErr != nil { - err = errors.Join(err, fmt.Errorf("close the offline hash vault iterator: %w", closeErr)) - } - }() - found, err = iterator.Next() - if err != nil { - return 0, false, fmt.Errorf("iterate the hash vault offline: %w", err) - } - if !found { - return 0, false, nil - } - key, _, err := iterator.GetKey() - if err != nil { - return 0, false, fmt.Errorf("read a hash vault key: %w", err) - } - blockNumber, err = decodeKey(key) - if err != nil { - return 0, false, fmt.Errorf("decode a hash vault key: %w", err) - } - return blockNumber, true, nil -} - -// vaultExists reports whether the vault's directory exists. -func vaultExists(cfg config.HashVaultConfig) (bool, error) { - if _, err := os.Stat(cfg.DataDir); err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, fmt.Errorf("stat the hash vault dir %q: %w", cfg.DataDir, err) - } - return true, nil -} diff --git a/sei-db/state_db/sc/hashvault/offline_test.go b/sei-db/state_db/sc/hashvault/offline_test.go deleted file mode 100644 index dfa7681ccc..0000000000 --- a/sei-db/state_db/sc/hashvault/offline_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package hashvault - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -// A vault that was never created records nothing, which is what a node booting for the first time finds. -func TestStoredRangeOfAMissingVault(t *testing.T) { - _, _, recorded, err := StoredRange(testConfig(t, true)) - require.NoError(t, err) - require.False(t, recorded) -} - -// A vault that exists but holds no hashes records nothing either. -func TestStoredRangeOfAnEmptyVault(t *testing.T) { - cfg := testConfig(t, true) - v, err := Open(cfg) - require.NoError(t, err) - require.NoError(t, v.Close()) - - _, _, recorded, err := StoredRange(cfg) - require.NoError(t, err) - require.False(t, recorded) -} - -// The range read offline is the one the vault holds when open. -func TestStoredRangeOfAPopulatedVault(t *testing.T) { - cfg := testConfig(t, true) - v, err := Open(cfg) - require.NoError(t, err) - commitRange(t, v, 4, 9) - require.NoError(t, v.Close()) - - oldest, newest, recorded, err := StoredRange(cfg) - require.NoError(t, err) - require.True(t, recorded) - require.Equal(t, uint64(4), oldest) - require.Equal(t, uint64(9), newest) -} - -// PruneAfter keeps the hashes up to its block and drops the rest, and the vault reopens on what is kept. -func TestPruneAfterKeepsThePrefix(t *testing.T) { - cfg := testConfig(t, true) - v, err := Open(cfg) - require.NoError(t, err) - commitRange(t, v, 1, 9) - require.NoError(t, v.Close()) - - require.NoError(t, PruneAfter(cfg, 5)) - - _, newest, recorded, err := StoredRange(cfg) - require.NoError(t, err) - require.True(t, recorded) - require.Equal(t, uint64(5), newest) - - reopened := openVault(t, cfg) - requireHash(t, reopened, 5, hashOf(5)) - require.NoError(t, reopened.Commit(6, hashOf(0xEE)), "commits carry on from the kept prefix") -} - -// Pruning below every recorded hash leaves an empty vault, and pruning a missing one does nothing. -func TestPruneAfterBelowEveryHashEmptiesTheVault(t *testing.T) { - cfg := testConfig(t, true) - require.NoError(t, PruneAfter(cfg, 0), "a vault that was never created has nothing to prune") - - v, err := Open(cfg) - require.NoError(t, err) - commitRange(t, v, 3, 5) - require.NoError(t, v.Close()) - - require.NoError(t, PruneAfter(cfg, 2)) - _, _, recorded, err := StoredRange(cfg) - require.NoError(t, err) - require.False(t, recorded) -} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault.go b/sei-db/state_db/sc/hashvault/pebble_hashvault.go new file mode 100644 index 0000000000..e0448c9557 --- /dev/null +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault.go @@ -0,0 +1,493 @@ +package hashvault + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "fmt" + "os" + "sync" + "sync/atomic" + + "github.com/cockroachdb/pebble/v2" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/sei-protocol/seilog" + + gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" +) + +var _ HashVault = (*PebbleHashVault)(nil) + +var logger = seilog.NewLogger("db", "state-db", "sc", "hashvault") + +// PebbleHashVault is a PebbleDB-backed implementation of the HashVault interface. +type PebbleHashVault struct { + config HashVaultConfig + db *pebble.DB + writeOpts *pebble.WriteOptions + + mu sync.Mutex + // closed is true after Close. Every other public method returns ErrClosed once set. + closed bool + // pruneBoundary is the lowest height that may still be committed. + pruneBoundary uint64 + cache *lru.Cache[uint64, []byte] + + // head is the newest recorded height. Meaningless when recorded is false. + head uint64 + // recorded is true when the vault holds at least one hash. + recorded bool + + // outerFloor is the floor PruneBelow has raised. Only ever rises. + outerFloor atomic.Uint64 + // gcFloor is the floor PruneHistory has raised. Only ever rises. + gcFloor atomic.Uint64 +} + +// NewPebbleHashVault opens (or creates) a PebbleHashVault rooted at config.DataDir. +func NewPebbleHashVault(ctx context.Context, config HashVaultConfig) (*PebbleHashVault, error) { + if !config.Fsync { + logger.Info("forcing fsync on for production PebbleHashVault", "dataDir", config.DataDir) + } + config.Fsync = true + return newPebbleHashVault(ctx, config) +} + +// NewUnsafePebbleHashVault opens (or creates) a PebbleHashVault rooted at config.DataDir. Honors +// config.Fsync as set; intended for tests only. Never use in production: disabling fsync means a +// well-timed crash can lose the most recent committed hash and let the node vote a different hash +// for that block on the next boot. +func NewUnsafePebbleHashVault(ctx context.Context, config HashVaultConfig) (*PebbleHashVault, error) { + return newPebbleHashVault(ctx, config) +} + +func newPebbleHashVault(_ context.Context, config HashVaultConfig) (*PebbleHashVault, error) { + if err := config.Validate(); err != nil { + return nil, fmt.Errorf("invalid hashvault config: %w", err) + } + + if err := deleteLegacyVault(config.LegacyPebbleDir); err != nil { + return nil, fmt.Errorf("failed to open hashvault: %w", err) + } + + if err := os.MkdirAll(config.DataDir, 0o750); err != nil { + return nil, fmt.Errorf("failed to create hashvault data dir %q: %w", config.DataDir, err) + } + + db, err := pebble.Open(config.DataDir, &pebble.Options{}) + if err != nil { + return nil, fmt.Errorf("failed to open hashvault pebble db at %q: %w", config.DataDir, err) + } + + writeOpts := pebble.Sync + if !config.Fsync { + writeOpts = pebble.NoSync + } + + p := &PebbleHashVault{ + config: config, + db: db, + writeOpts: writeOpts, + cache: lru.NewCache[uint64, []byte](config.CacheSize), + } + + if err := p.loadPruneBoundary(); err != nil { + _ = db.Close() + return nil, err + } + + if err := p.loadHead(); err != nil { + _ = db.Close() + return nil, fmt.Errorf("failed to open hashvault: %w", err) + } + + empty, err := p.isEmpty() + if err != nil { + _ = db.Close() + return nil, err + } + if empty { + // Surface the fresh-start case: an operator who expected this node to already have an + // equivocation history on disk (e.g. after a restart) should notice an empty vault. + logger.Info("opened hashvault with no data on disk; starting with an empty equivocation history", + "dataDir", config.DataDir) + } + + return p, nil +} + +// isEmpty reports whether the underlying DB holds no keys at all (a freshly created vault with no +// committed hashes and no prune boundary). +func (p *PebbleHashVault) isEmpty() (bool, error) { + iter, err := p.db.NewIter(nil) + if err != nil { + return false, fmt.Errorf("failed to open hashvault iterator: %w", err) + } + defer func() { _ = iter.Close() }() + return !iter.First(), nil +} + +// loadPruneBoundary reads the on-disk prune boundary (if any) and populates p.pruneBoundary. +func (p *PebbleHashVault) loadPruneBoundary() error { + raw, closer, err := p.db.Get(pruneBoundaryKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return nil + } + return fmt.Errorf("failed to read prune boundary: %w", err) + } + defer func() { _ = closer.Close() }() + + boundary, err := decodeBoundaryValue(raw) + if err != nil { + logger.Error("hashvault prune boundary is malformed; refusing to start", + "dataDir", p.config.DataDir, "rawHex", hex.EncodeToString(raw), "err", err) + return err + } + p.pruneBoundary = boundary + return nil +} + +// CommitToHash implements HashVault. +func (p *PebbleHashVault) CommitToHash(ctx context.Context, blockHeight uint64, hash []byte) error { + if err := ctx.Err(); err != nil { + return err + } + p.mu.Lock() + defer p.mu.Unlock() + + if p.closed { + return ErrClosed + } + if blockHeight < p.pruneBoundary { + return ErrBelowPruneBoundary + } + if len(hash) != BlockHashSize { + return ErrInvalidHashLength + } + if p.recorded && blockHeight > p.head && blockHeight-p.head > 1 { + return fmt.Errorf("block %d would leave a gap after the newest recorded block %d", blockHeight, p.head) + } + + if cached, ok := p.cache.Get(blockHeight); ok { + if !bytes.Equal(cached, hash) { + if !p.config.HaltOnMismatch { + return p.replaceFrom(blockHeight, cached, hash) + } + p.logHashMismatch(blockHeight, cached, hash) + return ErrHashMismatch + } + return nil + } + + key := hashKey(blockHeight) + raw, closer, err := p.db.Get(key) + switch { + case errors.Is(err, pebble.ErrNotFound): + if p.recorded && blockHeight <= p.head { + // Below the oldest recorded height, where there is nothing to check the hash against. + if !p.config.HaltOnMismatch { + return p.replaceFrom(blockHeight, nil, hash) + } + return ErrBelowPruneBoundary + } + // First commit for this height: write it. + value := encodeHashValue(blockHeight, hash) + if werr := p.db.Set(key, value, p.writeOpts); werr != nil { + return fmt.Errorf("failed to persist hash for block %d: %w", blockHeight, werr) + } + p.cache.Add(blockHeight, bytes.Clone(hash)) + p.head = max(p.head, blockHeight) + p.recorded = true + return nil + case err != nil: + return fmt.Errorf("failed to read hash for block %d: %w", blockHeight, err) + } + // Found an existing entry; clone the raw bytes so we can release the closer before doing + // further work. + cloned := bytes.Clone(raw) + _ = closer.Close() + + existing, err := decodeHashValue(blockHeight, cloned) + if err != nil { + logger.Error("hashvault detected on-disk corruption; DO NOT RESTART WITHOUT HUMAN INVESTIGATION", + "blockHeight", blockHeight, "rawHex", hex.EncodeToString(cloned), "err", err) + return err + } + if !bytes.Equal(existing, hash) { + if !p.config.HaltOnMismatch { + return p.replaceFrom(blockHeight, existing, hash) + } + p.logHashMismatch(blockHeight, existing, hash) + return ErrHashMismatch + } + p.cache.Add(blockHeight, existing) + return nil +} + +// Prune implements HashVault. The boundary advance and range deletion are written in a single +// atomic Pebble batch: a crash mid-Prune either rolls forward to the new boundary (with the +// deletions applied) or leaves the old state intact. On return, every height strictly below +// blockHeight is guaranteed durable-deleted (subject to config.Fsync). +func (p *PebbleHashVault) Prune(ctx context.Context, blockHeight uint64) error { + if err := ctx.Err(); err != nil { + return err + } + p.mu.Lock() + defer p.mu.Unlock() + + if p.closed { + return ErrClosed + } + if blockHeight <= p.pruneBoundary { + return nil + } + + batch := p.db.NewBatch() + defer func() { _ = batch.Close() }() + if err := batch.Set(pruneBoundaryKey, encodeBoundaryValue(blockHeight), nil); err != nil { + return fmt.Errorf("failed to stage prune boundary advance to %d: %w", blockHeight, err) + } + // DeleteRange's upper bound is exclusive, so hashKey(blockHeight) keeps the boundary block + // itself per the HashVault.Prune contract. + if err := batch.DeleteRange(hashKey(0), hashKey(blockHeight), nil); err != nil { + return fmt.Errorf("failed to stage prune deletion below %d: %w", blockHeight, err) + } + if err := batch.Commit(p.writeOpts); err != nil { + return fmt.Errorf("failed to commit prune to %d: %w", blockHeight, err) + } + + p.pruneBoundary = blockHeight + return nil +} + +// Close implements HashVault. Subsequent calls return nil. After Close, every other public method +// returns ErrClosed. +func (p *PebbleHashVault) Close(_ context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { + return nil + } + p.closed = true + p.cache.Purge() + if err := p.db.Close(); err != nil { + return fmt.Errorf("failed to close hashvault pebble db: %w", err) + } + return nil +} + +func (p *PebbleHashVault) logHashMismatch(blockHeight uint64, existing, incoming []byte) { + logger.Error("Hashvault detected app hash mismatch; node attempted to change its mind. "+ + "DO NOT RESTART WITHOUT HUMAN INVESTIGATION. If you are CERTAIN this is not a real "+ + "equivocation, you can bypass this guard by stopping the node and deleting the HashVault "+ + "data directory (hashVaultDir below), then restarting. WARNING: deleting it removes "+ + "equivocation protection — if the node then commits a conflicting hash for a height it has "+ + "already finalized, the validator may be SLASHED.", + "blockHeight", blockHeight, + "existingHex", hex.EncodeToString(existing), + "incomingHex", hex.EncodeToString(incoming), + "hashVaultDir", p.config.DataDir, + ) +} + +// deleteLegacyVault deletes the app-hash vault this one replaces, if it is present. Its hashes are app +// hashes rather than state hashes, so none of them can be carried over. +// +// This can be deleted once every node that ran the app-hash vault has started on this one. +func deleteLegacyVault(dir string) error { + if dir == "" { + return nil + } + if _, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("failed to stat legacy hashvault dir %q: %w", dir, err) + } + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("failed to delete legacy hashvault dir %q: %w", dir, err) + } + logger.Info("deleted the legacy app-hash hashvault; its app hashes cannot be compared with state hashes", + "dir", dir) + return nil +} + +// loadHead reads the newest recorded height from disk and populates p.head and p.recorded. +func (p *PebbleHashVault) loadHead() error { + _, head, recorded, err := storedRange(p.db) + if err != nil { + return fmt.Errorf("failed to read the newest recorded height: %w", err) + } + p.head = head + p.recorded = recorded + return nil +} + +// replaceFrom discards every hash from blockHeight up and records hash as blockHeight's, in one atomic +// batch. It is how a mismatch is resolved when HaltOnMismatch is false. p.mu must be held. +func (p *PebbleHashVault) replaceFrom(blockHeight uint64, existing []byte, hash []byte) error { + logger.Error("Hashvault detected a state hash mismatch; hash-vault-halt-on-mismatch is false, so the "+ + "recorded hashes from this block up are discarded and the new hash replaces them.", + "blockHeight", blockHeight, + "existingHex", hex.EncodeToString(existing), + "incomingHex", hex.EncodeToString(hash), + "hashVaultDir", p.config.DataDir, + ) + batch := p.db.NewBatch() + defer func() { _ = batch.Close() }() + if err := batch.DeleteRange(hashKey(blockHeight), hashKeyUpperBound(), nil); err != nil { + return fmt.Errorf("failed to stage discarding hashes from block %d: %w", blockHeight, err) + } + if err := batch.Set(hashKey(blockHeight), encodeHashValue(blockHeight, hash), nil); err != nil { + return fmt.Errorf("failed to stage the replacing hash for block %d: %w", blockHeight, err) + } + if err := batch.Commit(p.writeOpts); err != nil { + return fmt.Errorf("failed to replace hashes from block %d: %w", blockHeight, err) + } + p.cache.Purge() + p.cache.Add(blockHeight, bytes.Clone(hash)) + p.head = blockHeight + p.recorded = true + return nil +} + +// Head returns the newest recorded height, and false when the vault holds no hashes. +func (p *PebbleHashVault) Head() (uint64, bool) { + p.mu.Lock() + defer p.mu.Unlock() + return p.head, p.recorded +} + +// Get returns the hash recorded for blockHeight, without blocking. +func (p *PebbleHashVault) Get(blockHeight uint64) ([32]byte, gigatypes.BlockHashStatus, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { + return [32]byte{}, gigatypes.BlockHashStatusError, ErrClosed + } + if !p.recorded || blockHeight > p.head { + return [32]byte{}, gigatypes.BlockHashStatusNotReady, nil + } + if blockHeight < p.pruneBoundary { + return [32]byte{}, gigatypes.BlockHashStatusTooOld, nil + } + raw, closer, err := p.db.Get(hashKey(blockHeight)) + if errors.Is(err, pebble.ErrNotFound) { + return [32]byte{}, gigatypes.BlockHashStatusTooOld, nil + } + if err != nil { + return [32]byte{}, gigatypes.BlockHashStatusError, + fmt.Errorf("failed to read hash for block %d: %w", blockHeight, err) + } + defer func() { _ = closer.Close() }() + hash, err := decodeHashValue(blockHeight, raw) + if err != nil { + return [32]byte{}, gigatypes.BlockHashStatusError, + fmt.Errorf("failed to decode hash for block %d: %w", blockHeight, err) + } + var out [32]byte + copy(out[:], hash) + return out, gigatypes.BlockHashStatusFound, nil +} + +// Reset deletes every recorded hash and the prune boundary, and records hash as blockHeight's, leaving it +// the only one. +func (p *PebbleHashVault) Reset(ctx context.Context, blockHeight uint64, hash []byte) error { + if err := ctx.Err(); err != nil { + return fmt.Errorf("failed to reset hashvault: %w", err) + } + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { + return ErrClosed + } + if len(hash) != BlockHashSize { + return ErrInvalidHashLength + } + batch := p.db.NewBatch() + defer func() { _ = batch.Close() }() + if err := batch.DeleteRange(hashKey(0), hashKeyUpperBound(), nil); err != nil { + return fmt.Errorf("failed to stage hashvault reset: %w", err) + } + if err := batch.Delete(pruneBoundaryKey, nil); err != nil { + return fmt.Errorf("failed to stage prune boundary clear during reset: %w", err) + } + if err := batch.Set(hashKey(blockHeight), encodeHashValue(blockHeight, hash), nil); err != nil { + return fmt.Errorf("failed to stage hash for block %d during reset: %w", blockHeight, err) + } + if err := batch.Commit(p.writeOpts); err != nil { + return fmt.Errorf("failed to reset hashvault to block %d: %w", blockHeight, err) + } + p.pruneBoundary = 0 + p.cache.Purge() + p.cache.Add(blockHeight, bytes.Clone(hash)) + p.head = blockHeight + p.recorded = true + return nil +} + +// PruneBelow permits the hashes of blocks below blockHeight to be deleted, as far as the vault's owner is +// concerned. A hash is deleted only once PruneHistory has permitted it too. +func (p *PebbleHashVault) PruneBelow(blockHeight uint64) { + raiseFloor(&p.outerFloor, blockHeight) +} + +// raiseFloor raises floor to blockHeight, leaving it where it is when it is already higher. +func raiseFloor(floor *atomic.Uint64, blockHeight uint64) { + for { + current := floor.Load() + if blockHeight <= current || floor.CompareAndSwap(current, blockHeight) { + return + } + } +} + +// Name implements controller.PrunableStore. +func (p *PebbleHashVault) Name() string { + return "HashVault" +} + +// PruneHistory implements controller.PrunableStore. It permits the hashes of blocks below blockHeight to +// be deleted, as far as the storage garbage collector is concerned, and prunes every hash below both that +// and the floor PruneBelow has raised. The newest recorded hash is always kept. +func (p *PebbleHashVault) PruneHistory(blockHeight uint64) error { + raiseFloor(&p.gcFloor, blockHeight) + head, recorded := p.Head() + if !recorded { + return nil + } + floor := min(p.outerFloor.Load(), p.gcFloor.Load(), head) + if err := p.Prune(context.Background(), floor); err != nil { + return fmt.Errorf("failed to prune hashvault below %d: %w", floor, err) + } + return nil +} + +// PruneSnapshots implements controller.PrunableStore. The vault keeps no snapshots. +func (p *PebbleHashVault) PruneSnapshots(uint64) error { + return nil +} + +// ExternalPruning implements controller.PrunableStore. The vault has no pruner of its own. +func (p *PebbleHashVault) ExternalPruning() bool { + return true +} + +// GetRollbackFloor implements controller.PrunableStore. Every recorded block is readable directly, so the +// floor is the newest recorded block less rollbackWindow, or 0 when the window is deeper than that. +func (p *PebbleHashVault) GetRollbackFloor(rollbackWindow uint64) uint64 { + head, recorded := p.Head() + if !recorded || head < rollbackWindow { + return 0 + } + return head - rollbackWindow +} + +// GetLatestBlock implements controller.PrunableStore. It returns the newest recorded block, or 0 when the +// vault holds no hashes. +func (p *PebbleHashVault) GetLatestBlock() (uint64, error) { + head, _ := p.Head() + return head, nil +} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go new file mode 100644 index 0000000000..0fc679a075 --- /dev/null +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go @@ -0,0 +1,194 @@ +package hashvault + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" +) + +// commitHeights commits the hashes of heights first to last, each seeded with its own height. +func commitHeights(t *testing.T, v *PebbleHashVault, first uint64, last uint64) { + t.Helper() + for h := first; h <= last; h++ { + require.NoError(t, v.CommitToHash(context.Background(), h, bytesOfLen(byte(h), 32))) + } +} + +// requireStatus asserts the status Get reports for height. +func requireStatus(t *testing.T, v *PebbleHashVault, height uint64, want gigatypes.BlockHashStatus) { + t.Helper() + _, status, err := v.Get(height) + require.NoError(t, err) + require.Equal(t, want, status, "height %d", height) +} + +// requireRecorded asserts the vault holds a hash seeded with seed for height. +func requireRecorded(t *testing.T, v *PebbleHashVault, height uint64, seed byte) { + t.Helper() + hash, status, err := v.Get(height) + require.NoError(t, err) + require.Equal(t, gigatypes.BlockHashStatusFound, status, "height %d", height) + require.Equal(t, bytesOfLen(seed, 32), hash[:], "height %d", height) +} + +func warnOnMismatch(cfg *HashVaultConfig) { + cfg.HaltOnMismatch = false +} + +func TestCommitRefusesAGap(t *testing.T) { + v := newTestPebbleVault(t) + commitHeights(t, v, 1, 3) + require.ErrorContains(t, v.CommitToHash(context.Background(), 5, bytesOfLen(5, 32)), "gap") + head, recorded := v.Head() + require.True(t, recorded) + require.Equal(t, uint64(3), head) +} + +func TestHeadSurvivesARestart(t *testing.T) { + v := newTestPebbleVault(t) + commitHeights(t, v, 1, 3) + v2 := reopenTestPebbleVault(t, v) + head, recorded := v2.Head() + require.True(t, recorded) + require.Equal(t, uint64(3), head) + require.ErrorContains(t, v2.CommitToHash(context.Background(), 5, bytesOfLen(5, 32)), "gap") +} + +func TestMismatchReplacesTheRecordWhenNotHalting(t *testing.T) { + v := newTestPebbleVault(t, warnOnMismatch) + commitHeights(t, v, 1, 5) + + require.NoError(t, v.CommitToHash(context.Background(), 3, bytesOfLen(0xEE, 32))) + requireRecorded(t, v, 2, 2) + requireRecorded(t, v, 3, 0xEE) + requireStatus(t, v, 4, gigatypes.BlockHashStatusNotReady) + require.NoError(t, v.CommitToHash(context.Background(), 4, bytesOfLen(0xEF, 32))) +} + +func TestCommitBelowTheOldestRecordedHeight(t *testing.T) { + t.Run("halting", func(t *testing.T) { + v := newTestPebbleVault(t) + commitHeights(t, v, 10, 12) + require.ErrorIs(t, v.CommitToHash(context.Background(), 5, bytesOfLen(5, 32)), ErrBelowPruneBoundary) + }) + t.Run("not halting", func(t *testing.T) { + v := newTestPebbleVault(t, warnOnMismatch) + commitHeights(t, v, 10, 12) + require.NoError(t, v.CommitToHash(context.Background(), 5, bytesOfLen(5, 32))) + requireRecorded(t, v, 5, 5) + requireStatus(t, v, 10, gigatypes.BlockHashStatusNotReady) + }) +} + +func TestGetStatuses(t *testing.T) { + v := newTestPebbleVault(t) + requireStatus(t, v, 1, gigatypes.BlockHashStatusNotReady) + + commitHeights(t, v, 1, 10) + require.NoError(t, v.Prune(context.Background(), 5)) + requireStatus(t, v, 4, gigatypes.BlockHashStatusTooOld) + requireRecorded(t, v, 5, 5) + requireStatus(t, v, 11, gigatypes.BlockHashStatusNotReady) + + require.NoError(t, v.Close(context.Background())) + _, status, err := v.Get(5) + require.ErrorIs(t, err, ErrClosed) + require.Equal(t, gigatypes.BlockHashStatusError, status) +} + +func TestResetLeavesOnlyTheGivenHeight(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + commitHeights(t, v, 1, 5) + require.NoError(t, v.Prune(ctx, 3)) + + require.NoError(t, v.Reset(ctx, 1000, bytesOfLen(0xAB, 32))) + requireRecorded(t, v, 1000, 0xAB) + requireStatus(t, v, 4, gigatypes.BlockHashStatusTooOld) + require.NoError(t, v.CommitToHash(ctx, 1001, bytesOfLen(0xAC, 32))) + + cfg := v.config + require.NoError(t, v.Close(ctx)) + oldest, newest, recorded, err := StoredRange(cfg) + require.NoError(t, err) + require.True(t, recorded) + require.Equal(t, uint64(1000), oldest) + require.Equal(t, uint64(1001), newest) +} + +func TestPruneHistoryDeletesOnlyBelowBothFloors(t *testing.T) { + v := newTestPebbleVault(t) + commitHeights(t, v, 1, 150) + + require.NoError(t, v.PruneHistory(60)) + requireRecorded(t, v, 1, 1) + + v.PruneBelow(100) + require.NoError(t, v.PruneHistory(60)) + requireStatus(t, v, 59, gigatypes.BlockHashStatusTooOld) + requireRecorded(t, v, 60, 60) + + require.NoError(t, v.PruneHistory(200)) + requireStatus(t, v, 99, gigatypes.BlockHashStatusTooOld) + requireRecorded(t, v, 100, 100) + + v.PruneBelow(1000) + require.NoError(t, v.PruneHistory(1000)) + requireStatus(t, v, 149, gigatypes.BlockHashStatusTooOld) + requireRecorded(t, v, 150, 150) +} + +func TestRollbackFloorIsTheHeadLessTheWindow(t *testing.T) { + v := newTestPebbleVault(t) + require.Equal(t, uint64(0), v.GetRollbackFloor(10)) + commitHeights(t, v, 1, 30) + require.Equal(t, uint64(20), v.GetRollbackFloor(10)) + require.Equal(t, uint64(0), v.GetRollbackFloor(40)) + latest, err := v.GetLatestBlock() + require.NoError(t, err) + require.Equal(t, uint64(30), latest) +} + +func TestStoredRange(t *testing.T) { + ctx := context.Background() + cfg := DefaultHashVaultConfig() + cfg.DataDir = filepath.Join(t.TempDir(), "vault") + + _, _, recorded, err := StoredRange(cfg) + require.NoError(t, err) + require.False(t, recorded, "a vault that was never created records nothing") + + v, err := NewUnsafePebbleHashVault(ctx, cfg) + require.NoError(t, err) + require.NoError(t, v.Close(ctx)) + _, _, recorded, err = StoredRange(cfg) + require.NoError(t, err) + require.False(t, recorded, "an empty vault records nothing") + + v, err = NewUnsafePebbleHashVault(ctx, cfg) + require.NoError(t, err) + commitHeights(t, v, 4, 9) + require.NoError(t, v.Close(ctx)) + oldest, newest, recorded, err := StoredRange(cfg) + require.NoError(t, err) + require.True(t, recorded) + require.Equal(t, uint64(4), oldest) + require.Equal(t, uint64(9), newest) +} + +func TestOpenDeletesTheLegacyVault(t *testing.T) { + legacy := filepath.Join(t.TempDir(), "hashvault") + require.NoError(t, os.MkdirAll(legacy, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(legacy, "000001.log"), []byte("x"), 0o600)) + + newTestPebbleVault(t, func(cfg *HashVaultConfig) { cfg.LegacyPebbleDir = legacy }) + _, err := os.Stat(legacy) + require.True(t, os.IsNotExist(err), "the legacy vault must be gone, got %v", err) + + newTestPebbleVault(t, func(cfg *HashVaultConfig) { cfg.LegacyPebbleDir = legacy }) +} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_codec.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_codec.go new file mode 100644 index 0000000000..139632d8d5 --- /dev/null +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault_codec.go @@ -0,0 +1,160 @@ +package hashvault + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "fmt" + "math" + "strconv" +) + +// Wire format +// +// Hash entries live under keys "h" , where is fixed-width 20-digit zero-padded +// ASCII decimal (twenty digits is the exact width of math.MaxUint64). Fixed width is what makes +// Pebble's lexicographic key order match numeric height order, which is what lets Prune wipe a +// contiguous range with a single DeleteRange. ASCII (vs binary BE) is chosen so a raw Pebble dump +// shows recognizable numbers (e.g. "h00000000000000000042") rather than opaque bytes. +// +// Each hash entry's value is the raw block hash followed by a 32-byte SHA-256 trailer computed +// over (be-uint64 height || hash). The trailer is what protects against the validator +// double-voting after silent corruption: the height is folded into the SHA so a stale entry +// returned from the wrong key path also fails verification. The trailer's height encoding is +// binary BE because that representation never appears on disk in raw form (it's consumed entirely +// by the SHA), so the human-readability argument doesn't apply there. +// +// The prune boundary lives under the single key "prune_boundary". Its value is the boundary +// height as variable-width unpadded ASCII decimal (e.g. "5" or "18446744073709551615"). No +// padding because there's only one such row and no range scan to satisfy. No checksum: the +// boundary is GC bookkeeping, a silent flip is not slashable, and Pebble's own block-level CRC +// catches bit-rot in normal operation. +// +// "h" and "prune_boundary" have disjoint first bytes ('h' vs 'p'), so the two namespaces can never +// alias regardless of what digits follow the hash prefix. + +const checksumSize = sha256.Size + +// heightDigits is the on-disk width (in ASCII bytes) of every encoded height. math.MaxUint64 is +// 18446744073709551615, exactly 20 digits. +const heightDigits = 20 + +var ( + hashKeyPrefix = []byte("h") + pruneBoundaryKey = []byte("prune_boundary") +) + +// hashKey returns the Pebble key for the given block height: hashKeyPrefix followed by the height +// as 20-digit zero-padded ASCII decimal. +func hashKey(height uint64) []byte { + out := make([]byte, 0, len(hashKeyPrefix)+heightDigits) + out = append(out, hashKeyPrefix...) + return appendHeight(out, height) +} + +// decodeHashKey is the inverse of hashKey: validates the length and prefix, then parses the +// trailing decimal digits. Returns ErrCorruption on any malformedness. +func decodeHashKey(key []byte) (uint64, error) { + if len(key) != len(hashKeyPrefix)+heightDigits { + return 0, fmt.Errorf("%w: unexpected hash key length %d", ErrCorruption, len(key)) + } + if !bytes.HasPrefix(key, hashKeyPrefix) { + return 0, fmt.Errorf("%w: hash key missing prefix", ErrCorruption) + } + return parseHeight(key[len(hashKeyPrefix):]) +} + +// hashKeyUpperBound returns an end-exclusive Pebble key that is strictly greater than every key +// hashKey can produce (i.e. up to and including hashKey(math.MaxUint64)). Safe to use as an +// IterOptions.UpperBound or as the upper end of a DeleteRange covering the entire hash namespace. +func hashKeyUpperBound() []byte { + // One byte longer than any valid hash key, so lex-greater than all of them. + return append(hashKey(math.MaxUint64), 0x00) +} + +// encodeHashValue returns the on-disk value for the given (height, hash) pair: the hash bytes +// followed by SHA-256(be(height) || hash). +func encodeHashValue(height uint64, hash []byte) []byte { + out := make([]byte, 0, len(hash)+checksumSize) + out = append(out, hash...) + out = append(out, hashChecksum(height, hash)...) + return out +} + +// decodeHashValue verifies the trailing SHA-256 of raw against (height, hash[:len(raw)-32]) and +// returns the hash bytes on success. Returns ErrCorruption if the trailer is missing or wrong. +func decodeHashValue(height uint64, raw []byte) ([]byte, error) { + if len(raw) < checksumSize { + return nil, fmt.Errorf("%w: value too short for height %d (%d bytes)", ErrCorruption, height, len(raw)) + } + split := len(raw) - checksumSize + hash := raw[:split] + trailer := raw[split:] + expected := hashChecksum(height, hash) + if !bytes.Equal(trailer, expected) { + return nil, fmt.Errorf("%w: checksum mismatch for height %d", ErrCorruption, height) + } + return bytes.Clone(hash), nil +} + +// encodeBoundaryValue returns the on-disk value for the prune boundary: variable-width unpadded +// ASCII decimal. There's only one boundary row in the DB and nothing range-scans the value, so +// fixed-width padding (as used for keys) buys nothing here. +func encodeBoundaryValue(boundary uint64) []byte { + return strconv.AppendUint(nil, boundary, 10) +} + +// decodeBoundaryValue parses an ASCII-decimal boundary value. Empty/oversized/non-digit inputs all +// trip ErrCorruption; Pebble's own CRC handles bit-rot within an otherwise-valid value. +func decodeBoundaryValue(raw []byte) (uint64, error) { + // math.MaxUint64 is 20 digits; anything longer can't be a valid uint64 and is suspect. + if len(raw) == 0 || len(raw) > heightDigits { + return 0, fmt.Errorf("%w: unexpected boundary value length %d", ErrCorruption, len(raw)) + } + n, err := strconv.ParseUint(string(raw), 10, 64) + if err != nil { + return 0, fmt.Errorf("%w: invalid boundary digits %q: %v", ErrCorruption, raw, err) + } + return n, nil +} + +// hashChecksum returns SHA-256(be(height) || hash). The height is encoded as binary BE here, not +// ASCII, because the result is hashed in place and never appears on disk in raw form. +func hashChecksum(height uint64, hash []byte) []byte { + h := sha256.New() + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], height) + _, _ = h.Write(buf[:]) + _, _ = h.Write(hash) + return h.Sum(nil) +} + +// appendHeight appends 20-digit zero-padded decimal to dst and returns the result. +func appendHeight(dst []byte, height uint64) []byte { + var buf [heightDigits]byte + i := len(buf) + for height > 0 { + i-- + buf[i] = byte('0' + height%10) + height /= 10 + } + for i > 0 { + i-- + buf[i] = '0' + } + return append(dst, buf[:]...) +} + +// parseHeight parses exactly heightDigits decimal bytes into a uint64. Returns ErrCorruption on +// any non-digit byte; uint64 cannot overflow because 20 digits is the exact width of math.MaxUint64 +// and ParseUint with bitSize=64 rejects values above MaxUint64. +func parseHeight(raw []byte) (uint64, error) { + if len(raw) != heightDigits { + return 0, fmt.Errorf("%w: expected %d digits, got %d", ErrCorruption, heightDigits, len(raw)) + } + n, err := strconv.ParseUint(string(raw), 10, 64) + if err != nil { + return 0, fmt.Errorf("%w: invalid height digits %q: %v", ErrCorruption, raw, err) + } + return n, nil +} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback.go new file mode 100644 index 0000000000..dc991fb6be --- /dev/null +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback.go @@ -0,0 +1,159 @@ +package hashvault + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "math" + "os" + + "github.com/cockroachdb/pebble/v2" +) + +// HardRollbackPebbleHashVault deletes every recorded hash strictly above blockHeight from the +// on-disk vault rooted at config.DataDir and clears the prune boundary. This is a break-glass +// operator tool: after it returns, commits at any height are allowed until Prune is run again. +func HardRollbackPebbleHashVault(_ context.Context, config HashVaultConfig, blockHeight uint64) error { + if err := config.Validate(); err != nil { + return fmt.Errorf("invalid hashvault config: %w", err) + } + + // Refuse if the data dir doesn't already exist. pebble.Open would otherwise silently create a + // fresh empty DB at a typo'd path and report a "successful" no-op rollback, which is exactly + // the kind of operator-error-eaten-by-tooling we want to avoid in a CLI tool. + if _, err := os.Stat(config.DataDir); err != nil { + return fmt.Errorf("hashvault data dir %q is not accessible: %w", config.DataDir, err) + } + + db, err := pebble.Open(config.DataDir, &pebble.Options{}) + if err != nil { + return fmt.Errorf("failed to open hashvault pebble db at %q: %w", config.DataDir, err) + } + defer func() { _ = db.Close() }() + + boundary, err := readPersistedBoundary(db) + if err != nil { + return err + } + + if blockHeight < boundary { + return wipeEntireStore(db, config.DataDir, blockHeight, boundary) + } + + // Partial rollback uses DeleteRange(hashKey(blockHeight+1), ...). At math.MaxUint64 the +1 + // wraps to 0, so hashKey(0) becomes the range start and every hash entry is deleted. + if blockHeight == math.MaxUint64 { + return fmt.Errorf("cannot hard rollback above block %d: %w", blockHeight, ErrRollbackHeightOverflow) + } + + return hardRollbackAbove(db, config.DataDir, blockHeight) +} + +// hardRollbackAbove deletes hashes strictly above blockHeight and clears the prune boundary in one +// atomic batch. +func hardRollbackAbove(db *pebble.DB, dataDir string, blockHeight uint64) error { + batch := db.NewBatch() + defer func() { _ = batch.Close() }() + if err := batch.DeleteRange(hashKey(blockHeight+1), hashKeyUpperBound(), nil); err != nil { + return fmt.Errorf("failed to stage hard rollback above block %d: %w", blockHeight, err) + } + if err := batch.Delete(pruneBoundaryKey, nil); err != nil { + return fmt.Errorf("failed to stage prune boundary clear during hard rollback: %w", err) + } + if err := batch.Commit(pebble.Sync); err != nil { + return fmt.Errorf("failed to hard rollback above block %d: %w", blockHeight, err) + } + logger.Info("hashvault hard rollback completed", + "dataDir", dataDir, "blockHeight", blockHeight) + return nil +} + +// readPersistedBoundary returns the on-disk prune boundary, or zero if none has ever been written. +// A malformed boundary record is logged and surfaced as ErrCorruption so the operator must +// investigate before proceeding. +func readPersistedBoundary(db *pebble.DB) (uint64, error) { + raw, closer, err := db.Get(pruneBoundaryKey) + if errors.Is(err, pebble.ErrNotFound) { + return 0, nil + } + if err != nil { + return 0, fmt.Errorf("failed to read prune boundary: %w", err) + } + defer func() { _ = closer.Close() }() + + boundary, err := decodeBoundaryValue(raw) + if err != nil { + logger.Error("hashvault prune boundary is malformed; refusing rollback", + "rawHex", hex.EncodeToString(raw), "err", err) + return 0, err + } + return boundary, nil +} + +// wipeEntireStore drops every hash entry and the prune boundary record in a single atomic Pebble +// batch, leaving the store indistinguishable from a freshly-initialized vault. +func wipeEntireStore(db *pebble.DB, dataDir string, target, boundary uint64) error { + batch := db.NewBatch() + defer func() { _ = batch.Close() }() + if err := batch.DeleteRange(hashKey(0), hashKeyUpperBound(), nil); err != nil { + return fmt.Errorf("failed to stage wipe of hash range: %w", err) + } + if err := batch.Delete(pruneBoundaryKey, nil); err != nil { + return fmt.Errorf("failed to stage wipe of prune boundary: %w", err) + } + if err := batch.Commit(pebble.Sync); err != nil { + return fmt.Errorf("failed to wipe hashvault store: %w", err) + } + logger.Warn("hashvault rollback target is below prune boundary; wiped entire store", + "dataDir", dataDir, "rollbackTarget", target, "pruneBoundary", boundary) + return nil +} + +// StoredRange returns the oldest and newest heights the closed vault under config.DataDir records, and +// false when it records none. A vault that has never been created records none. +func StoredRange(config HashVaultConfig) (oldest uint64, newest uint64, recorded bool, err error) { + if _, err := os.Stat(config.DataDir); err != nil { + if os.IsNotExist(err) { + return 0, 0, false, nil + } + return 0, 0, false, fmt.Errorf("hashvault data dir %q is not accessible: %w", config.DataDir, err) + } + db, err := pebble.Open(config.DataDir, &pebble.Options{ReadOnly: true}) + if err != nil { + return 0, 0, false, fmt.Errorf("failed to open hashvault pebble db at %q read-only: %w", + config.DataDir, err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("failed to close read-only hashvault: %w", closeErr)) + } + }() + return storedRange(db) +} + +// storedRange returns the oldest and newest heights db records, and false when it records none. +func storedRange(db *pebble.DB) (oldest uint64, newest uint64, recorded bool, err error) { + iter, err := db.NewIter(&pebble.IterOptions{LowerBound: hashKey(0), UpperBound: hashKeyUpperBound()}) + if err != nil { + return 0, 0, false, fmt.Errorf("failed to open hashvault iterator: %w", err) + } + defer func() { + if closeErr := iter.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("failed to close hashvault iterator: %w", closeErr)) + } + }() + if !iter.First() { + return 0, 0, false, nil + } + if oldest, err = decodeHashKey(iter.Key()); err != nil { + return 0, 0, false, fmt.Errorf("failed to decode the oldest hashvault key: %w", err) + } + if !iter.Last() { + return 0, 0, false, fmt.Errorf("hashvault iterator found an oldest key but no newest key") + } + if newest, err = decodeHashKey(iter.Key()); err != nil { + return 0, 0, false, fmt.Errorf("failed to decode the newest hashvault key: %w", err) + } + return oldest, newest, true, nil +} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go new file mode 100644 index 0000000000..10bdbd59e4 --- /dev/null +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go @@ -0,0 +1,144 @@ +package hashvault + +import ( + "context" + "math" + "path/filepath" + "testing" + + "github.com/cockroachdb/pebble/v2" + gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" + "github.com/stretchr/testify/require" +) + +// TestHardRollbackPebbleHashVault covers the happy path: an existing commit at height 10 is +// removed by rolling back to height 5; a fresh commit at 10 with a different hash then succeeds +// and is itself locked in. The vault must be closed before invoking the static function (Pebble's +// directory lock would otherwise refuse). +func TestHardRollbackPebbleHashVault(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + + a := bytesOfLen(0xAA, 32) + b := bytesOfLen(0xBB, 32) + require.NoError(t, v.CommitToHash(ctx, 10, a)) + + cfg := v.config + require.NoError(t, v.Close(ctx)) + + require.NoError(t, HardRollbackPebbleHashVault(ctx, cfg, 5)) + + v2, err := NewUnsafePebbleHashVault(ctx, cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = v2.Close(ctx) }) + + require.NoError(t, v2.CommitToHash(ctx, 10, b)) + require.ErrorIs(t, v2.CommitToHash(ctx, 10, a), ErrHashMismatch) +} + +func TestHardRollbackPebbleHashVaultBelowPruneBoundaryWipesStore(t *testing.T) { + // When the rollback target is strictly below the boundary, "partial rollback" is incoherent: + // every surviving hash has height >= boundary > target, so there is no consistent state to + // preserve. The function wipes everything (hashes + boundary record) so the next boot looks + // like a freshly-initialized vault. + ctx := context.Background() + v := newTestPebbleVault(t) + + for h := uint64(30); h <= 50; h++ { + require.NoError(t, v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32))) + } + require.NoError(t, v.Prune(ctx, 30)) + + cfg := v.config + require.NoError(t, v.Close(ctx)) + + require.NoError(t, HardRollbackPebbleHashVault(ctx, cfg, 10)) + + v2, err := NewUnsafePebbleHashVault(ctx, cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = v2.Close(ctx) }) + + // Boundary is gone, so commits below the old boundary are now accepted. + require.NoError(t, v2.CommitToHash(ctx, 5, bytesOfLen(0xCC, 32))) + // Every previously-locked hash is also gone: height 50 used to be 0x32, and nothing is recorded + // there now. + _, status, err := v2.Get(50) + require.NoError(t, err) + require.Equal(t, gigatypes.BlockHashStatusNotReady, status) +} + +func TestHardRollbackPebbleHashVaultEqualToPruneBoundary(t *testing.T) { + // Rollback target == boundary: hashes above the target are removed, the boundary block is kept, + // and the prune boundary record is cleared so commits below the old boundary are allowed again. + ctx := context.Background() + v := newTestPebbleVault(t) + require.NoError(t, v.Prune(ctx, 100)) + cfg := v.config + require.NoError(t, v.Close(ctx)) + + require.NoError(t, HardRollbackPebbleHashVault(ctx, cfg, 100)) + + v2, err := NewUnsafePebbleHashVault(ctx, cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = v2.Close(ctx) }) + require.NoError(t, v2.CommitToHash(ctx, 50, bytesOfLen(0xAA, 32))) +} + +func TestHardRollbackPebbleHashVaultRejectsLockedDir(t *testing.T) { + // Sanity check that the static function fails fast when a live vault still holds the Pebble + // directory lock — the whole point of being out-of-process is to make accidental concurrent + // use a clean error, not a silent corruption. + ctx := context.Background() + v := newTestPebbleVault(t) + cfg := v.config + + err := HardRollbackPebbleHashVault(ctx, cfg, 5) + require.Error(t, err, "must refuse to open while the live vault holds the lock") +} + +func TestHardRollbackPebbleHashVaultRefusesMalformedBoundary(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + cfg := v.config + require.NoError(t, v.Close(ctx)) + + // Plant a malformed boundary record so the function has to reject before performing any write. + directWritePebble(t, cfg.DataDir, func(db *pebble.DB) { + require.NoError(t, db.Set(pruneBoundaryKey, []byte{0x00, 0x01}, pebble.Sync)) + }) + + err := HardRollbackPebbleHashVault(ctx, cfg, 100) + require.ErrorIs(t, err, ErrCorruption) +} + +func TestHardRollbackPebbleHashVaultRejectsMaxUint64Height(t *testing.T) { + // blockHeight+1 must not be used for DeleteRange start keys: at MaxUint64 it wraps to 0 and + // would wipe the entire vault. Refuse rather than silently destroy data. + ctx := context.Background() + v := newTestPebbleVault(t) + require.NoError(t, v.CommitToHash(ctx, math.MaxUint64, bytesOfLen(0xFF, 32))) + + cfg := v.config + require.NoError(t, v.Close(ctx)) + + err := HardRollbackPebbleHashVault(ctx, cfg, math.MaxUint64) + require.ErrorIs(t, err, ErrRollbackHeightOverflow) + + v2, err := NewUnsafePebbleHashVault(ctx, cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = v2.Close(ctx) }) + + require.ErrorIs(t, v2.CommitToHash(ctx, math.MaxUint64, bytesOfLen(0xEE, 32)), ErrHashMismatch) +} + +func TestHardRollbackPebbleHashVaultRejectsMissingDir(t *testing.T) { + ctx := context.Background() + cfg := DefaultHashVaultConfig() + // Point at a path that definitely doesn't exist; pebble.Open is the source of truth for the + // error here — we just want to verify the static function surfaces it rather than silently + // creating a fresh empty vault and pretending the rollback succeeded. + cfg.DataDir = filepath.Join(t.TempDir(), "does-not-exist", "vault") + + err := HardRollbackPebbleHashVault(ctx, cfg, 5) + require.Error(t, err) +} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_test.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_test.go new file mode 100644 index 0000000000..d19ef80d78 --- /dev/null +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault_test.go @@ -0,0 +1,338 @@ +package hashvault + +import ( + "bytes" + "context" + "path/filepath" + "sync" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/stretchr/testify/require" +) + +// newTestPebbleVault constructs an unsafe Pebble vault rooted in t.TempDir() and arranges for it +// to be closed at end-of-test. +func newTestPebbleVault(t *testing.T, configMutators ...func(*HashVaultConfig)) *PebbleHashVault { + t.Helper() + cfg := DefaultHashVaultConfig() + cfg.DataDir = filepath.Join(t.TempDir(), "vault") + for _, m := range configMutators { + m(&cfg) + } + v, err := NewUnsafePebbleHashVault(context.Background(), cfg) + require.NoError(t, err) + t.Cleanup(func() { + _ = v.Close(context.Background()) + }) + return v +} + +// reopenTestPebbleVault closes v then reopens a fresh PebbleHashVault at the same DataDir. The +// returned vault is cleaned up at end-of-test. +func reopenTestPebbleVault(t *testing.T, v *PebbleHashVault) *PebbleHashVault { + t.Helper() + dir := v.config.DataDir + require.NoError(t, v.Close(context.Background())) + cfg := DefaultHashVaultConfig() + cfg.DataDir = dir + reopened, err := NewUnsafePebbleHashVault(context.Background(), cfg) + require.NoError(t, err) + t.Cleanup(func() { + _ = reopened.Close(context.Background()) + }) + return reopened +} + +// directWritePebble opens the Pebble dir at path, applies fn to the db, then closes. Used by +// corruption tests to poke at on-disk values without going through the HashVault. +func directWritePebble(t *testing.T, path string, fn func(*pebble.DB)) { + t.Helper() + db, err := pebble.Open(path, &pebble.Options{}) + require.NoError(t, err) + defer func() { require.NoError(t, db.Close()) }() + fn(db) +} + +func TestRestartRecoversPruneBoundary(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + + require.NoError(t, v.Prune(ctx, 50)) + v2 := reopenTestPebbleVault(t, v) + + // Strictly below the recovered boundary is rejected. + require.ErrorIs(t, v2.CommitToHash(ctx, 25, bytesOfLen(0xAA, 32)), ErrBelowPruneBoundary) + require.ErrorIs(t, v2.CommitToHash(ctx, 49, bytesOfLen(0xAA, 32)), ErrBelowPruneBoundary) + // At and above the boundary are allowed. + require.NoError(t, v2.CommitToHash(ctx, 50, bytesOfLen(0xAA, 32))) + require.NoError(t, v2.CommitToHash(ctx, 51, bytesOfLen(0xAA, 32))) +} + +func TestRestartRecoversHashes(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + + hash := bytesOfLen(0xCD, 32) + require.NoError(t, v.CommitToHash(ctx, 99, hash)) + + v2 := reopenTestPebbleVault(t, v) + + // Re-commit the same hash: succeeds. + require.NoError(t, v2.CommitToHash(ctx, 99, hash)) + // Different hash: locked out. + require.ErrorIs(t, + v2.CommitToHash(ctx, 99, bytesOfLen(0xFF, 32)), + ErrHashMismatch, + ) +} + +func TestPruneRemovesDataOnDisk(t *testing.T) { + // The shared suite already covers the externally-visible Prune contract; this test exists to + // pin the on-disk effect — i.e. that the deleted heights are actually gone from Pebble (and + // not merely shadowed by the in-memory boundary check). + ctx := context.Background() + const total = uint64(50) + + v := newTestPebbleVault(t) + for h := uint64(1); h <= total; h++ { + require.NoError(t, v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32))) + } + + require.NoError(t, v.Prune(ctx, total)) + + dir := v.config.DataDir + require.NoError(t, v.Close(ctx)) + + var remaining []uint64 + directWritePebble(t, dir, func(db *pebble.DB) { + iter, err := db.NewIter(&pebble.IterOptions{ + LowerBound: hashKey(0), + UpperBound: hashKeyUpperBound(), + }) + require.NoError(t, err) + defer func() { _ = iter.Close() }() + for iter.First(); iter.Valid(); iter.Next() { + h, err := decodeHashKey(iter.Key()) + require.NoError(t, err) + remaining = append(remaining, h) + } + }) + // Per the Prune contract the boundary block itself is kept, so only height==total should remain. + require.Equal(t, []uint64{total}, remaining, + "only the prune-boundary block should remain after Prune") +} + +func TestKeyEncodingRoundtrip(t *testing.T) { + cases := []uint64{0, 1, 7, 1 << 30, 1<<63 - 1, ^uint64(0)} + for _, h := range cases { + k := hashKey(h) + require.Len(t, k, len(hashKeyPrefix)+heightDigits) + require.True(t, bytes.HasPrefix(k, hashKeyPrefix)) + got, err := decodeHashKey(k) + require.NoError(t, err) + require.Equal(t, h, got) + } +} + +func TestKeyEncodingOrderingMatchesNumeric(t *testing.T) { + // Spot-check that lex order over hashKey() matches numeric order, including non-adjacent + // magnitudes. This is the whole reason for zero-padded fixed-width encoding. + heights := []uint64{0, 1, 9, 10, 99, 100, 1<<32 - 1, 1 << 32, 1<<63 - 1, ^uint64(0) - 1, ^uint64(0)} + for i := 0; i+1 < len(heights); i++ { + a, b := hashKey(heights[i]), hashKey(heights[i+1]) + require.Lessf(t, bytes.Compare(a, b), 0, + "hashKey(%d)=%q must sort before hashKey(%d)=%q", heights[i], a, heights[i+1], b) + } +} + +func TestKeyEncodingHumanReadable(t *testing.T) { + // Pin the on-disk layout so a future "let's switch back to binary BE for size" PR has to + // explicitly delete this test. + require.Equal(t, "h00000000000000000042", string(hashKey(42))) + require.Equal(t, "h18446744073709551615", string(hashKey(^uint64(0)))) +} + +func TestDecodeHashKeyRejectsMalformed(t *testing.T) { + _, err := decodeHashKey([]byte("h0000000000000000004")) + require.ErrorIs(t, err, ErrCorruption, "short length") + _, err = decodeHashKey([]byte("x00000000000000000042")) + require.ErrorIs(t, err, ErrCorruption, "wrong prefix") + _, err = decodeHashKey([]byte("h0000000000000000004x")) + require.ErrorIs(t, err, ErrCorruption, "non-digit byte") +} + +func TestValueCodecRoundtrip(t *testing.T) { + hash := bytesOfLen(0xAA, 32) + for _, h := range []uint64{0, 1, 7, 1 << 30, ^uint64(0)} { + raw := encodeHashValue(h, hash) + got, err := decodeHashValue(h, raw) + require.NoError(t, err) + require.Equal(t, hash, got) + } + + for _, b := range []uint64{0, 1, 1234567890, ^uint64(0)} { + raw := encodeBoundaryValue(b) + got, err := decodeBoundaryValue(raw) + require.NoError(t, err) + require.Equal(t, b, got) + } +} + +func TestBoundaryValueIsUnpaddedDecimal(t *testing.T) { + // Boundary encoding is variable-width by design (one row, no range scan to satisfy). Pin it + // so a future "let's pad for symmetry with keys" change has to explicitly delete this test. + require.Equal(t, "0", string(encodeBoundaryValue(0))) + require.Equal(t, "42", string(encodeBoundaryValue(42))) + require.Equal(t, "18446744073709551615", string(encodeBoundaryValue(^uint64(0)))) +} + +func TestDecodeBoundaryValueRejectsMalformed(t *testing.T) { + _, err := decodeBoundaryValue(nil) + require.ErrorIs(t, err, ErrCorruption, "empty") + _, err = decodeBoundaryValue([]byte{}) + require.ErrorIs(t, err, ErrCorruption, "zero length") + _, err = decodeBoundaryValue([]byte("123abc")) + require.ErrorIs(t, err, ErrCorruption, "non-digit byte") + // 21 digits cannot fit in uint64. + _, err = decodeBoundaryValue([]byte("184467440737095516150")) + require.ErrorIs(t, err, ErrCorruption, "too long") +} + +func TestValueCodecHeightTamper(t *testing.T) { + // The whole reason we feed the height into the SHA is to detect a value that was stored under + // a different key from the one we're now reading. Decoding under the wrong height MUST fail. + hash := bytesOfLen(0x77, 32) + raw := encodeHashValue(100, hash) + _, err := decodeHashValue(101, raw) + require.ErrorIs(t, err, ErrCorruption) +} + +func TestValueCodecBitFlip(t *testing.T) { + hash := bytesOfLen(0x77, 32) + raw := encodeHashValue(100, hash) + + // Flip one bit in every byte position and confirm each flip is caught. + for i := 0; i < len(raw); i++ { + corrupted := bytes.Clone(raw) + corrupted[i] ^= 0x01 + _, err := decodeHashValue(100, corrupted) + require.ErrorIsf(t, err, ErrCorruption, "expected ErrCorruption at byte %d", i) + } +} + +func TestValueCodecShortValue(t *testing.T) { + // A value shorter than the trailer can't carry a valid checksum. + for n := 0; n < checksumSize; n++ { + _, err := decodeHashValue(0, make([]byte, n)) + require.ErrorIs(t, err, ErrCorruption) + } +} + +func TestCommitDetectsDiskCorruption(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + hash := bytesOfLen(0xAA, 32) + require.NoError(t, v.CommitToHash(ctx, 42, hash)) + + dir := v.config.DataDir + require.NoError(t, v.Close(ctx)) + + // Flip one bit of the stored value via direct Pebble access. + directWritePebble(t, dir, func(db *pebble.DB) { + key := hashKey(42) + raw, closer, err := db.Get(key) + require.NoError(t, err) + corrupted := bytes.Clone(raw) + _ = closer.Close() + corrupted[0] ^= 0x01 + require.NoError(t, db.Set(key, corrupted, pebble.Sync)) + }) + + reopenCfg := DefaultHashVaultConfig() + reopenCfg.DataDir = dir + v2, err := NewUnsafePebbleHashVault(ctx, reopenCfg) + require.NoError(t, err) + t.Cleanup(func() { _ = v2.Close(ctx) }) + + err = v2.CommitToHash(ctx, 42, hash) + require.ErrorIs(t, err, ErrCorruption) +} + +func TestStartupRejectsMalformedBoundary(t *testing.T) { + // The boundary value has no checksum (it's just GC bookkeeping), so a flipped byte is + // indistinguishable from a legitimately-written boundary and is accepted silently. The only + // failure mode startup still catches is a length mismatch, which indicates the record was + // truncated/extended outside Pebble's normal write path. + ctx := context.Background() + v := newTestPebbleVault(t) + require.NoError(t, v.Prune(ctx, 7)) + + dir := v.config.DataDir + require.NoError(t, v.Close(ctx)) + + directWritePebble(t, dir, func(db *pebble.DB) { + require.NoError(t, db.Set(pruneBoundaryKey, []byte{0x00, 0x01, 0x02}, pebble.Sync)) + }) + + reopenCfg := DefaultHashVaultConfig() + reopenCfg.DataDir = dir + _, err := NewUnsafePebbleHashVault(ctx, reopenCfg) + require.Error(t, err) + require.ErrorIs(t, err, ErrCorruption) +} + +func TestProductionConstructorForcesFsync(t *testing.T) { + ctx := context.Background() + // Caller asks for no fsync, but the production constructor must override that. + cfg := DefaultHashVaultConfig() + cfg.DataDir = filepath.Join(t.TempDir(), "vault") + cfg.Fsync = false + v, err := NewPebbleHashVault(ctx, cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = v.Close(ctx) }) + + require.True(t, v.config.Fsync, "NewPebbleHashVault must force Fsync=true") + require.Equal(t, pebble.Sync, v.writeOpts, "writeOpts must be pebble.Sync in production") +} + +func TestUnsafeConstructorHonorsFsync(t *testing.T) { + ctx := context.Background() + cfg := DefaultHashVaultConfig() + cfg.DataDir = filepath.Join(t.TempDir(), "vault") + cfg.Fsync = false + v, err := NewUnsafePebbleHashVault(ctx, cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = v.Close(ctx) }) + + require.False(t, v.config.Fsync) + require.Equal(t, pebble.NoSync, v.writeOpts) +} + +func TestContextCancelledCommit(t *testing.T) { + // A pre-cancelled ctx must short-circuit before we touch any state. We don't otherwise check + // the ctx mid-operation (the work is all local, fast, and uninterruptible once started). + ctx, cancel := context.WithCancel(context.Background()) + cancel() + v := newTestPebbleVault(t) + err := v.CommitToHash(ctx, 1, bytesOfLen(0xAA, 32)) + require.ErrorIs(t, err, context.Canceled) +} + +// Cross-check: nothing in the production code accidentally panics on simultaneous Close+Commit. +func TestCloseConcurrentWithCommits(t *testing.T) { + ctx := context.Background() + v := newTestPebbleVault(t) + + var wg sync.WaitGroup + for i := 0; i < 32; i++ { + wg.Add(1) + go func(h uint64) { + defer wg.Done() + _ = v.CommitToHash(ctx, h, bytesOfLen(byte(h), 32)) + }(uint64(i + 1)) + } + // Race Close against the commits. + _ = v.Close(ctx) + wg.Wait() +} diff --git a/sei-db/state_db/sc/hashvault/prune.go b/sei-db/state_db/sc/hashvault/prune.go deleted file mode 100644 index 5a88cdb917..0000000000 --- a/sei-db/state_db/sc/hashvault/prune.go +++ /dev/null @@ -1,75 +0,0 @@ -package hashvault - -import ( - "fmt" - "sync/atomic" - - "github.com/sei-protocol/sei-chain/sei-db/controller" -) - -var _ controller.PrunableStore = (*HashVault)(nil) - -// PruneBelow permits the hashes of blocks below blockNumber to be deleted, as far as the vault's owner is -// concerned. A hash is deleted only once PruneHistory() has permitted it too. -func (v *HashVault) PruneBelow(blockNumber uint64) { - raiseFloor(&v.outerFloor, blockNumber) -} - -// gcFilter reports whether LittDB may delete key: only a block below both floors may go. -func (v *HashVault) gcFilter(key []byte, _ bool) (bool, error) { - blockNumber, err := decodeKey(key) - if err != nil { - return false, fmt.Errorf("decode a hash vault key: %w", err) - } - return blockNumber < min(v.outerFloor.Load(), v.gcFloor.Load()), nil -} - -// raiseFloor raises floor to blockNumber, leaving it where it is when it is already higher. -func raiseFloor(floor *atomic.Uint64, blockNumber uint64) { - for { - current := floor.Load() - if blockNumber <= current || floor.CompareAndSwap(current, blockNumber) { - return - } - } -} - -// Name implements controller.PrunableStore. -func (v *HashVault) Name() string { - return "HashVault" -} - -// PruneHistory implements controller.PrunableStore. It permits the hashes of blocks below blockNumber to be -// deleted, as far as the storage garbage collector is concerned. A hash is deleted only once PruneBelow() -// has permitted it too. -func (v *HashVault) PruneHistory(blockNumber uint64) error { - raiseFloor(&v.gcFloor, blockNumber) - return nil -} - -// PruneSnapshots implements controller.PrunableStore. The vault keeps no snapshots. -func (v *HashVault) PruneSnapshots(uint64) error { - return nil -} - -// ExternalPruning implements controller.PrunableStore. The vault has no pruner of its own. -func (v *HashVault) ExternalPruning() bool { - return true -} - -// GetRollbackFloor implements controller.PrunableStore. Every recorded block is readable directly, so the -// floor is the newest recorded block less rollbackWindow, or 0 when the window is deeper than that. -func (v *HashVault) GetRollbackFloor(rollbackWindow uint64) uint64 { - head, ok := v.Head() - if !ok || head < rollbackWindow { - return 0 - } - return head - rollbackWindow -} - -// GetLatestBlock implements controller.PrunableStore. It returns the newest recorded block, or 0 when the -// vault holds no hashes. -func (v *HashVault) GetLatestBlock() (uint64, error) { - head, _ := v.Head() - return head, nil -} diff --git a/sei-db/state_db/sc/hashvault/record.go b/sei-db/state_db/sc/hashvault/record.go deleted file mode 100644 index f39692bc4a..0000000000 --- a/sei-db/state_db/sc/hashvault/record.go +++ /dev/null @@ -1,55 +0,0 @@ -package hashvault - -import ( - "encoding/binary" - "fmt" -) - -// recordFormatVersion is the format version every value this package writes starts with. -const recordFormatVersion byte = 1 - -// hashSize is the length of a recorded block hash. -const hashSize = 32 - -// keySize is the length of a key: a block number, big-endian. -const keySize = 8 - -// valueSize is the length of a value: the format version, then the hash. -const valueSize = 1 + hashSize - -// encodeKey returns the key a block's hash is recorded under. -func encodeKey(blockNumber uint64) []byte { - key := make([]byte, keySize) - binary.BigEndian.PutUint64(key, blockNumber) - return key -} - -// decodeKey returns the block number a key records. -func decodeKey(key []byte) (uint64, error) { - if len(key) != keySize { - return 0, fmt.Errorf("hash vault key is %d bytes, expected %d", len(key), keySize) - } - return binary.BigEndian.Uint64(key), nil -} - -// encodeValue returns the value a hash is recorded as. -func encodeValue(hash [32]byte) []byte { - value := make([]byte, valueSize) - value[0] = recordFormatVersion - copy(value[1:], hash[:]) - return value -} - -// decodeValue returns the hash a value records. -func decodeValue(value []byte) ([32]byte, error) { - var hash [32]byte - if len(value) != valueSize { - return hash, fmt.Errorf("hash vault value is %d bytes, expected %d", len(value), valueSize) - } - if value[0] != recordFormatVersion { - return hash, fmt.Errorf("hash vault value has format version %d, expected %d", - value[0], recordFormatVersion) - } - copy(hash[:], value[1:]) - return hash, nil -} diff --git a/sei-tendermint/config/config.go b/sei-tendermint/config/config.go index d589f77899..dad32af7ee 100644 --- a/sei-tendermint/config/config.go +++ b/sei-tendermint/config/config.go @@ -12,7 +12,7 @@ import ( "time" "github.com/sei-protocol/sei-chain/ratelimiter" - seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashvault" mempoolcfg "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool" tmos "github.com/sei-protocol/sei-chain/sei-tendermint/libs/os" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" @@ -111,8 +111,8 @@ func DefaultConfig() *Config { Instrumentation: DefaultInstrumentationConfig(), PrivValidator: DefaultPrivValidatorConfig(), SelfRemediation: DefaultSelfRemediationConfig(), - HashVaultHaltOnMismatch: seidbconfig.DefaultHashVaultConfig().HaltOnMismatch, - HashVaultEmptyRollbackBlocks: seidbconfig.DefaultHashVaultConfig().EmptyVaultRollbackBlocks, + HashVaultHaltOnMismatch: hashvault.DefaultHashVaultConfig().HaltOnMismatch, + HashVaultEmptyRollbackBlocks: hashvault.DefaultHashVaultConfig().EmptyVaultRollbackBlocks, } } From 812e87880f908d72b625359d8dbb664e8b59efc7 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Fri, 25 Sep 2026 09:35:03 -0500 Subject: [PATCH 3/5] handle some edge cases --- sei-db/common/utils/path.go | 5 -- sei-db/config/giga_config.go | 6 +- sei-db/state_db/giga/state_db.go | 3 + .../state_db/giga/state_db_hash_vault_test.go | 51 ++++++++++++++++ sei-db/state_db/giga/state_db_replay.go | 38 ++++++++++++ .../state_db/sc/hashvault/hashvault_config.go | 4 -- .../state_db/sc/hashvault/pebble_hashvault.go | 60 ++++++------------- .../hashvault/pebble_hashvault_branch_test.go | 13 ---- .../pebble_hashvault_rollback_test.go | 6 -- sei-tendermint/node/setup.go | 2 - 10 files changed, 113 insertions(+), 75 deletions(-) diff --git a/sei-db/common/utils/path.go b/sei-db/common/utils/path.go index 28e951cfb7..6edb519c67 100644 --- a/sei-db/common/utils/path.go +++ b/sei-db/common/utils/path.go @@ -43,11 +43,6 @@ func GetFlatKVPath(homePath string) string { return filepath.Join(homePath, "data", "state_commit", "flatkv") } -// GetHashVaultPath returns the path for the hash vault that guards the live state DB's block hashes. -func GetHashVaultPath(homePath string) string { - return filepath.Join(homePath, "data", "state_commit", "hashvault") -} - // GetStateStorePath returns the path for the Cosmos state store (SS). // New nodes use data/state_store/cosmos/{backend}; existing nodes with // data/{backend} continue using the legacy path for backward compatibility. diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index 716696a9f9..133399f124 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "path/filepath" "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/ledger_db/block/littblock" @@ -28,7 +29,6 @@ const gigaReceiptBackend = "littidx" // DefaultGigaStorageConfig returns a config rooted at homePath: // // data/state_commit/flatkv -// data/state_commit/hashvault // data/state_store/evm/{backend} // data/ledger/receipt/{backend} // data/ledger/block @@ -50,7 +50,9 @@ func DefaultGigaStorageConfig(homePath string) (*GigaStorageConfig, error) { ssConfig.DisableInternalWAL = true hashVaultConfig := hashvault.DefaultHashVaultConfig() - hashVaultConfig.DataDir = utils.GetHashVaultPath(homePath) + // Existing Autobahn nodes keep their hash vault here, under the persistent state dir. Moving it loses + // the hashes they have recorded. + hashVaultConfig.DataDir = filepath.Join(homePath, "hashvault") receiptConfig := DefaultReceiptStoreConfig() receiptConfig.Backend = gigaReceiptBackend diff --git a/sei-db/state_db/giga/state_db.go b/sei-db/state_db/giga/state_db.go index 767b17ef65..926194d950 100644 --- a/sei-db/state_db/giga/state_db.go +++ b/sei-db/state_db/giga/state_db.go @@ -125,6 +125,9 @@ func NewStateDB( } } + if err := requireAgreementWithoutWAL(sc, ss, vault, wal); err != nil { + return nil, fmt.Errorf("open the state DB on an empty state WAL: %w", err) + } if err := recordLoadedBlockHash(sc, vault); err != nil { return nil, fmt.Errorf("record the loaded block's hash in the hash vault: %w", err) } diff --git a/sei-db/state_db/giga/state_db_hash_vault_test.go b/sei-db/state_db/giga/state_db_hash_vault_test.go index b54f15288d..2e2aaeaaf8 100644 --- a/sei-db/state_db/giga/state_db_hash_vault_test.go +++ b/sei-db/state_db/giga/state_db_hash_vault_test.go @@ -14,9 +14,11 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/controller" gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashvault" + "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" ) // vaultTestStores is the set of configs a StateDB under test opens, kept so the test can reopen it. @@ -319,3 +321,52 @@ func TestTheVaultJoinsThePruneCycle(t *testing.T) { } require.Contains(t, names, "HashVault") } + +// emptyWAL drops every block from the closed StateDB's WAL, leaving the stores where they are. +func emptyWAL(t *testing.T, stores *vaultTestStores) { + t.Helper() + require.NoError(t, statewal.PruneAfter(flatkv.StateWALConfig(stores.flatkvCfg.DataDir), 0)) +} + +// With no WAL blocks nothing can replay, so an open over stores that already agree is the only one allowed. +func TestAnEmptyWALOpensWhenTheStoresAgree(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 3) + require.NoError(t, db.Close()) + emptyWAL(t, stores) + + reopened := stores.open(t) + defer func() { require.NoError(t, reopened.Close()) }() + require.Equal(t, uint64(3), reopened.GetBlockHeight()) +} + +// An empty WAL cannot bring SS up to SC, so a difference between them is refused, even for an SS that holds +// nothing yet. +func TestAnEmptyWALRefusesStoresOnDifferentHeights(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 3) + require.NoError(t, db.Close()) + emptyWAL(t, stores) + + stores.ssCfg = config.DefaultStateStoreConfig() + stores.ssCfg.Enable = true + stores.ssCfg.EVMDBDirectory = filepath.Join(t.TempDir(), "ss") + + _, err := stores.openErr() + require.ErrorContains(t, err, "the EVM state store is on block 0") +} + +// An empty WAL cannot replay the loaded block, so a vault without its hash is refused. +func TestAnEmptyWALRefusesAVaultWithoutTheLoadedBlock(t *testing.T) { + stores := newVaultTestStores(t) + db := stores.open(t) + commitBlocks(t, db, 1, 3) + require.NoError(t, db.Close()) + emptyWAL(t, stores) + require.NoError(t, os.RemoveAll(stores.hashVaultCfg.DataDir)) + + _, err := stores.openErr() + require.ErrorContains(t, err, "the hash vault holds no hash for block 3") +} diff --git a/sei-db/state_db/giga/state_db_replay.go b/sei-db/state_db/giga/state_db_replay.go index a9969f11ff..9a02eadfbb 100644 --- a/sei-db/state_db/giga/state_db_replay.go +++ b/sei-db/state_db/giga/state_db_replay.go @@ -8,8 +8,10 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/proto" + gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashvault" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/evm" "github.com/sei-protocol/sei-chain/sei-db/state_db/statewal" ) @@ -334,3 +336,39 @@ func matchHeight(sc *flatkv.CommitStore, ss *evm.EVMStateStore, wal statewal.Sta } return fmt.Errorf("the EVM state store landed on %d", got) } + +// requireAgreementWithoutWAL refuses a WAL that holds no blocks unless SC, SS and the hash vault already +// agree on the block SC is on, since no replay can bring them together. ss is nil when SS is disabled. +func requireAgreementWithoutWAL( + sc *flatkv.CommitStore, + ss *evm.EVMStateStore, + vault *hashvault.PebbleHashVault, + wal statewal.StateWAL, +) error { + stored, _, _, err := wal.GetStoredRange() + if err != nil { + return fmt.Errorf("read the state WAL's range: %w", err) + } + if stored { + return nil + } + height := sc.Version() + if height == 0 { + return nil + } + if ss != nil { + if got := ss.GetLatestVersion(); got != height { + return fmt.Errorf("the state commit store is on block %d but the EVM state store is on "+ + "block %d", height, got) + } + } + _, status, err := vault.Get(uint64(height)) //nolint:gosec // a committed version is never negative + if err != nil { + return fmt.Errorf("read the hash vault at block %d: %w", height, err) + } + if status != gigatypes.BlockHashStatusFound { + return fmt.Errorf("the hash vault holds no hash for block %d, the block the state commit store is on", + height) + } + return nil +} diff --git a/sei-db/state_db/sc/hashvault/hashvault_config.go b/sei-db/state_db/sc/hashvault/hashvault_config.go index 25432211f7..fea75b00ce 100644 --- a/sei-db/state_db/sc/hashvault/hashvault_config.go +++ b/sei-db/state_db/sc/hashvault/hashvault_config.go @@ -27,10 +27,6 @@ type HashVaultConfig struct { // EmptyVaultRollbackBlocks is how many blocks the state DB rewinds and replays when it opens over an // empty vault, so that the vault holds the hashes of recent blocks and not just the loaded one. EmptyVaultRollbackBlocks uint64 - - // LegacyPebbleDir is the directory of the app-hash vault this one replaces, deleted when the vault - // opens. Empty when there is none to delete. - LegacyPebbleDir string } // DefaultHashVaultConfig returns a HashVaultConfig with production defaults. diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault.go b/sei-db/state_db/sc/hashvault/pebble_hashvault.go index e0448c9557..4ded427cd1 100644 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault.go +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault.go @@ -34,10 +34,10 @@ type PebbleHashVault struct { pruneBoundary uint64 cache *lru.Cache[uint64, []byte] - // head is the newest recorded height. Meaningless when recorded is false. + // head is the newest recorded height. Meaningless when notEmpty is false. head uint64 - // recorded is true when the vault holds at least one hash. - recorded bool + // notEmpty is true when the vault holds at least one hash. + notEmpty bool // outerFloor is the floor PruneBelow has raised. Only ever rises. outerFloor atomic.Uint64 @@ -67,10 +67,6 @@ func newPebbleHashVault(_ context.Context, config HashVaultConfig) (*PebbleHashV return nil, fmt.Errorf("invalid hashvault config: %w", err) } - if err := deleteLegacyVault(config.LegacyPebbleDir); err != nil { - return nil, fmt.Errorf("failed to open hashvault: %w", err) - } - if err := os.MkdirAll(config.DataDir, 0o750); err != nil { return nil, fmt.Errorf("failed to create hashvault data dir %q: %w", config.DataDir, err) } @@ -166,14 +162,14 @@ func (p *PebbleHashVault) CommitToHash(ctx context.Context, blockHeight uint64, if len(hash) != BlockHashSize { return ErrInvalidHashLength } - if p.recorded && blockHeight > p.head && blockHeight-p.head > 1 { + if p.notEmpty && blockHeight > p.head+1 { return fmt.Errorf("block %d would leave a gap after the newest recorded block %d", blockHeight, p.head) } if cached, ok := p.cache.Get(blockHeight); ok { if !bytes.Equal(cached, hash) { if !p.config.HaltOnMismatch { - return p.replaceFrom(blockHeight, cached, hash) + return p.acceptMismatchedHash(blockHeight, cached, hash) } p.logHashMismatch(blockHeight, cached, hash) return ErrHashMismatch @@ -185,10 +181,10 @@ func (p *PebbleHashVault) CommitToHash(ctx context.Context, blockHeight uint64, raw, closer, err := p.db.Get(key) switch { case errors.Is(err, pebble.ErrNotFound): - if p.recorded && blockHeight <= p.head { + if p.notEmpty && blockHeight <= p.head { // Below the oldest recorded height, where there is nothing to check the hash against. if !p.config.HaltOnMismatch { - return p.replaceFrom(blockHeight, nil, hash) + return p.acceptMismatchedHash(blockHeight, nil, hash) } return ErrBelowPruneBoundary } @@ -199,7 +195,7 @@ func (p *PebbleHashVault) CommitToHash(ctx context.Context, blockHeight uint64, } p.cache.Add(blockHeight, bytes.Clone(hash)) p.head = max(p.head, blockHeight) - p.recorded = true + p.notEmpty = true return nil case err != nil: return fmt.Errorf("failed to read hash for block %d: %w", blockHeight, err) @@ -217,7 +213,7 @@ func (p *PebbleHashVault) CommitToHash(ctx context.Context, blockHeight uint64, } if !bytes.Equal(existing, hash) { if !p.config.HaltOnMismatch { - return p.replaceFrom(blockHeight, existing, hash) + return p.acceptMismatchedHash(blockHeight, existing, hash) } p.logHashMismatch(blockHeight, existing, hash) return ErrHashMismatch @@ -292,28 +288,6 @@ func (p *PebbleHashVault) logHashMismatch(blockHeight uint64, existing, incoming ) } -// deleteLegacyVault deletes the app-hash vault this one replaces, if it is present. Its hashes are app -// hashes rather than state hashes, so none of them can be carried over. -// -// This can be deleted once every node that ran the app-hash vault has started on this one. -func deleteLegacyVault(dir string) error { - if dir == "" { - return nil - } - if _, err := os.Stat(dir); err != nil { - if os.IsNotExist(err) { - return nil - } - return fmt.Errorf("failed to stat legacy hashvault dir %q: %w", dir, err) - } - if err := os.RemoveAll(dir); err != nil { - return fmt.Errorf("failed to delete legacy hashvault dir %q: %w", dir, err) - } - logger.Info("deleted the legacy app-hash hashvault; its app hashes cannot be compared with state hashes", - "dir", dir) - return nil -} - // loadHead reads the newest recorded height from disk and populates p.head and p.recorded. func (p *PebbleHashVault) loadHead() error { _, head, recorded, err := storedRange(p.db) @@ -321,13 +295,13 @@ func (p *PebbleHashVault) loadHead() error { return fmt.Errorf("failed to read the newest recorded height: %w", err) } p.head = head - p.recorded = recorded + p.notEmpty = recorded return nil } -// replaceFrom discards every hash from blockHeight up and records hash as blockHeight's, in one atomic -// batch. It is how a mismatch is resolved when HaltOnMismatch is false. p.mu must be held. -func (p *PebbleHashVault) replaceFrom(blockHeight uint64, existing []byte, hash []byte) error { +// Records a hash that differs from the recorded one, discarding every hash from blockHeight up in the same +// atomic batch. Used when HaltOnMismatch is false. p.mu must be held. +func (p *PebbleHashVault) acceptMismatchedHash(blockHeight uint64, existing []byte, hash []byte) error { logger.Error("Hashvault detected a state hash mismatch; hash-vault-halt-on-mismatch is false, so the "+ "recorded hashes from this block up are discarded and the new hash replaces them.", "blockHeight", blockHeight, @@ -349,7 +323,7 @@ func (p *PebbleHashVault) replaceFrom(blockHeight uint64, existing []byte, hash p.cache.Purge() p.cache.Add(blockHeight, bytes.Clone(hash)) p.head = blockHeight - p.recorded = true + p.notEmpty = true return nil } @@ -357,7 +331,7 @@ func (p *PebbleHashVault) replaceFrom(blockHeight uint64, existing []byte, hash func (p *PebbleHashVault) Head() (uint64, bool) { p.mu.Lock() defer p.mu.Unlock() - return p.head, p.recorded + return p.head, p.notEmpty } // Get returns the hash recorded for blockHeight, without blocking. @@ -367,7 +341,7 @@ func (p *PebbleHashVault) Get(blockHeight uint64) ([32]byte, gigatypes.BlockHash if p.closed { return [32]byte{}, gigatypes.BlockHashStatusError, ErrClosed } - if !p.recorded || blockHeight > p.head { + if !p.notEmpty || blockHeight > p.head { return [32]byte{}, gigatypes.BlockHashStatusNotReady, nil } if blockHeight < p.pruneBoundary { @@ -424,7 +398,7 @@ func (p *PebbleHashVault) Reset(ctx context.Context, blockHeight uint64, hash [] p.cache.Purge() p.cache.Add(blockHeight, bytes.Clone(hash)) p.head = blockHeight - p.recorded = true + p.notEmpty = true return nil } diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go index 0fc679a075..e76459b81d 100644 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go @@ -2,7 +2,6 @@ package hashvault import ( "context" - "os" "path/filepath" "testing" @@ -180,15 +179,3 @@ func TestStoredRange(t *testing.T) { require.Equal(t, uint64(4), oldest) require.Equal(t, uint64(9), newest) } - -func TestOpenDeletesTheLegacyVault(t *testing.T) { - legacy := filepath.Join(t.TempDir(), "hashvault") - require.NoError(t, os.MkdirAll(legacy, 0o750)) - require.NoError(t, os.WriteFile(filepath.Join(legacy, "000001.log"), []byte("x"), 0o600)) - - newTestPebbleVault(t, func(cfg *HashVaultConfig) { cfg.LegacyPebbleDir = legacy }) - _, err := os.Stat(legacy) - require.True(t, os.IsNotExist(err), "the legacy vault must be gone, got %v", err) - - newTestPebbleVault(t, func(cfg *HashVaultConfig) { cfg.LegacyPebbleDir = legacy }) -} diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go index 10bdbd59e4..03593ca137 100644 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault_rollback_test.go @@ -123,12 +123,6 @@ func TestHardRollbackPebbleHashVaultRejectsMaxUint64Height(t *testing.T) { err := HardRollbackPebbleHashVault(ctx, cfg, math.MaxUint64) require.ErrorIs(t, err, ErrRollbackHeightOverflow) - - v2, err := NewUnsafePebbleHashVault(ctx, cfg) - require.NoError(t, err) - t.Cleanup(func() { _ = v2.Close(ctx) }) - - require.ErrorIs(t, v2.CommitToHash(ctx, math.MaxUint64, bytesOfLen(0xEE, 32)), ErrHashMismatch) } func TestHardRollbackPebbleHashVaultRejectsMissingDir(t *testing.T) { diff --git a/sei-tendermint/node/setup.go b/sei-tendermint/node/setup.go index 9ca2d4ae5d..02d31a70df 100644 --- a/sei-tendermint/node/setup.go +++ b/sei-tendermint/node/setup.go @@ -393,8 +393,6 @@ func openAutobahnStorageManager( storageConfig.BlockDBConfig = &blockConfig storageConfig.HashVaultConfig.HaltOnMismatch = conf.HashVaultHaltOnMismatch storageConfig.HashVaultConfig.EmptyVaultRollbackBlocks = conf.HashVaultEmptyRollbackBlocks - // The Pebble-backed vault the giga router kept here before the vault moved into the state DB. - storageConfig.HashVaultConfig.LegacyPebbleDir = filepath.Join(directory, "hashvault") return bootstrap.NewGigaStorageManager(ctx, storageConfig) } From 1e059dc657c2b47770e4d85213b28da7d3375946 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Mon, 28 Sep 2026 13:20:50 -0500 Subject: [PATCH 4/5] remove pruning API --- giga/evmonly/giga_store.go | 18 ----------- giga/evmonly/giga_store_test.go | 18 ----------- giga/evmonly/memory_store.go | 5 ---- sei-db/bench/cryptosim/block_hashes.go | 8 ----- sei-db/bench/cryptosim/database.go | 4 --- sei-db/bench/gigasim/block_hashes.go | 8 ----- sei-db/bench/gigasim/execution_state.go | 5 ---- sei-db/config/giga_config.go | 4 +-- sei-db/state_db/giga/state_db.go | 6 ---- .../state_db/giga/state_db_hash_vault_test.go | 1 + sei-db/state_db/giga/types/state_db.go | 3 -- .../state_db/sc/hashvault/hashvault_config.go | 2 +- .../state_db/sc/hashvault/pebble_hashvault.go | 30 ++----------------- .../hashvault/pebble_hashvault_branch_test.go | 15 ++++------ .../sc/hashvault/pebble_hashvault_test.go | 2 ++ sei-tendermint/config/autobahn_toml_test.go | 8 ++--- sei-tendermint/config/config_fuzz_test.go | 10 +++---- 17 files changed, 24 insertions(+), 123 deletions(-) diff --git a/giga/evmonly/giga_store.go b/giga/evmonly/giga_store.go index f0814ae3f1..13c5992be5 100644 --- a/giga/evmonly/giga_store.go +++ b/giga/evmonly/giga_store.go @@ -23,10 +23,6 @@ var ( var _ StateReader = gigaSnapshotStateReader{} -// placeholderBlockHashRetention is how many of the newest blocks keep their state hashes. It is a -// placeholder until a real threshold is wired in. -const placeholderBlockHashRetention = 10_000 - // NamedChangeSetEncoder converts an executor-native state result into the // on-disk changesets understood by a giga store. It is called synchronously // while the block's read snapshot is still open. It must treat the input as @@ -101,11 +97,6 @@ func (e *Executor) executePreparedBlockWithStore(ctx context.Context, req Prepar if err := stateStore.CommitStateChanges(blockNumber, changesets); err != nil { return nil, fmt.Errorf("commit state changes for block %d: %w", req.Context.Number, err) } - // PLACEHOLDER: keeps the newest placeholderBlockHashRetention blocks' hashes. A real threshold, set by - // what giga execution needs block hashes for, should be wired in here. - if err := stateStore.PruneBlockHashesBelow(blockHashesPrunedBelow(req.Context.Number)); err != nil { - return nil, fmt.Errorf("prune block hashes after block %d: %w", req.Context.Number, err) - } ok = true return result, nil } @@ -155,12 +146,3 @@ func (r gigaSnapshotStateReader) GetState(addr common.Address, key common.Hash) func (r gigaSnapshotStateReader) useMissingState(addr common.Address) bool { return r.missingState != nil && !r.snapshot.AccountExists(addr) } - -// blockHashesPrunedBelow returns the block below which state hashes may be pruned once blockNumber is -// committed. -func blockHashesPrunedBelow(blockNumber uint64) uint64 { - if blockNumber < placeholderBlockHashRetention { - return 0 - } - return blockNumber - placeholderBlockHashRetention -} diff --git a/giga/evmonly/giga_store_test.go b/giga/evmonly/giga_store_test.go index 8133d26a89..9d65a422e5 100644 --- a/giga/evmonly/giga_store_test.go +++ b/giga/evmonly/giga_store_test.go @@ -21,7 +21,6 @@ type recordingGigaStore struct { commitErr error commitBlock []int64 commits [][]*proto.NamedChangeSet - pruneBelow []uint64 } func (s *recordingGigaStore) CommitStateChanges(blockNum int64, changeset []*proto.NamedChangeSet) error { @@ -49,11 +48,6 @@ func (s *recordingGigaStore) GetBlockHash(uint64) ([32]byte, gigatypes.BlockHash return [32]byte{}, gigatypes.BlockHashStatusNotReady, nil } -func (s *recordingGigaStore) PruneBlockHashesBelow(blockNumber uint64) error { - s.pruneBelow = append(s.pruneBelow, blockNumber) - return nil -} - func (s *recordingGigaStore) Close() error { return nil } type memoryGigaSnapshot struct { @@ -239,8 +233,6 @@ func TestExecutorCommitsGigaStoreStateChanges(t *testing.T) { require.Equal(t, 1, store.openCount) require.Equal(t, 1, snapshot.closeCount) require.Equal(t, []int64{41}, store.commitBlock) - require.Equal(t, []uint64{0}, store.pruneBelow, - "the block hashes are pruned after the commit, and none are old enough to go yet") require.Equal(t, [][]*proto.NamedChangeSet{wantChangesets}, store.commits) require.Contains(t, result.ChangeSet.Balances, BalanceChange{Address: recipient, Balance: big.NewInt(7)}) result.Release() @@ -461,13 +453,3 @@ func TestExecutorGigaStoreFailuresDoNotCommitPartialState(t *testing.T) { require.Empty(t, store.commits) }) } - -// The placeholder retention keeps the newest placeholderBlockHashRetention blocks' hashes, and never wraps -// below block 0 on a young chain. -func TestBlockHashesPrunedBelowKeepsThePlaceholderRetention(t *testing.T) { - require.Equal(t, uint64(0), blockHashesPrunedBelow(0)) - require.Equal(t, uint64(0), blockHashesPrunedBelow(placeholderBlockHashRetention-1)) - require.Equal(t, uint64(0), blockHashesPrunedBelow(placeholderBlockHashRetention)) - require.Equal(t, uint64(1), blockHashesPrunedBelow(placeholderBlockHashRetention+1)) - require.Equal(t, uint64(5_000), blockHashesPrunedBelow(placeholderBlockHashRetention+5_000)) -} diff --git a/giga/evmonly/memory_store.go b/giga/evmonly/memory_store.go index c0869ee049..43c8888ab8 100644 --- a/giga/evmonly/memory_store.go +++ b/giga/evmonly/memory_store.go @@ -387,11 +387,6 @@ func (s *MemoryStore) GetBlockHash(uint64) ([32]byte, gigatypes.BlockHashStatus, return [32]byte{}, gigatypes.BlockHashStatusNotReady, nil } -// PruneBlockHashesBelow does nothing, since this store records no block hashes. -func (s *MemoryStore) PruneBlockHashesBelow(uint64) error { - return nil -} - // Close releases nothing. This store holds no handle outside its own maps, which go with it. func (s *MemoryStore) Close() error { return nil } diff --git a/sei-db/bench/cryptosim/block_hashes.go b/sei-db/bench/cryptosim/block_hashes.go index 017e62f57a..af6d85fe99 100644 --- a/sei-db/bench/cryptosim/block_hashes.go +++ b/sei-db/bench/cryptosim/block_hashes.go @@ -112,11 +112,3 @@ func (w *blockHashWaiter) takeHash() (*lthash.BlockHash, error) { "%d blocks behind the block just committed", w.waitTimeout, w.lagBlocks) } } - -// blockHashRetention is how many of the newest blocks keep their hashes. -const blockHashRetention = 10_000 - -// blockHashesPrunedBelow returns the block below which the hashes of blocks up to blockNum may be pruned. -func blockHashesPrunedBelow(blockNum int64) uint64 { - return uint64(max(blockNum-blockHashRetention, 0)) //nolint:gosec // clamped non-negative -} diff --git a/sei-db/bench/cryptosim/database.go b/sei-db/bench/cryptosim/database.go index 7030430d3c..caac90a44e 100644 --- a/sei-db/bench/cryptosim/database.go +++ b/sei-db/bench/cryptosim/database.go @@ -229,10 +229,6 @@ func (d *Database) FinalizeBlock( if err := d.db.CommitStateChanges(blockNum, changeSets); err != nil { return fmt.Errorf("failed to commit block %d: %w", blockNum, err) } - // The same placeholder retention giga execution uses, until a real threshold is wired in there. - if err := d.db.PruneBlockHashesBelow(blockHashesPrunedBelow(blockNum)); err != nil { - return fmt.Errorf("failed to prune block hashes after committing block %d: %w", blockNum, err) - } d.nextBlockNumber++ d.metrics.ReportDBCommit() d.reopenView() diff --git a/sei-db/bench/gigasim/block_hashes.go b/sei-db/bench/gigasim/block_hashes.go index b418ee9be3..a3d236c6ea 100644 --- a/sei-db/bench/gigasim/block_hashes.go +++ b/sei-db/bench/gigasim/block_hashes.go @@ -104,11 +104,3 @@ func (w *blockHashWaiter) takeHash() (*lthash.BlockHash, error) { "%d blocks behind the block just committed", w.waitTimeout, w.lagBlocks) } } - -// blockHashRetention is how many of the newest blocks keep their hashes. -const blockHashRetention = 10_000 - -// blockHashesPrunedBelow returns the block below which the hashes of blocks up to blockNum may be pruned. -func blockHashesPrunedBelow(blockNum int64) uint64 { - return uint64(max(blockNum-blockHashRetention, 0)) //nolint:gosec // clamped non-negative -} diff --git a/sei-db/bench/gigasim/execution_state.go b/sei-db/bench/gigasim/execution_state.go index a167594507..198877af84 100644 --- a/sei-db/bench/gigasim/execution_state.go +++ b/sei-db/bench/gigasim/execution_state.go @@ -120,11 +120,6 @@ func (s *executionState) commitBlock(blockNum int64, writes blockWrites) error { } s.metrics.ReportStateCommit(int64(len(writes.changeSets[0].Changeset.Pairs))) - // The same placeholder retention giga execution uses, until a real threshold is wired in there. - if err := s.db.PruneBlockHashesBelow(blockHashesPrunedBelow(blockNum)); err != nil { - return fmt.Errorf("failed to prune block hashes after committing block %d: %w", blockNum, err) - } - // Committing a block is not finishing it: the hash of a block committed a bounded number of blocks // ago is taken here, and waited for when hashing has fallen behind execution. Reopening the view // is charged here too, being a fraction of a percent that no one reads as a stage of its own. diff --git a/sei-db/config/giga_config.go b/sei-db/config/giga_config.go index 133399f124..e2592cdb11 100644 --- a/sei-db/config/giga_config.go +++ b/sei-db/config/giga_config.go @@ -50,8 +50,8 @@ func DefaultGigaStorageConfig(homePath string) (*GigaStorageConfig, error) { ssConfig.DisableInternalWAL = true hashVaultConfig := hashvault.DefaultHashVaultConfig() - // Existing Autobahn nodes keep their hash vault here, under the persistent state dir. Moving it loses - // the hashes they have recorded. + // Existing Autobahn nodes already have a hash vault here, holding the ABCI app hashes the GigaRouter + // recorded rather than FlatKV checksums, so the first hash checked after an upgrade does not match. hashVaultConfig.DataDir = filepath.Join(homePath, "hashvault") receiptConfig := DefaultReceiptStoreConfig() diff --git a/sei-db/state_db/giga/state_db.go b/sei-db/state_db/giga/state_db.go index 926194d950..340938deed 100644 --- a/sei-db/state_db/giga/state_db.go +++ b/sei-db/state_db/giga/state_db.go @@ -344,9 +344,3 @@ func (s *StateDB) GetBlockHash(blockNumber uint64) ([32]byte, gigatypes.BlockHas } return hash, status, nil } - -// PruneBlockHashesBelow permits the hash vault to delete the hashes of blocks below blockNumber. -func (s *StateDB) PruneBlockHashesBelow(blockNumber uint64) error { - s.vault.PruneBelow(blockNumber) - return nil -} diff --git a/sei-db/state_db/giga/state_db_hash_vault_test.go b/sei-db/state_db/giga/state_db_hash_vault_test.go index 2e2aaeaaf8..f3719cdbb2 100644 --- a/sei-db/state_db/giga/state_db_hash_vault_test.go +++ b/sei-db/state_db/giga/state_db_hash_vault_test.go @@ -45,6 +45,7 @@ func newVaultTestStores(t *testing.T) *vaultTestStores { hashVaultCfg := hashvault.DefaultHashVaultConfig() hashVaultCfg.DataDir = filepath.Join(t.TempDir(), "hashvault") hashVaultCfg.Fsync = false + hashVaultCfg.HaltOnMismatch = true return &vaultTestStores{ flatkvCfg: flatkvCfg, ssCfg: config.StateStoreConfig{Enable: false}, diff --git a/sei-db/state_db/giga/types/state_db.go b/sei-db/state_db/giga/types/state_db.go index 8b903a8f26..9bb81dc2bb 100644 --- a/sei-db/state_db/giga/types/state_db.go +++ b/sei-db/state_db/giga/types/state_db.go @@ -65,9 +65,6 @@ type StateDB interface { // BlockHashStatusError. A returned hash is crash durable. GetBlockHash(blockNumber uint64) (hash [32]byte, status BlockHashStatus, err error) - // PruneBlockHashesBelow permits the hashes of blocks below blockNumber to be deleted. - PruneBlockHashesBelow(blockNumber uint64) error - // Close releases everything this StateDB was built over, reporting every failure rather than // stopping at the first. Close() error diff --git a/sei-db/state_db/sc/hashvault/hashvault_config.go b/sei-db/state_db/sc/hashvault/hashvault_config.go index fea75b00ce..24205d09e7 100644 --- a/sei-db/state_db/sc/hashvault/hashvault_config.go +++ b/sei-db/state_db/sc/hashvault/hashvault_config.go @@ -34,7 +34,7 @@ func DefaultHashVaultConfig() HashVaultConfig { return HashVaultConfig{ Fsync: false, CacheSize: 1024, - HaltOnMismatch: true, + HaltOnMismatch: false, EmptyVaultRollbackBlocks: 1000, } } diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault.go b/sei-db/state_db/sc/hashvault/pebble_hashvault.go index 4ded427cd1..d8b5686cec 100644 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault.go +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "sync" - "sync/atomic" "github.com/cockroachdb/pebble/v2" "github.com/ethereum/go-ethereum/common/lru" @@ -38,11 +37,6 @@ type PebbleHashVault struct { head uint64 // notEmpty is true when the vault holds at least one hash. notEmpty bool - - // outerFloor is the floor PruneBelow has raised. Only ever rises. - outerFloor atomic.Uint64 - // gcFloor is the floor PruneHistory has raised. Only ever rises. - gcFloor atomic.Uint64 } // NewPebbleHashVault opens (or creates) a PebbleHashVault rooted at config.DataDir. @@ -402,37 +396,19 @@ func (p *PebbleHashVault) Reset(ctx context.Context, blockHeight uint64, hash [] return nil } -// PruneBelow permits the hashes of blocks below blockHeight to be deleted, as far as the vault's owner is -// concerned. A hash is deleted only once PruneHistory has permitted it too. -func (p *PebbleHashVault) PruneBelow(blockHeight uint64) { - raiseFloor(&p.outerFloor, blockHeight) -} - -// raiseFloor raises floor to blockHeight, leaving it where it is when it is already higher. -func raiseFloor(floor *atomic.Uint64, blockHeight uint64) { - for { - current := floor.Load() - if blockHeight <= current || floor.CompareAndSwap(current, blockHeight) { - return - } - } -} - // Name implements controller.PrunableStore. func (p *PebbleHashVault) Name() string { return "HashVault" } -// PruneHistory implements controller.PrunableStore. It permits the hashes of blocks below blockHeight to -// be deleted, as far as the storage garbage collector is concerned, and prunes every hash below both that -// and the floor PruneBelow has raised. The newest recorded hash is always kept. +// PruneHistory implements controller.PrunableStore. It deletes the hashes of blocks below blockHeight, +// always keeping the newest recorded hash. func (p *PebbleHashVault) PruneHistory(blockHeight uint64) error { - raiseFloor(&p.gcFloor, blockHeight) head, recorded := p.Head() if !recorded { return nil } - floor := min(p.outerFloor.Load(), p.gcFloor.Load(), head) + floor := min(blockHeight, head) if err := p.Prune(context.Background(), floor); err != nil { return fmt.Errorf("failed to prune hashvault below %d: %w", floor, err) } diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go index e76459b81d..6995b54f8a 100644 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault_branch_test.go @@ -120,23 +120,20 @@ func TestResetLeavesOnlyTheGivenHeight(t *testing.T) { require.Equal(t, uint64(1001), newest) } -func TestPruneHistoryDeletesOnlyBelowBothFloors(t *testing.T) { +func TestPruneHistoryDeletesBelowTheCutLine(t *testing.T) { v := newTestPebbleVault(t) commitHeights(t, v, 1, 150) - require.NoError(t, v.PruneHistory(60)) - requireRecorded(t, v, 1, 1) - - v.PruneBelow(100) require.NoError(t, v.PruneHistory(60)) requireStatus(t, v, 59, gigatypes.BlockHashStatusTooOld) requireRecorded(t, v, 60, 60) - require.NoError(t, v.PruneHistory(200)) - requireStatus(t, v, 99, gigatypes.BlockHashStatusTooOld) - requireRecorded(t, v, 100, 100) + // A lower cut line than one already applied restores nothing. + require.NoError(t, v.PruneHistory(30)) + requireStatus(t, v, 59, gigatypes.BlockHashStatusTooOld) + requireRecorded(t, v, 60, 60) - v.PruneBelow(1000) + // A cut line above the head keeps the newest hash. require.NoError(t, v.PruneHistory(1000)) requireStatus(t, v, 149, gigatypes.BlockHashStatusTooOld) requireRecorded(t, v, 150, 150) diff --git a/sei-db/state_db/sc/hashvault/pebble_hashvault_test.go b/sei-db/state_db/sc/hashvault/pebble_hashvault_test.go index d19ef80d78..0a24389c68 100644 --- a/sei-db/state_db/sc/hashvault/pebble_hashvault_test.go +++ b/sei-db/state_db/sc/hashvault/pebble_hashvault_test.go @@ -17,6 +17,7 @@ func newTestPebbleVault(t *testing.T, configMutators ...func(*HashVaultConfig)) t.Helper() cfg := DefaultHashVaultConfig() cfg.DataDir = filepath.Join(t.TempDir(), "vault") + cfg.HaltOnMismatch = true for _, m := range configMutators { m(&cfg) } @@ -36,6 +37,7 @@ func reopenTestPebbleVault(t *testing.T, v *PebbleHashVault) *PebbleHashVault { require.NoError(t, v.Close(context.Background())) cfg := DefaultHashVaultConfig() cfg.DataDir = dir + cfg.HaltOnMismatch = true reopened, err := NewUnsafePebbleHashVault(context.Background(), cfg) require.NoError(t, err) t.Cleanup(func() { diff --git a/sei-tendermint/config/autobahn_toml_test.go b/sei-tendermint/config/autobahn_toml_test.go index 84f3e58107..8f530f47ea 100644 --- a/sei-tendermint/config/autobahn_toml_test.go +++ b/sei-tendermint/config/autobahn_toml_test.go @@ -26,7 +26,7 @@ func TestAutobahnKeysParseFromTopLevel(t *testing.T) { const content = ` autobahn-config-file = "/etc/sei/autobahn.json" -hash-vault-halt-on-mismatch = false +hash-vault-halt-on-mismatch = true hash-vault-empty-rollback-blocks = 7 [rpc] @@ -41,7 +41,7 @@ laddr = "tcp://127.0.0.1:26657" cfg, err := commands.ParseConfig(tmconfig.DefaultConfig()) require.NoError(t, err) require.Equal(t, "/etc/sei/autobahn.json", cfg.AutobahnConfigFile) - require.False(t, cfg.HashVaultHaltOnMismatch) + require.True(t, cfg.HashVaultHaltOnMismatch) require.Equal(t, uint64(7), cfg.HashVaultEmptyRollbackBlocks) } @@ -56,7 +56,7 @@ func TestAutobahnKeysIgnoredUnderSectionHeader(t *testing.T) { const content = ` [self-remediation] autobahn-config-file = "/etc/sei/autobahn.json" -hash-vault-halt-on-mismatch = false +hash-vault-halt-on-mismatch = true hash-vault-empty-rollback-blocks = 7 ` configPath := filepath.Join(t.TempDir(), "config.toml") @@ -70,7 +70,7 @@ hash-vault-empty-rollback-blocks = 7 // The field ends up empty — viper saw self-remediation.autobahn-config-file // instead of the top-level key mapstructure was looking for. require.Empty(t, cfg.AutobahnConfigFile) - require.True(t, cfg.HashVaultHaltOnMismatch) + require.False(t, cfg.HashVaultHaltOnMismatch) require.Equal(t, uint64(1000), cfg.HashVaultEmptyRollbackBlocks) } diff --git a/sei-tendermint/config/config_fuzz_test.go b/sei-tendermint/config/config_fuzz_test.go index 3f111e07b8..bb0a1f7ccb 100644 --- a/sei-tendermint/config/config_fuzz_test.go +++ b/sei-tendermint/config/config_fuzz_test.go @@ -152,7 +152,7 @@ func FuzzRootScopeKeysRequireRootScope(f *testing.F) { doc.WriteString("[p2p]\n") } doc.WriteString("autobahn-config-file = \"" + path + "\"\n") - doc.WriteString("hash-vault-halt-on-mismatch = false\n") + doc.WriteString("hash-vault-halt-on-mismatch = true\n") } conf, err := unmarshalConfigTOML(t, doc.String()) @@ -161,10 +161,10 @@ func FuzzRootScopeKeysRequireRootScope(f *testing.F) { } wantPath := "" - wantHalt := true + wantHalt := false if present && !underSection { wantPath = path - wantHalt = false + wantHalt = true } if conf.AutobahnConfigFile != wantPath { t.Fatalf("autobahn-config-file resolved to %q, want %q (present=%v underSection=%v); "+ @@ -295,8 +295,8 @@ func TestAutobahnPointerAbsenceDisablesTheSubsystem(t *testing.T) { if conf.AutobahnConfigFile != "" { t.Fatalf("the default autobahn pointer must be empty, got %q", conf.AutobahnConfigFile) } - if !conf.HashVaultHaltOnMismatch { - t.Fatal("the default must leave a hash vault mismatch halting the node") + if conf.HashVaultHaltOnMismatch { + t.Fatal("the default must log a hash vault mismatch rather than halt the node") } } From e7df1c378f4a6e5a0b7784a1fc4465e9e0076ba0 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 29 Sep 2026 11:14:00 -0500 Subject: [PATCH 5/5] fix broken test --- cmd/seid/cmd/legacy_config_fuzz_test.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/cmd/seid/cmd/legacy_config_fuzz_test.go b/cmd/seid/cmd/legacy_config_fuzz_test.go index 03c5a16860..684749080a 100644 --- a/cmd/seid/cmd/legacy_config_fuzz_test.go +++ b/cmd/seid/cmd/legacy_config_fuzz_test.go @@ -179,13 +179,11 @@ var tmKeys = []tmKey{ // FuzzHashVaultHaltOnMismatchResolution pins the root-scope switch that selects whether a hash vault // mismatch halts the node. // -// Two things make it worth its own target. It is a bool whose safe value is the default, so an absent -// key must resolve true — setting it false lets a node replace a recorded state hash with only an error -// log. And it lives at TOML root scope, before any [section] header: nested under a section it parses as -// a different key and is silently ignored, which reads as "I turned halting off" while the node still -// halts. The document is built from the fuzzer's choices rather than taken as free text, so the expected -// outcome follows from construction instead of being a second input the fuzzer can mutate out of -// agreement with the first. +// An absent key resolves false, so a mismatch replaces the recorded state hash with only an error log. +// The key lives at TOML root scope, before any [section] header: nested under a section it parses as a +// different key and is silently ignored. The document is built from the fuzzer's choices rather than +// taken as free text, so the expected outcome follows from construction instead of being a second input +// the fuzzer can mutate out of agreement with the first. func FuzzHashVaultHaltOnMismatchResolution(f *testing.F) { f.Add(false, false, false) f.Add(true, false, false) // root scope, false: a mismatch only logs @@ -210,7 +208,7 @@ func FuzzHashVaultHaltOnMismatchResolution(f *testing.F) { // Root scope is the only placement that resolves. Nested under a section the // key becomes p2p.hash-vault-halt-on-mismatch, which nothing reads. - wantHalt := true + wantHalt := false if present && !underSection { wantHalt = value } @@ -226,16 +224,15 @@ func FuzzHashVaultHaltOnMismatchResolution(f *testing.F) { }) } -// TestHashVaultDefaultsHaltOnMismatch states the defaults on their own, so the safe value is pinned even -// if every seed above were removed. -func TestHashVaultDefaultsHaltOnMismatch(t *testing.T) { +// TestHashVaultDefaults pins the hash vault defaults an empty home resolves to. +func TestHashVaultDefaults(t *testing.T) { configtest.Isolate(t) got := applyLegacy(t, configtest.NewHome(t), nil) if got.err != nil { t.Fatalf("Apply: %v", got.err) } - if !got.ctx.Config.HashVaultHaltOnMismatch { - t.Fatal("an empty home must leave a hash vault mismatch halting the node") + if got.ctx.Config.HashVaultHaltOnMismatch { + t.Fatal("an empty home must leave a hash vault mismatch replacing the recorded hash") } if got.ctx.Config.HashVaultEmptyRollbackBlocks != 1000 { t.Fatalf("an empty home must rewind 1000 blocks over an empty hash vault, got %d",