diff --git a/app/app.go b/app/app.go index 0f842a2387..a5e40fdc13 100644 --- a/app/app.go +++ b/app/app.go @@ -1902,6 +1902,27 @@ func (app *App) ProcessTXsWithOCCGiga(ctx sdk.Context, txs [][]byte, typedTxs [] return execResults, ctx } +// signalEVMServersStart lets the EVM HTTP and WS servers start. It only sends +// each signal once. +func (app *App) signalEVMServersStart() { + if !app.httpServerStartSignalSent { + app.httpServerStartSignalSent = true + app.httpServerStartSignal <- struct{}{} + } + if !app.wsServerStartSignalSent { + app.wsServerStartSignalSent = true + app.wsServerStartSignal <- struct{}{} + } +} + +// InitLastHeader initializes checkState from the last block header and lets +// the EVM servers start. A frozen node needs this after a restart, because it +// processes no block that would otherwise start them. +func (app *App) InitLastHeader(lastHeader *tmproto.Header) { + app.BaseApp.InitLastHeader(lastHeader) + app.signalEVMServersStart() +} + // ProcessBlock executes block transactions. If preDecoded is non-nil and len(preDecoded)==len(txs), // those decoded transactions are reused (bytes are not decoded again); EVM preprocessing still runs // on the block context. @@ -1924,16 +1945,7 @@ func (app *App) ProcessBlock(ctx sdk.Context, txs [][]byte, req *BlockProcessReq } }() - defer func() { - if !app.httpServerStartSignalSent { - app.httpServerStartSignalSent = true - app.httpServerStartSignal <- struct{}{} - } - if !app.wsServerStartSignalSent { - app.wsServerStartSignalSent = true - app.wsServerStartSignal <- struct{}{} - } - }() + defer app.signalEVMServersStart() ctx = ctx.WithIsOCCEnabled(app.OccEnabled()) diff --git a/app/init_last_header_test.go b/app/init_last_header_test.go new file mode 100644 index 0000000000..b1e72ce71d --- /dev/null +++ b/app/init_last_header_test.go @@ -0,0 +1,36 @@ +package app + +import ( + "testing" + "time" + + tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types" + "github.com/stretchr/testify/require" +) + +func TestInitLastHeaderSetsCheckStateAndStartsEVMServers(t *testing.T) { + app := Setup(t, false, false, false) + header := &tmproto.Header{ChainID: "sei-test", Height: 9, Time: time.Unix(1700000000, 0).UTC()} + + app.InitLastHeader(header) + + ctx := app.GetCheckCtx() + require.Equal(t, header.ChainID, ctx.ChainID()) + require.Equal(t, header.Height, ctx.BlockHeight()) + require.True(t, header.Time.Equal(ctx.BlockTime())) + for name, signal := range map[string]chan struct{}{ + "http": app.httpServerStartSignal, + "ws": app.wsServerStartSignal, + } { + select { + case <-signal: + default: + t.Fatalf("%s server start signal was not sent", name) + } + } + + // A later block must not block on the already-sent signals. + app.signalEVMServersStart() + require.Empty(t, app.httpServerStartSignal) + require.Empty(t, app.wsServerStartSignal) +} diff --git a/integration_test/frozen_rpc_router/router_test.go b/integration_test/frozen_rpc_router/router_test.go index c1734a1232..b8a59d0571 100644 --- a/integration_test/frozen_rpc_router/router_test.go +++ b/integration_test/frozen_rpc_router/router_test.go @@ -11,6 +11,7 @@ import ( "io" "net/http" "os" + "os/exec" "strconv" "strings" "testing" @@ -22,6 +23,7 @@ const ( defaultFrozen10URL = "http://127.0.0.1:8547" defaultFrozen20URL = "http://127.0.0.1:8549" defaultLiveURL = "http://127.0.0.1:8545" + defaultFrozen10Box = "sei-node-1" routeHeader = "Sei-RPC-Route" ) @@ -94,6 +96,59 @@ func TestFrozenRPCRouterReachesEveryNode(t *testing.T) { }) } +// A restarted frozen node commits no further blocks, so its EVM RPC has to +// come up from the stored chain state alone. +func TestFrozenNodeServesEVMRPCAfterRestart(t *testing.T) { + client := &http.Client{Timeout: 10 * time.Second} + frozen10URL := envOrDefault("FROZEN_RPC_NODE_10_URL", defaultFrozen10URL) + container := envOrDefault("FROZEN_RPC_NODE_10_CONTAINER", defaultFrozen10Box) + + waitForHead(t, client, frozen10URL, func(height uint64) bool { return height == 9 }, "frozen node at height 10") + restartSeid(t, container) + waitForHead(t, client, frozen10URL, func(height uint64) bool { return height == 9 }, "restarted frozen node at height 10") + + response, _, err := callRPC(t.Context(), client, frozen10URL, "eth_getBlockByNumber", []any{"0x5", false}) + if err != nil { + t.Fatal(err) + } + if response.Error != nil { + t.Fatalf("RPC returned error %d: %s", response.Error.Code, response.Error.Message) + } + var block struct { + Number string `json:"number"` + } + if err := json.Unmarshal(response.Result, &block); err != nil { + t.Fatalf("decode block response: %v", err) + } + if block.Number != "0x5" { + t.Fatalf("block number = %q, want %q", block.Number, "0x5") + } +} + +// restartSeid stops seid in the container and starts it again with the +// container's original start script. The container itself keeps running. +func restartSeid(t *testing.T, container string) { + t.Helper() + const stop = ` +seid_pids() { + for p in /proc/[0-9]*; do + [ "$(cat "$p/comm" 2>/dev/null)" = seid ] && echo "${p#/proc/}" + done +} +pids=$(seid_pids) +[ -n "$pids" ] || { echo "seid is not running" >&2; exit 1; } +kill $pids +while [ -n "$(seid_pids)" ]; do sleep 1; done +cp "build/generated/logs/seid-$ID.log" "build/generated/logs/seid-$ID-before-restart.log" +` + if output, err := exec.CommandContext(t.Context(), "docker", "exec", container, "sh", "-c", stop).CombinedOutput(); err != nil { + t.Fatalf("stop seid in %s: %v: %s", container, err, output) + } + if output, err := exec.CommandContext(t.Context(), "docker", "exec", "-d", container, "/usr/bin/start_sei.sh").CombinedOutput(); err != nil { + t.Fatalf("start seid in %s: %v: %s", container, err, output) + } +} + func waitForHead(t *testing.T, client *http.Client, endpoint string, ready func(uint64) bool, description string) { t.Helper() ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute) diff --git a/sei-cosmos/baseapp/abci.go b/sei-cosmos/baseapp/abci.go index 38a1e8fae4..e59cc14750 100644 --- a/sei-cosmos/baseapp/abci.go +++ b/sei-cosmos/baseapp/abci.go @@ -259,6 +259,13 @@ func (app *BaseApp) SetDeliverStateToCommit() { app.stateToCommit = app.deliverState } +// InitLastHeader sets checkState from the header of the last committed block, +// as Commit does. It is for nodes that start without committing a block, such +// as a frozen node after a restart. +func (app *BaseApp) InitLastHeader(lastHeader *tmproto.Header) { + app.setCheckState(*lastHeader) +} + // Commit implements the ABCI interface. It will commit all state that exists in // the deliver state's multi-store and includes the resulting commit ID in the // returned abci.ResponseCommit. Commit will set the check state based on the diff --git a/sei-tendermint/internal/proxy/proxy.go b/sei-tendermint/internal/proxy/proxy.go index de8abac1a9..2b77396f96 100644 --- a/sei-tendermint/internal/proxy/proxy.go +++ b/sei-tendermint/internal/proxy/proxy.go @@ -11,8 +11,15 @@ import ( "github.com/holiman/uint256" "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" + tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types" ) +// lastHeaderInitializer is implemented by apps that can be initialized from +// the last block header when they start without committing a block. +type lastHeaderInitializer interface { + InitLastHeader(lastHeader *tmproto.Header) +} + // Proxy wraps an ABCI application and records ABCI method timings. type Proxy struct { app types.Application @@ -54,6 +61,14 @@ func (app *Proxy) EvmBalance(addr common.Address, seiAddr []byte) uint256.Int { return app.app.EvmBalance(addr, seiAddr) } +// InitLastHeader passes the last block header to the app if it implements +// lastHeaderInitializer, and does nothing otherwise. +func (app *Proxy) InitLastHeader(lastHeader *tmproto.Header) { + if initializer, ok := app.app.(lastHeaderInitializer); ok { + initializer.InitLastHeader(lastHeader) + } +} + func (app *Proxy) Commit(ctx context.Context) (*types.ResponseCommit, error) { defer addTimeSample(app.metrics.MethodTiming.With("method", "commit", "type", "sync"))() return app.app.Commit(ctx) diff --git a/sei-tendermint/node/freeze_test.go b/sei-tendermint/node/freeze_test.go index 587549a9fd..1069b32d8c 100644 --- a/sei-tendermint/node/freeze_test.go +++ b/sei-tendermint/node/freeze_test.go @@ -7,10 +7,15 @@ import ( "slices" "testing" + "github.com/sei-protocol/sei-chain/sei-tendermint/abci/example/kvstore" "github.com/sei-protocol/sei-chain/sei-tendermint/config" mempoolreactor "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool/reactor" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/pubsub" rpccore "github.com/sei-protocol/sei-chain/sei-tendermint/internal/rpc/core" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/tcp" + tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types" "github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes" "github.com/sei-protocol/sei-chain/sei-tendermint/types" ) @@ -141,3 +146,61 @@ func TestFreezeModeDisablesMempoolTraffic(t *testing.T) { t.Fatal("mempool channel is advertised in freeze mode") } } + +type lastHeaderRecordingApp struct { + *kvstore.Application + lastHeaders []*tmproto.Header +} + +func (app *lastHeaderRecordingApp) InitLastHeader(lastHeader *tmproto.Header) { + app.lastHeaders = append(app.lastHeaders, lastHeader) +} + +// A frozen node commits no further blocks after a restart, so the node must +// initialize the app from the last stored header instead. +func TestFreezeModeRestartInitializesAppFromLastHeader(t *testing.T) { + cfg, err := config.ResetTestRoot(t.TempDir(), "freeze_restart_test") + require.NoError(t, err) + cfg.RPC.ListenAddress = "" + // The frozen node must reopen the validator's stores. + cfg.DBBackend = "goleveldb" + + validator, err := newLocalNodeService(t.Context(), cfg) + require.NoError(t, err) + blocks, err := validator.(*nodeImpl).EventBus().SubscribeWithArgs(t.Context(), pubsub.SubscribeArgs{ + ClientID: "freeze_restart_test", + Query: types.EventQueryNewBlock, + Limit: 10, + }) + require.NoError(t, err) + require.NoError(t, validator.Start(t.Context())) + for range 2 { + _, err := blocks.Next(t.Context()) + require.NoError(t, err) + } + validator.Stop() + validator.Wait() + + cfg.Mode = config.ModeFull + app := &lastHeaderRecordingApp{Application: kvstore.NewApplication()} + app.SetValidators(utils.OrPanic1(types.GenesisDocFromFile(cfg.GenesisFile())).ValidatorUpdates()) + frozenService, err := New(t.Context(), cfg, func() {}, app, nil, nil, DefaultMetricsProvider(cfg.Instrumentation)(cfg.ChainID()), types.DefaultConsensusPolicy(), WithFreezeHeight(1000)) + require.NoError(t, err) + frozen := frozenService.(*nodeImpl) + lastHeight := frozen.blockStore.Height() + require.GreaterOrEqual(t, lastHeight, int64(2)) + want := frozen.blockStore.LoadBlockMeta(lastHeight).Header + + require.NoError(t, frozen.Start(t.Context())) + t.Cleanup(func() { + frozen.Stop() + frozen.Wait() + }) + + require.Equal(t, 1, len(app.lastHeaders)) + got, err := types.HeaderFromProto(app.lastHeaders[0]) + require.NoError(t, err) + require.Equal(t, want.Hash(), got.Hash()) + require.Equal(t, lastHeight, got.Height) + require.Equal(t, frozen.genesisDoc.ChainID, got.ChainID) +} diff --git a/sei-tendermint/node/node.go b/sei-tendermint/node/node.go index 0c3689c7b0..1b4ff161fa 100644 --- a/sei-tendermint/node/node.go +++ b/sei-tendermint/node/node.go @@ -507,6 +507,16 @@ func (n *nodeImpl) OnStart(ctx context.Context) error { return fmt.Errorf("cannot load state: %w", err) } + // A frozen node never commits another block, so after a restart nothing + // would initialize the app (e.g. its query context and EVM RPC servers). + if n.freezeHeight > 0 && state.LastBlockHeight > 0 { + meta := n.blockStore.LoadBlockMeta(state.LastBlockHeight) + if meta == nil { + return fmt.Errorf("freeze mode: block %d is missing from the block store", state.LastBlockHeight) + } + n.rpcEnv.App.InitLastHeader(meta.Header.ToProto()) + } + logNodeStartupInfo(state, n.rpcEnv.PubKey, n.config.Mode) // TODO: Fetch and provide real options and do proper p2p bootstrapping.