Initialize app from last header on frozen restart for v6.6

app/app.go

@@ -1902,6 +1902,27 @@ func (app *App) ProcessTXsWithOCCGiga(ctx sdk.Context, txs [][]byte, typedTxs []
 	return execResults, ctx
 }
 
+// signalEVMServersStart lets the EVM HTTP and WS servers start. It only sends
+// each signal once.
+func (app *App) signalEVMServersStart() {
+	if !app.httpServerStartSignalSent {
+		app.httpServerStartSignalSent = true
+		app.httpServerStartSignal <- struct{}{}
+	}
+	if !app.wsServerStartSignalSent {
+		app.wsServerStartSignalSent = true
+		app.wsServerStartSignal <- struct{}{}
+	}
+}
+
+// InitLastHeader initializes checkState from the last block header and lets
+// the EVM servers start. A frozen node needs this after a restart, because it
+// processes no block that would otherwise start them.
+func (app *App) InitLastHeader(lastHeader *tmproto.Header) {
+	app.BaseApp.InitLastHeader(lastHeader)
+	app.signalEVMServersStart()
+}
+
 // ProcessBlock executes block transactions. If preDecoded is non-nil and len(preDecoded)==len(txs),
 // those decoded transactions are reused (bytes are not decoded again); EVM preprocessing still runs
 // on the block context.
@@ -1924,16 +1945,7 @@ func (app *App) ProcessBlock(ctx sdk.Context, txs [][]byte, req *BlockProcessReq
 		}
 	}()
 
-	defer func() {
-		if !app.httpServerStartSignalSent {
-			app.httpServerStartSignalSent = true
-			app.httpServerStartSignal <- struct{}{}
-		}
-		if !app.wsServerStartSignalSent {
-			app.wsServerStartSignalSent = true
-			app.wsServerStartSignal <- struct{}{}
-		}
-	}()
+	defer app.signalEVMServersStart()
 
 	ctx = ctx.WithIsOCCEnabled(app.OccEnabled())
 

app/init_last_header_test.go

@@ -0,0 +1,36 @@
+package app
+
+import (
+	"testing"
+	"time"
+
+	tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInitLastHeaderSetsCheckStateAndStartsEVMServers(t *testing.T) {
+	app := Setup(t, false, false, false)
+	header := &tmproto.Header{ChainID: "sei-test", Height: 9, Time: time.Unix(1700000000, 0).UTC()}
+
+	app.InitLastHeader(header)
+
+	ctx := app.GetCheckCtx()
+	require.Equal(t, header.ChainID, ctx.ChainID())
+	require.Equal(t, header.Height, ctx.BlockHeight())
+	require.True(t, header.Time.Equal(ctx.BlockTime()))
+	for name, signal := range map[string]chan struct{}{
+		"http": app.httpServerStartSignal,
+		"ws":   app.wsServerStartSignal,
+	} {
+		select {
+		case <-signal:
+		default:
+			t.Fatalf("%s server start signal was not sent", name)
+		}
+	}
+
+	// A later block must not block on the already-sent signals.
+	app.signalEVMServersStart()
+	require.Empty(t, app.httpServerStartSignal)
+	require.Empty(t, app.wsServerStartSignal)
+}

integration_test/frozen_rpc_router/router_test.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"os/exec"
 	"strconv"
 	"strings"
 	"testing"
@@ -22,6 +23,7 @@ const (
 	defaultFrozen10URL = "http://127.0.0.1:8547"
 	defaultFrozen20URL = "http://127.0.0.1:8549"
 	defaultLiveURL     = "http://127.0.0.1:8545"
+	defaultFrozen10Box = "sei-node-1"
 	routeHeader        = "Sei-RPC-Route"
 )
 
@@ -94,6 +96,59 @@ func TestFrozenRPCRouterReachesEveryNode(t *testing.T) {
 	})
 }
 
+// A restarted frozen node commits no further blocks, so its EVM RPC has to
+// come up from the stored chain state alone.
+func TestFrozenNodeServesEVMRPCAfterRestart(t *testing.T) {
+	client := &http.Client{Timeout: 10 * time.Second}
+	frozen10URL := envOrDefault("FROZEN_RPC_NODE_10_URL", defaultFrozen10URL)
+	container := envOrDefault("FROZEN_RPC_NODE_10_CONTAINER", defaultFrozen10Box)
+
+	waitForHead(t, client, frozen10URL, func(height uint64) bool { return height == 9 }, "frozen node at height 10")
+	restartSeid(t, container)
+	waitForHead(t, client, frozen10URL, func(height uint64) bool { return height == 9 }, "restarted frozen node at height 10")
+
+	response, _, err := callRPC(t.Context(), client, frozen10URL, "eth_getBlockByNumber", []any{"0x5", false})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if response.Error != nil {
+		t.Fatalf("RPC returned error %d: %s", response.Error.Code, response.Error.Message)
+	}
+	var block struct {
+		Number string `json:"number"`
+	}
+	if err := json.Unmarshal(response.Result, &block); err != nil {
+		t.Fatalf("decode block response: %v", err)
+	}
+	if block.Number != "0x5" {
+		t.Fatalf("block number = %q, want %q", block.Number, "0x5")
+	}
+}
+
+// restartSeid stops seid in the container and starts it again with the
+// container's original start script. The container itself keeps running.
+func restartSeid(t *testing.T, container string) {
+	t.Helper()
+	const stop = `
+seid_pids() {
+  for p in /proc/[0-9]*; do
+    [ "$(cat "$p/comm" 2>/dev/null)" = seid ] && echo "${p#/proc/}"
+  done
+}
+pids=$(seid_pids)
+[ -n "$pids" ] || { echo "seid is not running" >&2; exit 1; }
+kill $pids
+while [ -n "$(seid_pids)" ]; do sleep 1; done
+cp "build/generated/logs/seid-$ID.log" "build/generated/logs/seid-$ID-before-restart.log"
+`
+	if output, err := exec.CommandContext(t.Context(), "docker", "exec", container, "sh", "-c", stop).CombinedOutput(); err != nil {
+		t.Fatalf("stop seid in %s: %v: %s", container, err, output)
+	}
+	if output, err := exec.CommandContext(t.Context(), "docker", "exec", "-d", container, "/usr/bin/start_sei.sh").CombinedOutput(); err != nil {
+		t.Fatalf("start seid in %s: %v: %s", container, err, output)
+	}
+}
+
 func waitForHead(t *testing.T, client *http.Client, endpoint string, ready func(uint64) bool, description string) {
 	t.Helper()
 	ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute)

sei-cosmos/baseapp/abci.go

@@ -259,6 +259,13 @@ func (app *BaseApp) SetDeliverStateToCommit() {
 	app.stateToCommit = app.deliverState
 }
 
+// InitLastHeader sets checkState from the header of the last committed block,
+// as Commit does. It is for nodes that start without committing a block, such
+// as a frozen node after a restart.
+func (app *BaseApp) InitLastHeader(lastHeader *tmproto.Header) {
+	app.setCheckState(*lastHeader)
+}
+
 // Commit implements the ABCI interface. It will commit all state that exists in
 // the deliver state's multi-store and includes the resulting commit ID in the
 // returned abci.ResponseCommit. Commit will set the check state based on the

sei-tendermint/internal/proxy/proxy.go

@@ -11,8 +11,15 @@ import (
 	"github.com/holiman/uint256"
 
 	"github.com/sei-protocol/sei-chain/sei-tendermint/abci/types"
+	tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types"
 )
 
+// lastHeaderInitializer is implemented by apps that can be initialized from
+// the last block header when they start without committing a block.
+type lastHeaderInitializer interface {
+	InitLastHeader(lastHeader *tmproto.Header)
+}
+
 // Proxy wraps an ABCI application and records ABCI method timings.
 type Proxy struct {
 	app     types.Application
@@ -54,6 +61,14 @@ func (app *Proxy) EvmBalance(addr common.Address, seiAddr []byte) uint256.Int {
 	return app.app.EvmBalance(addr, seiAddr)
 }
 
+// InitLastHeader passes the last block header to the app if it implements
+// lastHeaderInitializer, and does nothing otherwise.
+func (app *Proxy) InitLastHeader(lastHeader *tmproto.Header) {
+	if initializer, ok := app.app.(lastHeaderInitializer); ok {
+		initializer.InitLastHeader(lastHeader)
+	}
+}
+
 func (app *Proxy) Commit(ctx context.Context) (*types.ResponseCommit, error) {
 	defer addTimeSample(app.metrics.MethodTiming.With("method", "commit", "type", "sync"))()
 	return app.app.Commit(ctx)

sei-tendermint/node/freeze_test.go

@@ -7,10 +7,15 @@ import (
 	"slices"
 	"testing"
 
+	"github.com/sei-protocol/sei-chain/sei-tendermint/abci/example/kvstore"
 	"github.com/sei-protocol/sei-chain/sei-tendermint/config"
 	mempoolreactor "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool/reactor"
+	"github.com/sei-protocol/sei-chain/sei-tendermint/internal/pubsub"
 	rpccore "github.com/sei-protocol/sei-chain/sei-tendermint/internal/rpc/core"
+	"github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils"
+	"github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require"
 	"github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/tcp"
+	tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types"
 	"github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes"
 	"github.com/sei-protocol/sei-chain/sei-tendermint/types"
 )
@@ -141,3 +146,61 @@ func TestFreezeModeDisablesMempoolTraffic(t *testing.T) {
 		t.Fatal("mempool channel is advertised in freeze mode")
 	}
 }
+
+type lastHeaderRecordingApp struct {
+	*kvstore.Application
+	lastHeaders []*tmproto.Header
+}
+
+func (app *lastHeaderRecordingApp) InitLastHeader(lastHeader *tmproto.Header) {
+	app.lastHeaders = append(app.lastHeaders, lastHeader)
+}
+
+// A frozen node commits no further blocks after a restart, so the node must
+// initialize the app from the last stored header instead.
+func TestFreezeModeRestartInitializesAppFromLastHeader(t *testing.T) {
+	cfg, err := config.ResetTestRoot(t.TempDir(), "freeze_restart_test")
+	require.NoError(t, err)
+	cfg.RPC.ListenAddress = ""
+	// The frozen node must reopen the validator's stores.
+	cfg.DBBackend = "goleveldb"
+
+	validator, err := newLocalNodeService(t.Context(), cfg)
+	require.NoError(t, err)
+	blocks, err := validator.(*nodeImpl).EventBus().SubscribeWithArgs(t.Context(), pubsub.SubscribeArgs{
+		ClientID: "freeze_restart_test",
+		Query:    types.EventQueryNewBlock,
+		Limit:    10,
+	})
+	require.NoError(t, err)
+	require.NoError(t, validator.Start(t.Context()))
+	for range 2 {
+		_, err := blocks.Next(t.Context())
+		require.NoError(t, err)
+	}
+	validator.Stop()
+	validator.Wait()
+
+	cfg.Mode = config.ModeFull
+	app := &lastHeaderRecordingApp{Application: kvstore.NewApplication()}
+	app.SetValidators(utils.OrPanic1(types.GenesisDocFromFile(cfg.GenesisFile())).ValidatorUpdates())
+	frozenService, err := New(t.Context(), cfg, func() {}, app, nil, nil, DefaultMetricsProvider(cfg.Instrumentation)(cfg.ChainID()), types.DefaultConsensusPolicy(), WithFreezeHeight(1000))
+	require.NoError(t, err)
+	frozen := frozenService.(*nodeImpl)
+	lastHeight := frozen.blockStore.Height()
+	require.GreaterOrEqual(t, lastHeight, int64(2))
+	want := frozen.blockStore.LoadBlockMeta(lastHeight).Header
+
+	require.NoError(t, frozen.Start(t.Context()))
+	t.Cleanup(func() {
+		frozen.Stop()
+		frozen.Wait()
+	})
+
+	require.Equal(t, 1, len(app.lastHeaders))
+	got, err := types.HeaderFromProto(app.lastHeaders[0])
+	require.NoError(t, err)
+	require.Equal(t, want.Hash(), got.Hash())
+	require.Equal(t, lastHeight, got.Height)
+	require.Equal(t, frozen.genesisDoc.ChainID, got.ChainID)
+}

sei-tendermint/node/node.go

@@ -507,6 +507,16 @@ func (n *nodeImpl) OnStart(ctx context.Context) error {
 		return fmt.Errorf("cannot load state: %w", err)
 	}
 
+	// A frozen node never commits another block, so after a restart nothing
+	// would initialize the app (e.g. its query context and EVM RPC servers).
+	if n.freezeHeight > 0 && state.LastBlockHeight > 0 {
+		meta := n.blockStore.LoadBlockMeta(state.LastBlockHeight)
+		if meta == nil {
+			return fmt.Errorf("freeze mode: block %d is missing from the block store", state.LastBlockHeight)
+		}
+		n.rpcEnv.App.InitLastHeader(meta.Header.ToProto())
+	}
+
 	logNodeStartupInfo(state, n.rpcEnv.PubKey, n.config.Mode)
 
 	// TODO: Fetch and provide real options and do proper p2p bootstrapping.