From ed71a1024f4a20992f0f812cddaf45d3554bfd52 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 26 Aug 2026 17:09:46 +0200 Subject: [PATCH 01/12] PLT-1072: Wire rate limiter into gRPC unary+stream interceptors (:9090) Applied per-IP token-bucket admission on native gRPC before handlers run, emitting rpc_rate_limit_rejected_total{plane="grpc"}. Co-authored-by: Cursor --- ratelimiter/method_bucket.go | 59 +++++++ ratelimiter/method_bucket_test.go | 13 ++ sei-cosmos/server/config/config.go | 49 ++++++ sei-cosmos/server/config/config_fuzz_test.go | 29 +++- sei-cosmos/server/config/config_test.go | 5 + .../server/config/testdata/grpc.keys.golden | 4 + .../config/testdata/server_config.golden | 4 + sei-cosmos/server/config/toml.go | 15 ++ sei-cosmos/server/grpc/rate_limit.go | 39 +++++ sei-cosmos/server/grpc/rate_limit_test.go | 162 ++++++++++++++++++ sei-cosmos/server/grpc/server.go | 17 +- 11 files changed, 391 insertions(+), 5 deletions(-) create mode 100644 sei-cosmos/server/grpc/rate_limit.go create mode 100644 sei-cosmos/server/grpc/rate_limit_test.go diff --git a/ratelimiter/method_bucket.go b/ratelimiter/method_bucket.go index e350c9aca5..8c99f717d1 100644 --- a/ratelimiter/method_bucket.go +++ b/ratelimiter/method_bucket.go @@ -5,6 +5,8 @@ import "strings" const ( // PlaneCometBFT is the rate-limit plane label for Tendermint RPC HTTP. PlaneCometBFT = "cometbft" + // PlaneGRPC is the rate-limit plane label for native gRPC (:9090). + PlaneGRPC = "grpc" // rpcMethodBucketOther is the fallback label for unrecognized methods. rpcMethodBucketOther = "other" @@ -75,6 +77,37 @@ var knownCometBFTRPCMethods = map[string]struct{}{ "websocket": {}, } +// knownGRPCServices lists protobuf service names registered on the native gRPC +// server. Rejection metrics on PlaneGRPC record the service name rather than +// the full /service/Method path, keeping OTel attribute cardinality bounded. +var knownGRPCServices = map[string]struct{}{ + "cosmos.auth.v1beta1.Query": {}, + "cosmos.authz.v1beta1.Query": {}, + "cosmos.bank.v1beta1.Query": {}, + "cosmos.base.reflection.v2alpha1.ReflectionService": {}, + "cosmos.base.tendermint.v1beta1.Service": {}, + "cosmos.consensus.v1.Query": {}, + "cosmos.distribution.v1beta1.Query": {}, + "cosmos.evidence.v1beta1.Query": {}, + "cosmos.feegrant.v1beta1.Query": {}, + "cosmos.gov.v1beta1.Query": {}, + "cosmos.mint.v1beta1.Query": {}, + "cosmos.params.v1beta1.Query": {}, + "cosmos.slashing.v1beta1.Query": {}, + "cosmos.staking.v1beta1.Query": {}, + "cosmos.tx.v1beta1.Service": {}, + "cosmos.upgrade.v1beta1.Query": {}, + "cosmos.vesting.v1beta1.Query": {}, + "grpc.reflection.v1alpha.ServerReflection": {}, + "seiprotocol.seichain.epoch.Query": {}, + "seiprotocol.seichain.evm.Query": {}, + "seiprotocol.seichain.mint.Query": {}, + "seiprotocol.seichain.oracle.Query": {}, + "seiprotocol.seichain.tokenfactory.Query": {}, + "cosmos.circuit.v1.Query": {}, + "cosmwasm.wasm.v1.Query": {}, +} + // bucketRPCMethod maps a raw JSON-RPC method name to a low-cardinality label // suitable for OTel/Prometheus metrics. Attacker-controlled method strings // collapse to rpcMethodBucketOther. @@ -85,9 +118,35 @@ func bucketRPCMethod(plane, method string) string { if plane == PlaneCometBFT { return bucketCometBFTRPCMethod(method) } + if plane == PlaneGRPC { + return bucketGRPCMethod(method) + } return bucketNamespacedRPCMethod(method) } +func bucketGRPCMethod(fullMethod string) string { + if fullMethod == "" || len(fullMethod) > maxRPCMethodLen { + return rpcMethodBucketOther + } + svc := grpcServiceFromFullMethod(fullMethod) + if svc == "" || len(svc) > maxRPCMethodLen { + return rpcMethodBucketOther + } + if _, ok := knownGRPCServices[svc]; ok { + return svc + } + return rpcMethodBucketOther +} + +func grpcServiceFromFullMethod(fullMethod string) string { + method := strings.TrimPrefix(fullMethod, "/") + slash := strings.LastIndexByte(method, '/') + if slash <= 0 { + return "" + } + return method[:slash] +} + func bucketCometBFTRPCMethod(method string) string { if method == "" || len(method) > maxRPCMethodLen { return rpcMethodBucketOther diff --git a/ratelimiter/method_bucket_test.go b/ratelimiter/method_bucket_test.go index 100e3bb5ea..9a2778b2fd 100644 --- a/ratelimiter/method_bucket_test.go +++ b/ratelimiter/method_bucket_test.go @@ -61,4 +61,17 @@ func TestBucketRPCMethod_CometBFTUnknownMethods(t *testing.T) { func TestBucketRPCMethod_Invalid(t *testing.T) { require.Equal(t, MethodInvalid, bucketRPCMethod("evm", MethodInvalid)) require.Equal(t, MethodInvalid, bucketRPCMethod(PlaneCometBFT, MethodInvalid)) + require.Equal(t, MethodInvalid, bucketRPCMethod(PlaneGRPC, MethodInvalid)) +} + +func TestBucketRPCMethod_GrpcknownServices(t *testing.T) { + require.Equal(t, "cosmos.bank.v1beta1.Query", bucketRPCMethod(PlaneGRPC, "/cosmos.bank.v1beta1.Query/Balance")) + require.Equal(t, "cosmos.tx.v1beta1.Service", bucketRPCMethod(PlaneGRPC, "/cosmos.tx.v1beta1.Service/Simulate")) +} + +func TestBucketRPCMethod_GrpcUnknownServices(t *testing.T) { + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "")) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "/bogus.Service/Call")) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "not-a-grpc-path")) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, strings.Repeat("a", maxRPCMethodLen+1))) } diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index ee732b85c9..546ddb696f 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/sei-protocol/sei-chain/ratelimiter" storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" "github.com/sei-protocol/sei-chain/sei-cosmos/telemetry" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" @@ -258,6 +259,33 @@ type GRPCConfig struct { // KeepalivePermitWithoutStream defines whether the server allows keepalive // pings even when there are no active streams. KeepalivePermitWithoutStream bool `mapstructure:"keepalive-permit-without-stream"` + + // IPRateLimitRPS is the per-IP sustained request rate in requests/second for + // native gRPC (:9090). Zero disables the token bucket (Allow always returns + // true) and does not bypass the admission interceptor when + // rate-limiting-enabled is true. + IPRateLimitRPS float64 `mapstructure:"ip-rate-limit-rps"` + + // IPRateLimitBurst is the maximum per-IP burst size. Zero disables the token + // bucket (same effect as ip-rate-limit-rps = 0) and does not bypass the + // admission interceptor when rate-limiting-enabled is true. + IPRateLimitBurst int `mapstructure:"ip-rate-limit-burst"` + + // RateLimitingEnabled is the master switch for gRPC rate-limit admission. + RateLimitingEnabled bool `mapstructure:"rate-limiting-enabled"` + + // TrustedProxyCIDRs lists CIDRs whose x-forwarded-for metadata is trusted + // when resolving the client IP for rate limiting. Empty means trust no proxy. + TrustedProxyCIDRs []string `mapstructure:"trusted-proxy-cidrs"` +} + +// RateLimiterConfig builds the ratelimiter.Config used by native gRPC admission. +func (c GRPCConfig) RateLimiterConfig() ratelimiter.Config { + return ratelimiter.Config{ + RPS: c.IPRateLimitRPS, + Burst: c.IPRateLimitBurst, + TrustedProxyCIDRs: c.TrustedProxyCIDRs, + } } // GRPCWebConfig defines configuration for the gRPC-web server. @@ -386,6 +414,10 @@ func DefaultConfig() *Config { KeepaliveTimeout: DefaultGRPCKeepaliveTimeout, KeepaliveMinTime: DefaultGRPCKeepaliveMinTime, KeepalivePermitWithoutStream: DefaultGRPCKeepalivePermitWithoutStream, + IPRateLimitRPS: ratelimiter.DefaultRPS, + IPRateLimitBurst: ratelimiter.DefaultBurst, + RateLimitingEnabled: false, + TrustedProxyCIDRs: nil, }, Rosetta: RosettaConfig{ Enable: false, @@ -572,6 +604,19 @@ func GetConfig(v *viper.Viper) (Config, error) { grpcMaxConnectionAge := clampNonNegativeDuration(v.GetDuration("grpc.max-connection-age"), DefaultGRPCMaxConnectionAge) grpcMaxConnectionAgeGrace := clampNonNegativeDuration(v.GetDuration("grpc.max-connection-age-grace"), DefaultGRPCMaxConnectionAgeGrace) + grpcIPRateLimitRPS := ratelimiter.DefaultRPS + if v.IsSet("grpc.ip-rate-limit-rps") { + grpcIPRateLimitRPS = v.GetFloat64("grpc.ip-rate-limit-rps") + } + grpcIPRateLimitBurst := ratelimiter.DefaultBurst + if v.IsSet("grpc.ip-rate-limit-burst") { + grpcIPRateLimitBurst = v.GetInt("grpc.ip-rate-limit-burst") + } + grpcTrustedProxyCIDRs := []string(nil) + if v.IsSet("grpc.trusted-proxy-cidrs") { + grpcTrustedProxyCIDRs = v.GetStringSlice("grpc.trusted-proxy-cidrs") + } + cfg := Config{ BaseConfig: BaseConfig{ MinGasPrices: v.GetString("minimum-gas-prices"), @@ -627,6 +672,10 @@ func GetConfig(v *viper.Viper) (Config, error) { KeepaliveTimeout: grpcKeepaliveTimeout, KeepaliveMinTime: grpcKeepaliveMinTime, KeepalivePermitWithoutStream: v.GetBool("grpc.keepalive-permit-without-stream"), + IPRateLimitRPS: grpcIPRateLimitRPS, + IPRateLimitBurst: grpcIPRateLimitBurst, + RateLimitingEnabled: v.GetBool("grpc.rate-limiting-enabled"), + TrustedProxyCIDRs: grpcTrustedProxyCIDRs, }, GRPCWeb: GRPCWebConfig{ Enable: v.GetBool("grpc-web.enable"), diff --git a/sei-cosmos/server/config/config_fuzz_test.go b/sei-cosmos/server/config/config_fuzz_test.go index 04ef74c8a4..d62525eaea 100644 --- a/sei-cosmos/server/config/config_fuzz_test.go +++ b/sei-cosmos/server/config/config_fuzz_test.go @@ -1166,6 +1166,10 @@ func TestGetConfigAbsentSectionDivergences(t *testing.T) { "grpc.keepalive-permit-without-stream", cfg.GRPC.KeepalivePermitWithoutStream, def.GRPC.KeepalivePermitWithoutStream, false, }, + { + "grpc.rate-limiting-enabled", + cfg.GRPC.RateLimitingEnabled, def.GRPC.RateLimitingEnabled, false, + }, {"telemetry.service-name", cfg.Telemetry.ServiceName, def.Telemetry.ServiceName, false}, {"telemetry.enable-hostname", cfg.Telemetry.EnableHostname, def.Telemetry.EnableHostname, false}, { @@ -1354,7 +1358,7 @@ func TestBaseConfigManifestNamesEveryField(t *testing.T) { ) } -// grpcKeys covers the three [grpc] keys read as plain casts. +// grpcKeys covers the four [grpc] keys read as plain casts. // // The section is where the guarding in this reader is most complete, which is why only three keys // are rows. Eight others are read behind v.IsSet or through clampNonNegativeDuration and so resolve @@ -1380,6 +1384,12 @@ var grpcKeys = []configtest.KeySpec{ Why: "whether a client may ping with no active stream; false either way, so this row states " + "the key is read rather than recording a divergence", }, + { + Key: "grpc.rate-limiting-enabled", Path: "RateLimitingEnabled", + Cast: configtest.CastBool, Unguarded: true, + Why: "the declared default is false and an absent key resolves false, so this row states " + + "the key is read rather than recording a divergence", + }, } // grpcKeysWithTargetsOfTheirOwn are the [grpc] keys whose resolution a row cannot describe, recorded @@ -1406,6 +1416,9 @@ var grpcKeysWithTargetsOfTheirOwn = []configtest.KeyName{ "grpc.keepalive-time", "grpc.keepalive-timeout", "grpc.keepalive-min-time", + "grpc.ip-rate-limit-rps", + "grpc.ip-rate-limit-burst", + "grpc.trusted-proxy-cidrs", } func readGRPC(t testing.TB) func(configtest.AppOpts) (any, error) { @@ -1419,6 +1432,7 @@ func FuzzGRPCConfig(f *testing.F) { seeds.AddRow(uint(0), fuzzing.KindBool, "", int64(0), true) seeds.AddRow(uint(1), fuzzing.KindString, "127.0.0.1:19090", int64(0), false) seeds.AddRow(uint(2), fuzzing.KindBool, "", int64(0), true) + seeds.AddRow(uint(3), fuzzing.KindBool, "", int64(0), true) configtest.CheckEveryRowHasADiscriminatingSeed(f, "grpc", readGRPC(f), grpcKeys, seeds, grpcKeysWithTargetsOfTheirOwn...) @@ -1429,8 +1443,8 @@ func FuzzGRPCConfig(f *testing.F) { }) } -// TestGRPCKeyNamesMatchTheRecordedNames pins all eleven [grpc] key names, the three rows and the -// eight driven elsewhere. +// TestGRPCKeyNamesMatchTheRecordedNames pins all fifteen [grpc] key names, the four rows and the +// eleven driven elsewhere. // // The eight had no record before this, because their target carries a local struct rather than a // KeySpec table, so nothing held their spelling. That is the gap this closes. @@ -1450,6 +1464,9 @@ func TestGRPCManifestNamesEveryField(t *testing.T) { "KeepaliveTime", "KeepaliveTimeout", "KeepaliveMinTime", + "IPRateLimitRPS", + "IPRateLimitBurst", + "TrustedProxyCIDRs", ) } @@ -1485,6 +1502,8 @@ func TestGetConfigGRPCAbsentReads(t *testing.T) { {"grpc.keepalive-time", got.KeepaliveTime, def.KeepaliveTime}, {"grpc.keepalive-timeout", got.KeepaliveTimeout, def.KeepaliveTimeout}, {"grpc.keepalive-min-time", got.KeepaliveMinTime, def.KeepaliveMinTime}, + {"grpc.ip-rate-limit-rps", got.IPRateLimitRPS, def.IPRateLimitRPS}, + {"grpc.ip-rate-limit-burst", got.IPRateLimitBurst, def.IPRateLimitBurst}, } { if c.absent != c.declared { t.Errorf("an absent %s resolved to %v rather than the declared %v, so its v.IsSet guard "+ @@ -1493,6 +1512,10 @@ func TestGetConfigGRPCAbsentReads(t *testing.T) { } } + if got.TrustedProxyCIDRs != nil { + t.Errorf("an absent grpc.trusted-proxy-cidrs resolved to %v rather than nil", got.TrustedProxyCIDRs) + } + // Read unconditionally and clamped. Nothing guards these, so the assertion is on the coincidence // itself: the declared default is the getter's zero, which is why an absent key looks correct. for _, c := range []struct { diff --git a/sei-cosmos/server/config/config_test.go b/sei-cosmos/server/config/config_test.go index 38a1abc0b9..1d1f09072d 100644 --- a/sei-cosmos/server/config/config_test.go +++ b/sei-cosmos/server/config/config_test.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/require" + "github.com/sei-protocol/sei-chain/ratelimiter" storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config" @@ -81,6 +82,10 @@ func TestDefaultGRPCConfig(t *testing.T) { require.Equal(t, DefaultGRPCKeepaliveTimeout, cfg.GRPC.KeepaliveTimeout) require.Equal(t, DefaultGRPCKeepaliveMinTime, cfg.GRPC.KeepaliveMinTime) require.Equal(t, DefaultGRPCKeepalivePermitWithoutStream, cfg.GRPC.KeepalivePermitWithoutStream) + require.Equal(t, ratelimiter.DefaultRPS, cfg.GRPC.IPRateLimitRPS) + require.Equal(t, ratelimiter.DefaultBurst, cfg.GRPC.IPRateLimitBurst) + require.False(t, cfg.GRPC.RateLimitingEnabled) + require.Nil(t, cfg.GRPC.TrustedProxyCIDRs) } // seedViperWithDefaultConfig renders the default app config template and reads diff --git a/sei-cosmos/server/config/testdata/grpc.keys.golden b/sei-cosmos/server/config/testdata/grpc.keys.golden index f3bb885cbe..b06deb44ed 100644 --- a/sei-cosmos/server/config/testdata/grpc.keys.golden +++ b/sei-cosmos/server/config/testdata/grpc.keys.golden @@ -1,6 +1,7 @@ "grpc.enable" "grpc.address" "grpc.keepalive-permit-without-stream" +"grpc.rate-limiting-enabled" # keys with a target of their own "grpc.max-recv-msg-size" "grpc.max-open-connections" @@ -10,3 +11,6 @@ "grpc.keepalive-time" "grpc.keepalive-timeout" "grpc.keepalive-min-time" +"grpc.ip-rate-limit-rps" +"grpc.ip-rate-limit-burst" +"grpc.trusted-proxy-cidrs" diff --git a/sei-cosmos/server/config/testdata/server_config.golden b/sei-cosmos/server/config/testdata/server_config.golden index e0ec8c8138..e6a63e08b2 100644 --- a/sei-cosmos/server/config/testdata/server_config.golden +++ b/sei-cosmos/server/config/testdata/server_config.golden @@ -38,6 +38,10 @@ GRPC.KeepaliveTime = time.Duration(2h0m0s) GRPC.KeepaliveTimeout = time.Duration(20s) GRPC.KeepaliveMinTime = time.Duration(5m0s) GRPC.KeepalivePermitWithoutStream = bool(false) +GRPC.IPRateLimitRPS = float64(200) +GRPC.IPRateLimitBurst = int(400) +GRPC.RateLimitingEnabled = bool(false) +GRPC.TrustedProxyCIDRs = Rosetta.Address = string(":8080") Rosetta.Blockchain = string("app") Rosetta.Network = string("network") diff --git a/sei-cosmos/server/config/toml.go b/sei-cosmos/server/config/toml.go index eb7d7f4ea0..c61b666726 100644 --- a/sei-cosmos/server/config/toml.go +++ b/sei-cosmos/server/config/toml.go @@ -260,6 +260,21 @@ keepalive-min-time = "{{ .GRPC.KeepaliveMinTime }}" # KeepalivePermitWithoutStream defines whether the server allows keepalive pings even when there are no active streams. keepalive-permit-without-stream = {{ .GRPC.KeepalivePermitWithoutStream }} +# ip-rate-limit-rps is the per-IP sustained request rate in requests/second for native gRPC (:9090). +# Zero disables per-IP throttling; set rate-limiting-enabled = false for a full bypass. +ip-rate-limit-rps = {{ .GRPC.IPRateLimitRPS }} + +# ip-rate-limit-burst is the maximum per-IP burst above the sustained rate. +# Zero disables per-IP throttling (same effect as ip-rate-limit-rps = 0). +ip-rate-limit-burst = {{ .GRPC.IPRateLimitBurst }} + +# rate-limiting-enabled is the master switch for gRPC rate-limit admission interceptors. +rate-limiting-enabled = {{ .GRPC.RateLimitingEnabled }} + +# trusted-proxy-cidrs lists CIDRs whose x-forwarded-for metadata is trusted when +# resolving the client IP for rate limiting. Empty means trust no proxy. +trusted-proxy-cidrs = [{{- range $i, $c := .GRPC.TrustedProxyCIDRs }}{{- if $i }}, {{ end }}"{{ $c }}"{{- end }}] + ############################################################################### ### gRPC Web Configuration (Auto-managed) ### ############################################################################### diff --git a/sei-cosmos/server/grpc/rate_limit.go b/sei-cosmos/server/grpc/rate_limit.go new file mode 100644 index 0000000000..f9e5bee2a1 --- /dev/null +++ b/sei-cosmos/server/grpc/rate_limit.go @@ -0,0 +1,39 @@ +package grpc + +import ( + "context" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/sei-protocol/sei-chain/ratelimiter" +) + +var errRateLimited = status.Error(codes.ResourceExhausted, "too many requests") + +// UnaryRateLimitInterceptor returns a server interceptor that applies per-IP +// token-bucket rate limiting before the handler runs. registry must be non-nil. +func UnaryRateLimitInterceptor(registry *ratelimiter.Registry) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + ip := registry.IPFromGRPCContext(ctx) + if !registry.Allow(ctx, ip, ratelimiter.PlaneGRPC, info.FullMethod) { + return nil, errRateLimited + } + return handler(ctx, req) + } +} + +// StreamRateLimitInterceptor returns a server interceptor that applies per-IP +// token-bucket rate limiting when a client stream is established. registry must +// be non-nil. +func StreamRateLimitInterceptor(registry *ratelimiter.Registry) grpc.StreamServerInterceptor { + return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + ctx := ss.Context() + ip := registry.IPFromGRPCContext(ctx) + if !registry.Allow(ctx, ip, ratelimiter.PlaneGRPC, info.FullMethod) { + return errRateLimited + } + return handler(srv, ss) + } +} diff --git a/sei-cosmos/server/grpc/rate_limit_test.go b/sei-cosmos/server/grpc/rate_limit_test.go new file mode 100644 index 0000000000..56c9e8656b --- /dev/null +++ b/sei-cosmos/server/grpc/rate_limit_test.go @@ -0,0 +1,162 @@ +package grpc + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + + "github.com/sei-protocol/sei-chain/ratelimiter" +) + +func cfg(rps float64, burst int, cidrs ...string) ratelimiter.Config { + return ratelimiter.Config{RPS: rps, Burst: burst, TrustedProxyCIDRs: cidrs} +} + +func mustNewRegistry(t *testing.T, c ratelimiter.Config) *ratelimiter.Registry { + t.Helper() + r, err := ratelimiter.New(c) + require.NoError(t, err) + return r +} + +type mockAddr string + +func (a mockAddr) Network() string { return "tcp" } +func (a mockAddr) String() string { return string(a) } + +func grpcCtx(ctx context.Context, peerAddr string, xff ...string) context.Context { + ctx = peer.NewContext(ctx, &peer.Peer{Addr: mockAddr(peerAddr)}) + if len(xff) > 0 { + md := metadata.MD{"x-forwarded-for": xff} + ctx = metadata.NewIncomingContext(ctx, md) + } + return ctx +} + +func TestUnaryRateLimitInterceptor_AllowThenReject(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1)) + ic := UnaryRateLimitInterceptor(reg) + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return "ok", nil + } + info := &grpc.UnaryServerInfo{FullMethod: "/cosmos.bank.v1beta1.Query/Balance"} + ctx := grpcCtx(t.Context(), "10.0.0.1:9000") + + _, err := ic(ctx, nil, info, handler) + require.NoError(t, err) + + _, err = ic(ctx, nil, info, handler) + require.Error(t, err) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +func TestUnaryRateLimitInterceptor_PerIPIsolation(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1)) + ic := UnaryRateLimitInterceptor(reg) + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return "ok", nil + } + info := &grpc.UnaryServerInfo{FullMethod: "/cosmos.bank.v1beta1.Query/Balance"} + + _, err := ic(grpcCtx(t.Context(), "1.1.1.1:9000"), nil, info, handler) + require.NoError(t, err) + _, err = ic(grpcCtx(t.Context(), "1.1.1.1:9000"), nil, info, handler) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + + _, err = ic(grpcCtx(t.Context(), "2.2.2.2:9000"), nil, info, handler) + require.NoError(t, err) +} + +func TestUnaryRateLimitInterceptor_TrustedProxyXFF(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1, "10.0.0.0/8")) + ic := UnaryRateLimitInterceptor(reg) + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return "ok", nil + } + info := &grpc.UnaryServerInfo{FullMethod: "/cosmos.bank.v1beta1.Query/Balance"} + ctx := grpcCtx(t.Context(), "10.0.0.2:9000", "203.0.113.5, 10.0.0.2") + + _, err := ic(ctx, nil, info, handler) + require.NoError(t, err) + _, err = ic(ctx, nil, info, handler) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +type mockServerStream struct { + grpc.ServerStream + ctx context.Context +} + +func (m mockServerStream) Context() context.Context { return m.ctx } + +func TestStreamRateLimitInterceptor_AllowThenReject(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1)) + ic := StreamRateLimitInterceptor(reg) + handler := func(srv interface{}, stream grpc.ServerStream) error { + return nil + } + info := &grpc.StreamServerInfo{FullMethod: "/cosmos.bank.v1beta1.Query/Balance"} + stream := mockServerStream{ctx: grpcCtx(t.Context(), "10.0.0.1:9000")} + + require.NoError(t, ic(nil, stream, info, handler)) + + stream = mockServerStream{ctx: grpcCtx(t.Context(), "10.0.0.1:9000")} + err := ic(nil, stream, info, handler) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +func TestUnaryRateLimitInterceptor_ZeroRPSBypassesBucket(t *testing.T) { + reg := mustNewRegistry(t, cfg(0, 10)) + ic := UnaryRateLimitInterceptor(reg) + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return "ok", nil + } + info := &grpc.UnaryServerInfo{FullMethod: "/cosmos.bank.v1beta1.Query/Balance"} + ctx := grpcCtx(t.Context(), "10.0.0.1:9000") + + for range 10 { + _, err := ic(ctx, nil, info, handler) + require.NoError(t, err) + } +} + +func TestUnaryRateLimitInterceptor_NoPeerStillRateLimited(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1)) + ic := UnaryRateLimitInterceptor(reg) + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return "ok", nil + } + info := &grpc.UnaryServerInfo{FullMethod: "/cosmos.bank.v1beta1.Query/Balance"} + ctx := t.Context() + + _, err := ic(ctx, nil, info, handler) + require.NoError(t, err) + _, err = ic(ctx, nil, info, handler) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +func TestUnaryRateLimitInterceptor_RealTCPAddr(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1)) + ic := UnaryRateLimitInterceptor(reg) + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return "ok", nil + } + info := &grpc.UnaryServerInfo{FullMethod: "/cosmos.bank.v1beta1.Query/Balance"} + + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + ctx := peer.NewContext(t.Context(), &peer.Peer{Addr: ln.Addr()}) + + _, err = ic(ctx, nil, info, handler) + require.NoError(t, err) + _, err = ic(ctx, nil, info, handler) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} diff --git a/sei-cosmos/server/grpc/server.go b/sei-cosmos/server/grpc/server.go index cf24007c15..e7550692a8 100644 --- a/sei-cosmos/server/grpc/server.go +++ b/sei-cosmos/server/grpc/server.go @@ -10,6 +10,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/keepalive" + "github.com/sei-protocol/sei-chain/ratelimiter" "github.com/sei-protocol/sei-chain/sei-cosmos/client" "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" "github.com/sei-protocol/sei-chain/sei-cosmos/server/grpc/gogoreflection" @@ -25,7 +26,7 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config maxRecvMsgSize = config.DefaultGRPCMaxRecvMsgSize } - grpcSrv := grpc.NewServer( + serverOpts := []grpc.ServerOption{ grpc.MaxConcurrentStreams(100), // MaxRecvMsgSize bounds per-request memory allocation before the rate // limiter fires, preventing an oversized request from exhausting memory. @@ -41,7 +42,19 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config MinTime: cfg.KeepaliveMinTime, PermitWithoutStream: cfg.KeepalivePermitWithoutStream, }), - ) + } + if cfg.RateLimitingEnabled { + rateLimitRegistry, err := ratelimiter.New(cfg.RateLimiterConfig()) + if err != nil { + return nil, fmt.Errorf("grpc rate limiter: %w", err) + } + serverOpts = append(serverOpts, + grpc.ChainUnaryInterceptor(UnaryRateLimitInterceptor(rateLimitRegistry)), + grpc.ChainStreamInterceptor(StreamRateLimitInterceptor(rateLimitRegistry)), + ) + } + + grpcSrv := grpc.NewServer(serverOpts...) app.RegisterGRPCServer(grpcSrv) // reflection allows consumers to build dynamic clients that can write // to any cosmos-sdk application without relying on application packages at compile time From c6808999a10d0e64896227897e09200e98e73b35 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Fri, 28 Aug 2026 15:32:52 +0200 Subject: [PATCH 02/12] PLT-1072: Apply server-level unary interceptors to gRPC queries BaseApp.RegisterGRPCServer rebuilt every grpc.MethodDesc with a handler that discarded the grpc.UnaryServerInterceptor grpc-go passes in. Since grpc-go hands s.opts.unaryInt to MethodDesc.Handler rather than invoking it itself, every service registered through RegisterGRPCServer -- all module Query services, cosmos.tx.v1beta1.Service and cosmos.base.tendermint.v1beta1.Service -- ran no server-level interceptor. The rate limiter wired onto :9090 therefore covered only the directly-registered reflection service. Thread the incoming interceptor into the chain at that wrapper handler, the single point every unary query passes through, so any grpc.ChainUnaryInterceptor applies without each registration site re-adding it. The query-context interceptor runs last, so admission control rejects a call before it costs a query context. Add tests that register through RegisterGRPCServer on a real grpc.Server and dial it, covering both the generic interceptor contract and the rate limiter reaching ResourceExhausted once a burst is spent. The existing interceptor tests invoked the closures directly, which is why the bypass was invisible. Also correct the stale [grpc] key counts and clamp line reference in the configuration characterization suite. Co-Authored-By: Claude Opus 5 (1M context) --- sei-cosmos/baseapp/grpcserver.go | 20 +++-- sei-cosmos/baseapp/grpcserver_test.go | 88 +++++++++++++++++++ sei-cosmos/server/config/config_fuzz_test.go | 8 +- .../server/grpc/rate_limit_dispatch_test.go | 76 ++++++++++++++++ 4 files changed, 183 insertions(+), 9 deletions(-) create mode 100644 sei-cosmos/baseapp/grpcserver_test.go create mode 100644 sei-cosmos/server/grpc/rate_limit_dispatch_test.go diff --git a/sei-cosmos/baseapp/grpcserver.go b/sei-cosmos/baseapp/grpcserver.go index d00a56a6e3..805c5ce923 100644 --- a/sei-cosmos/baseapp/grpcserver.go +++ b/sei-cosmos/baseapp/grpcserver.go @@ -78,11 +78,8 @@ func (app *BaseApp) RegisterGRPCServer(server gogogrpc.Server) { methodHandler := method.Handler newMethods[i] = grpc.MethodDesc{ MethodName: method.MethodName, - Handler: func(srv interface{}, ctx context.Context, dec func(interface{}) error, _ grpc.UnaryServerInterceptor) (interface{}, error) { - return methodHandler(srv, ctx, dec, grpcmiddleware.ChainUnaryServer( - grpcrecovery.UnaryServerInterceptor(), - interceptor, - )) + Handler: func(srv interface{}, ctx context.Context, dec func(interface{}) error, serverInterceptor grpc.UnaryServerInterceptor) (interface{}, error) { + return methodHandler(srv, ctx, dec, chainQueryInterceptors(serverInterceptor, interceptor)) }, } } @@ -98,3 +95,16 @@ func (app *BaseApp) RegisterGRPCServer(server gogogrpc.Server) { server.RegisterService(newDesc, data.handler) } } + +// chainQueryInterceptors returns the chain a unary query runs through: panic +// recovery, then serverInterceptor when non-nil, then queryCtx last so a rejected +// call never costs a query context. +func chainQueryInterceptors(serverInterceptor, queryCtx grpc.UnaryServerInterceptor) grpc.UnaryServerInterceptor { + chain := make([]grpc.UnaryServerInterceptor, 0, 3) + chain = append(chain, grpcrecovery.UnaryServerInterceptor()) + if serverInterceptor != nil { + chain = append(chain, serverInterceptor) + } + chain = append(chain, queryCtx) + return grpcmiddleware.ChainUnaryServer(chain...) +} diff --git a/sei-cosmos/baseapp/grpcserver_test.go b/sei-cosmos/baseapp/grpcserver_test.go new file mode 100644 index 0000000000..5c2ae4a9d3 --- /dev/null +++ b/sei-cosmos/baseapp/grpcserver_test.go @@ -0,0 +1,88 @@ +package baseapp + +import ( + "context" + "net" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + "github.com/sei-protocol/sei-chain/sei-cosmos/codec/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/testutil/testdata" +) + +// serveTestQuery registers the testdata Query service on app, exposes it on a real +// grpc.Server built with serverOpts, and returns a client dialled to it. +// +// The service is registered through RegisterGRPCServer so calls travel grpc-go's +// own dispatch path. An interceptor supplied through serverOpts reaches handlers +// only if that path is intact, which invoking the interceptor closure directly +// cannot show. +func serveTestQuery(t *testing.T, app *BaseApp, serverOpts ...grpc.ServerOption) testdata.QueryClient { + t.Helper() + + interfaceRegistry := types.NewInterfaceRegistry() + testdata.RegisterInterfaces(interfaceRegistry) + app.SetInterfaceRegistry(interfaceRegistry) + testdata.RegisterQueryServer(app.GRPCQueryRouter(), testdata.QueryImpl{}) + + srv := grpc.NewServer(serverOpts...) + app.RegisterGRPCServer(srv) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = srv.Serve(listener) }() + t.Cleanup(srv.Stop) + + conn, err := grpc.Dial(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + return testdata.NewQueryClient(conn) +} + +func TestRegisterGRPCServerAppliesServerInterceptor(t *testing.T) { + var calls atomic.Int64 + spy := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + calls.Add(1) + return handler(ctx, req) + } + + client := serveTestQuery(t, setupBaseApp(t), grpc.ChainUnaryInterceptor(spy)) + + res, err := client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.NoError(t, err) + require.Equal(t, "hello", res.Message) + require.Equal(t, int64(1), calls.Load(), "server-level interceptor never reached the query handler") +} + +func TestRegisterGRPCServerServerInterceptorCanRejectQuery(t *testing.T) { + var handlerCalls atomic.Int64 + reject := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + return nil, status.Error(codes.ResourceExhausted, "too many requests") + } + count := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + handlerCalls.Add(1) + return handler(ctx, req) + } + + client := serveTestQuery(t, setupBaseApp(t), grpc.ChainUnaryInterceptor(reject, count)) + + _, err := client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.Error(t, err) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + require.Zero(t, handlerCalls.Load(), "rejected query still ran the rest of the chain") +} + +func TestRegisterGRPCServerWithoutServerInterceptor(t *testing.T) { + client := serveTestQuery(t, setupBaseApp(t)) + + res, err := client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.NoError(t, err) + require.Equal(t, "hello", res.Message) +} diff --git a/sei-cosmos/server/config/config_fuzz_test.go b/sei-cosmos/server/config/config_fuzz_test.go index d62525eaea..dba6f668fd 100644 --- a/sei-cosmos/server/config/config_fuzz_test.go +++ b/sei-cosmos/server/config/config_fuzz_test.go @@ -1360,8 +1360,8 @@ func TestBaseConfigManifestNamesEveryField(t *testing.T) { // grpcKeys covers the four [grpc] keys read as plain casts. // -// The section is where the guarding in this reader is most complete, which is why only three keys -// are rows. Eight others are read behind v.IsSet or through clampNonNegativeDuration and so resolve +// The section is where the guarding in this reader is most complete, which is why only four keys +// are rows. Eleven others are read behind v.IsSet or through clampNonNegativeDuration and so resolve // an absent key to the in-code default rather than to a zero; CheckRow would predict the wrong // resolution for each, so they are driven by FuzzGetConfigGRPCDurationClamps and // TestGetConfigGRPCAbsentReads and recorded by name below. @@ -1395,9 +1395,9 @@ var grpcKeys = []configtest.KeySpec{ // grpcKeysWithTargetsOfTheirOwn are the [grpc] keys whose resolution a row cannot describe, recorded // for their names alone. // -// Six are read behind v.IsSet, so an absent key keeps the in-code default. The other two, +// Nine are read behind v.IsSet, so an absent key keeps the in-code default. The other two, // max-connection-age and max-connection-age-grace, are read unconditionally through the clamp -// (config.go:551-552), and their absent value matches the declared default only because both +// (config.go:604-605), and their absent value matches the declared default only because both // defaults are 0. The clamp rescues a negative value and does nothing for an absent one, so they are // unguarded reads whose clobber is invisible. TestGetConfigGRPCAbsentReads holds the two groups // apart for that reason. diff --git a/sei-cosmos/server/grpc/rate_limit_dispatch_test.go b/sei-cosmos/server/grpc/rate_limit_dispatch_test.go new file mode 100644 index 0000000000..18e92b0e79 --- /dev/null +++ b/sei-cosmos/server/grpc/rate_limit_dispatch_test.go @@ -0,0 +1,76 @@ +package grpc + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + "github.com/sei-protocol/sei-chain/ratelimiter" + "github.com/sei-protocol/sei-chain/sei-cosmos/baseapp" + codectypes "github.com/sei-protocol/sei-chain/sei-cosmos/codec/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/testutil" + "github.com/sei-protocol/sei-chain/sei-cosmos/testutil/testdata" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" +) + +// serveRateLimitedQuery exposes the testdata Query service behind registry on a +// real grpc.Server and returns a client dialled to it. +// +// Registration goes through BaseApp.RegisterGRPCServer, the same call +// StartGRPCServer makes, so the interceptor is admitted the way it is on a live +// node rather than by invoking its closure directly. +func serveRateLimitedQuery(t *testing.T, registry *ratelimiter.Registry) testdata.QueryClient { + t.Helper() + + app := baseapp.NewBaseApp(t.Name(), dbm.NewMemDB(), nil, nil, &testutil.TestAppOpts{}) + app.MountStores(sdk.NewKVStoreKey("test")) + require.NoError(t, app.LoadLatestVersion()) + + interfaceRegistry := codectypes.NewInterfaceRegistry() + testdata.RegisterInterfaces(interfaceRegistry) + app.SetInterfaceRegistry(interfaceRegistry) + testdata.RegisterQueryServer(app.GRPCQueryRouter(), testdata.QueryImpl{}) + + srv := grpc.NewServer(grpc.ChainUnaryInterceptor(UnaryRateLimitInterceptor(registry))) + app.RegisterGRPCServer(srv) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = srv.Serve(listener) }() + t.Cleanup(srv.Stop) + + conn, err := grpc.Dial(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + return testdata.NewQueryClient(conn) +} + +func TestUnaryRateLimitInterceptor_RegisteredQueryServiceAllowThenReject(t *testing.T) { + client := serveRateLimitedQuery(t, mustNewRegistry(t, cfg(0.001, 1))) + + res, err := client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.NoError(t, err) + require.Equal(t, "hello", res.Message) + + _, err = client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.Error(t, err) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +func TestUnaryRateLimitInterceptor_RegisteredQueryServiceUnlimited(t *testing.T) { + client := serveRateLimitedQuery(t, mustNewRegistry(t, cfg(1000, 1000))) + + for i := 0; i < 5; i++ { + res, err := client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.NoError(t, err) + require.Equal(t, "hello", res.Message) + } +} From 4be20b24eca08c4021c7fe47f045f45fa75690b5 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Fri, 28 Aug 2026 15:57:25 +0200 Subject: [PATCH 03/12] PLT-1072: Align gRPC rate-limit config registry and metrics bucketing Register the four new [grpc] keys in config/cosmosbase so characterization tests stay in sync, fix config-test doc/grouping drift, and correct knownGRPCServices for reflection metrics. Co-authored-by: Cursor --- config/cosmosbase/agreement_test.go | 4 ++++ config/cosmosbase/cosmosbase.go | 2 +- config/cosmosbase/cosmosbase_test.go | 2 ++ ratelimiter/method_bucket.go | 10 +++------- sei-cosmos/server/config/config_fuzz_test.go | 8 ++++---- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/config/cosmosbase/agreement_test.go b/config/cosmosbase/agreement_test.go index 103adeb464..53e35d716e 100644 --- a/config/cosmosbase/agreement_test.go +++ b/config/cosmosbase/agreement_test.go @@ -110,6 +110,10 @@ func readerValues(t *testing.T) map[string]string { "grpc.keepalive-timeout": fmt.Sprint(cfg.GRPC.KeepaliveTimeout), "grpc.keepalive-min-time": fmt.Sprint(cfg.GRPC.KeepaliveMinTime), "grpc.keepalive-permit-without-stream": fmt.Sprint(cfg.GRPC.KeepalivePermitWithoutStream), + "grpc.ip-rate-limit-rps": fmt.Sprint(cfg.GRPC.IPRateLimitRPS), + "grpc.ip-rate-limit-burst": fmt.Sprint(cfg.GRPC.IPRateLimitBurst), + "grpc.rate-limiting-enabled": fmt.Sprint(cfg.GRPC.RateLimitingEnabled), + "grpc.trusted-proxy-cidrs": fmt.Sprint(cfg.GRPC.TrustedProxyCIDRs), "telemetry.service-name": fmt.Sprint(cfg.Telemetry.ServiceName), "telemetry.enabled": fmt.Sprint(cfg.Telemetry.Enabled), "telemetry.enable-hostname": fmt.Sprint(cfg.Telemetry.EnableHostname), diff --git a/config/cosmosbase/cosmosbase.go b/config/cosmosbase/cosmosbase.go index f524a03447..c35b14e4d4 100644 --- a/config/cosmosbase/cosmosbase.go +++ b/config/cosmosbase/cosmosbase.go @@ -96,7 +96,7 @@ func apiDefaults(mode registry.Mode) any { return forMode(mode).API } // interface follows and for the same reason. The upstream default is on for every kind, so declaring that // would state an open interface on the nodes meant to expose the least. // -// Six of these eleven keys are read only when the key is present. Two more are durations read through a +// Nine of these fifteen keys are read only when the key is present. Two more are durations read through a // clamp that rescues a negative value and does nothing for an absent one, so those two are unguarded and // their clobber leaves no trace. The durations are declared as durations and written into a file as text, // which is the shape the reader parses back. diff --git a/config/cosmosbase/cosmosbase_test.go b/config/cosmosbase/cosmosbase_test.go index b8199582fa..29ff424f3b 100644 --- a/config/cosmosbase/cosmosbase_test.go +++ b/config/cosmosbase/cosmosbase_test.go @@ -77,6 +77,8 @@ func TestTheGRPCKeysAreTheOnesItsReaderResolves(t *testing.T) { "grpc.max-connection-idle", "grpc.max-connection-age", "grpc.max-connection-age-grace", "grpc.keepalive-time", "grpc.keepalive-timeout", "grpc.keepalive-min-time", "grpc.keepalive-permit-without-stream", + "grpc.ip-rate-limit-rps", "grpc.ip-rate-limit-burst", "grpc.rate-limiting-enabled", + "grpc.trusted-proxy-cidrs", }) } diff --git a/ratelimiter/method_bucket.go b/ratelimiter/method_bucket.go index 8c99f717d1..ed02c1120d 100644 --- a/ratelimiter/method_bucket.go +++ b/ratelimiter/method_bucket.go @@ -84,28 +84,24 @@ var knownGRPCServices = map[string]struct{}{ "cosmos.auth.v1beta1.Query": {}, "cosmos.authz.v1beta1.Query": {}, "cosmos.bank.v1beta1.Query": {}, + "cosmos.base.reflection.v1beta1.ReflectionService": {}, "cosmos.base.reflection.v2alpha1.ReflectionService": {}, "cosmos.base.tendermint.v1beta1.Service": {}, - "cosmos.consensus.v1.Query": {}, "cosmos.distribution.v1beta1.Query": {}, "cosmos.evidence.v1beta1.Query": {}, - "cosmos.feegrant.v1beta1.Query": {}, "cosmos.gov.v1beta1.Query": {}, - "cosmos.mint.v1beta1.Query": {}, "cosmos.params.v1beta1.Query": {}, "cosmos.slashing.v1beta1.Query": {}, "cosmos.staking.v1beta1.Query": {}, "cosmos.tx.v1beta1.Service": {}, "cosmos.upgrade.v1beta1.Query": {}, - "cosmos.vesting.v1beta1.Query": {}, - "grpc.reflection.v1alpha.ServerReflection": {}, + "cosmwasm.wasm.v1.Query": {}, + "grpc.reflection.v1.ServerReflection": {}, "seiprotocol.seichain.epoch.Query": {}, "seiprotocol.seichain.evm.Query": {}, "seiprotocol.seichain.mint.Query": {}, "seiprotocol.seichain.oracle.Query": {}, "seiprotocol.seichain.tokenfactory.Query": {}, - "cosmos.circuit.v1.Query": {}, - "cosmwasm.wasm.v1.Query": {}, } // bucketRPCMethod maps a raw JSON-RPC method name to a low-cardinality label diff --git a/sei-cosmos/server/config/config_fuzz_test.go b/sei-cosmos/server/config/config_fuzz_test.go index dba6f668fd..b80a8e28d9 100644 --- a/sei-cosmos/server/config/config_fuzz_test.go +++ b/sei-cosmos/server/config/config_fuzz_test.go @@ -1446,7 +1446,7 @@ func FuzzGRPCConfig(f *testing.F) { // TestGRPCKeyNamesMatchTheRecordedNames pins all fifteen [grpc] key names, the four rows and the // eleven driven elsewhere. // -// The eight had no record before this, because their target carries a local struct rather than a +// The eleven had no record before this, because their target carries a local struct rather than a // KeySpec table, so nothing held their spelling. That is the gap this closes. func TestGRPCKeyNamesMatchTheRecordedNames(t *testing.T) { configtest.CheckKeyNames(t, "grpc", grpcKeys, grpcKeysWithTargetsOfTheirOwn...) @@ -1457,6 +1457,9 @@ func TestGRPCManifestNamesEveryField(t *testing.T) { // Guarded reads, so an absent key keeps the in-code default rather than clobbering it. "MaxRecvMsgSize", "MaxOpenConnections", + "IPRateLimitRPS", + "IPRateLimitBurst", + "TrustedProxyCIDRs", // Clamped reads: a negative resolves to the in-code default rather than passing through. "MaxConnectionIdle", "MaxConnectionAge", @@ -1464,9 +1467,6 @@ func TestGRPCManifestNamesEveryField(t *testing.T) { "KeepaliveTime", "KeepaliveTimeout", "KeepaliveMinTime", - "IPRateLimitRPS", - "IPRateLimitBurst", - "TrustedProxyCIDRs", ) } From e633f59d2dabb8e41dd2520111540be7d48ab329 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Fri, 28 Aug 2026 16:34:50 +0200 Subject: [PATCH 04/12] using t.context --- sei-cosmos/baseapp/grpcserver_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sei-cosmos/baseapp/grpcserver_test.go b/sei-cosmos/baseapp/grpcserver_test.go index 5c2ae4a9d3..c9600750b8 100644 --- a/sei-cosmos/baseapp/grpcserver_test.go +++ b/sei-cosmos/baseapp/grpcserver_test.go @@ -55,7 +55,7 @@ func TestRegisterGRPCServerAppliesServerInterceptor(t *testing.T) { client := serveTestQuery(t, setupBaseApp(t), grpc.ChainUnaryInterceptor(spy)) - res, err := client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + res, err := client.Echo(t.Context(), &testdata.EchoRequest{Message: "hello"}) require.NoError(t, err) require.Equal(t, "hello", res.Message) require.Equal(t, int64(1), calls.Load(), "server-level interceptor never reached the query handler") @@ -73,7 +73,7 @@ func TestRegisterGRPCServerServerInterceptorCanRejectQuery(t *testing.T) { client := serveTestQuery(t, setupBaseApp(t), grpc.ChainUnaryInterceptor(reject, count)) - _, err := client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + _, err := client.Echo(t.Context(), &testdata.EchoRequest{Message: "hello"}) require.Error(t, err) require.Equal(t, codes.ResourceExhausted, status.Code(err)) require.Zero(t, handlerCalls.Load(), "rejected query still ran the rest of the chain") @@ -82,7 +82,7 @@ func TestRegisterGRPCServerServerInterceptorCanRejectQuery(t *testing.T) { func TestRegisterGRPCServerWithoutServerInterceptor(t *testing.T) { client := serveTestQuery(t, setupBaseApp(t)) - res, err := client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + res, err := client.Echo(t.Context(), &testdata.EchoRequest{Message: "hello"}) require.NoError(t, err) require.Equal(t, "hello", res.Message) } From fa2ebc6923ef7cdac7575a887f5203e63c84be94 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Mon, 31 Aug 2026 11:16:46 +0200 Subject: [PATCH 05/12] PLT-1072: Address review feedback on gRPC rate-limit admission - Log at startup when admission is enabled with a zeroed token bucket, matching the signal the CometBFT RPC plane already emits. - State the per-stream (not per-message) admission scope in the stream interceptor godoc. - Document that the interceptors and trusted-proxy-cidrs also govern gRPC-Web (:9091), which wraps the same server. - Cover StartGRPCServer's enable path, disable path, and malformed-CIDR failure against a real server. - Extend TestGetConfigGRPCOverrides with the three IsSet-guarded [grpc] rate-limit keys. Co-Authored-By: Claude Opus 5 (1M context) --- sei-cosmos/server/config/config.go | 3 + sei-cosmos/server/config/config_test.go | 8 ++ sei-cosmos/server/config/toml.go | 5 +- sei-cosmos/server/grpc/rate_limit.go | 9 +- sei-cosmos/server/grpc/server.go | 52 +++++-- .../server/grpc/server_rate_limit_test.go | 132 ++++++++++++++++++ 6 files changed, 193 insertions(+), 16 deletions(-) create mode 100644 sei-cosmos/server/grpc/server_rate_limit_test.go diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index cb1aff464f..ad0ec1ea72 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -272,10 +272,13 @@ type GRPCConfig struct { IPRateLimitBurst int `mapstructure:"ip-rate-limit-burst"` // RateLimitingEnabled is the master switch for gRPC rate-limit admission. + // The interceptors it installs also govern gRPC-Web (:9091), which wraps the + // same *grpc.Server, and its traffic draws from the same per-IP buckets. RateLimitingEnabled bool `mapstructure:"rate-limiting-enabled"` // TrustedProxyCIDRs lists CIDRs whose x-forwarded-for metadata is trusted // when resolving the client IP for rate limiting. Empty means trust no proxy. + // It applies to gRPC-Web (:9091) as well as native gRPC (:9090). TrustedProxyCIDRs []string `mapstructure:"trusted-proxy-cidrs"` } diff --git a/sei-cosmos/server/config/config_test.go b/sei-cosmos/server/config/config_test.go index 1d1f09072d..8cebbfeac3 100644 --- a/sei-cosmos/server/config/config_test.go +++ b/sei-cosmos/server/config/config_test.go @@ -175,6 +175,10 @@ func TestGetConfigGRPCOverrides(t *testing.T) { v.Set("grpc.keepalive-timeout", "10s") v.Set("grpc.keepalive-min-time", "30s") v.Set("grpc.keepalive-permit-without-stream", true) + v.Set("grpc.ip-rate-limit-rps", 42.5) + v.Set("grpc.ip-rate-limit-burst", 99) + v.Set("grpc.rate-limiting-enabled", true) + v.Set("grpc.trusted-proxy-cidrs", []string{"10.1.0.0/16", "192.168.0.0/16"}) cfg, err := GetConfig(v) require.NoError(t, err) @@ -187,6 +191,10 @@ func TestGetConfigGRPCOverrides(t *testing.T) { require.Equal(t, 10*time.Second, cfg.GRPC.KeepaliveTimeout) require.Equal(t, 30*time.Second, cfg.GRPC.KeepaliveMinTime) require.True(t, cfg.GRPC.KeepalivePermitWithoutStream) + require.Equal(t, 42.5, cfg.GRPC.IPRateLimitRPS) + require.Equal(t, 99, cfg.GRPC.IPRateLimitBurst) + require.True(t, cfg.GRPC.RateLimitingEnabled) + require.Equal(t, []string{"10.1.0.0/16", "192.168.0.0/16"}, cfg.GRPC.TrustedProxyCIDRs) } func TestDefaultGRPCWebConfig(t *testing.T) { diff --git a/sei-cosmos/server/config/toml.go b/sei-cosmos/server/config/toml.go index c61b666726..47aab9d055 100644 --- a/sei-cosmos/server/config/toml.go +++ b/sei-cosmos/server/config/toml.go @@ -269,10 +269,13 @@ ip-rate-limit-rps = {{ .GRPC.IPRateLimitRPS }} ip-rate-limit-burst = {{ .GRPC.IPRateLimitBurst }} # rate-limiting-enabled is the master switch for gRPC rate-limit admission interceptors. +# These interceptors also govern gRPC-Web (:9091), which wraps the same server, and its +# traffic draws from the same per-IP buckets as native gRPC (:9090). rate-limiting-enabled = {{ .GRPC.RateLimitingEnabled }} # trusted-proxy-cidrs lists CIDRs whose x-forwarded-for metadata is trusted when -# resolving the client IP for rate limiting. Empty means trust no proxy. +# resolving the client IP for rate limiting, on both :9090 and gRPC-Web (:9091). +# Empty means trust no proxy. trusted-proxy-cidrs = [{{- range $i, $c := .GRPC.TrustedProxyCIDRs }}{{- if $i }}, {{ end }}"{{ $c }}"{{- end }}] ############################################################################### diff --git a/sei-cosmos/server/grpc/rate_limit.go b/sei-cosmos/server/grpc/rate_limit.go index f9e5bee2a1..515e903a8f 100644 --- a/sei-cosmos/server/grpc/rate_limit.go +++ b/sei-cosmos/server/grpc/rate_limit.go @@ -25,8 +25,13 @@ func UnaryRateLimitInterceptor(registry *ratelimiter.Registry) grpc.UnaryServerI } // StreamRateLimitInterceptor returns a server interceptor that applies per-IP -// token-bucket rate limiting when a client stream is established. registry must -// be non-nil. +// token-bucket rate limiting at stream establishment. registry must be non-nil. +// +// Admission is per stream, not per message: one token is spent when the stream +// opens and messages sent on an admitted stream are not accounted. The only +// streaming surface on :9090 is server reflection, whose bidirectional +// ServerReflectionInfo can therefore serve unbounded descriptor lookups over a +// single admitted stream. func StreamRateLimitInterceptor(registry *ratelimiter.Registry) grpc.StreamServerInterceptor { return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { ctx := ss.Context() diff --git a/sei-cosmos/server/grpc/server.go b/sei-cosmos/server/grpc/server.go index e7550692a8..4468381e5e 100644 --- a/sei-cosmos/server/grpc/server.go +++ b/sei-cosmos/server/grpc/server.go @@ -6,6 +6,7 @@ import ( "net" "time" + "github.com/sei-protocol/seilog" "golang.org/x/net/netutil" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" @@ -19,6 +20,36 @@ import ( sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" ) +var logger = seilog.NewLogger("cosmos", "server", "grpc") + +// rateLimitServerOptions returns the unary and stream interceptor options that +// apply per-IP admission to the gRPC plane. It returns nil when cfg leaves +// rate limiting disabled. +func rateLimitServerOptions(cfg config.GRPCConfig) ([]grpc.ServerOption, error) { + if !cfg.RateLimitingEnabled { + return nil, nil + } + registry, err := ratelimiter.New(cfg.RateLimiterConfig()) + if err != nil { + return nil, fmt.Errorf("grpc rate limiter: %w", err) + } + // A zeroed bucket makes Allow admit unconditionally, so the interceptors + // install and throttle nothing. The CometBFT RPC plane logs the same + // combination; without this the gRPC plane would fail open silently. + if cfg.IPRateLimitRPS <= 0 || cfg.IPRateLimitBurst <= 0 { + logger.Info( + "gRPC rate-limit admission is enabled but the token bucket is disabled "+ + "(ip-rate-limit-rps and/or ip-rate-limit-burst <= 0); ResourceExhausted throttling will not occur", + "ip-rate-limit-rps", cfg.IPRateLimitRPS, + "ip-rate-limit-burst", cfg.IPRateLimitBurst, + ) + } + return []grpc.ServerOption{ + grpc.ChainUnaryInterceptor(UnaryRateLimitInterceptor(registry)), + grpc.ChainStreamInterceptor(StreamRateLimitInterceptor(registry)), + }, nil +} + // StartGRPCServer starts a gRPC server on the address given by cfg. func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config.GRPCConfig) (*grpc.Server, error) { maxRecvMsgSize := cfg.MaxRecvMsgSize @@ -26,7 +57,12 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config maxRecvMsgSize = config.DefaultGRPCMaxRecvMsgSize } - serverOpts := []grpc.ServerOption{ + rateLimitOpts, err := rateLimitServerOptions(cfg) + if err != nil { + return nil, err + } + + serverOpts := append([]grpc.ServerOption{ grpc.MaxConcurrentStreams(100), // MaxRecvMsgSize bounds per-request memory allocation before the rate // limiter fires, preventing an oversized request from exhausting memory. @@ -42,23 +78,13 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config MinTime: cfg.KeepaliveMinTime, PermitWithoutStream: cfg.KeepalivePermitWithoutStream, }), - } - if cfg.RateLimitingEnabled { - rateLimitRegistry, err := ratelimiter.New(cfg.RateLimiterConfig()) - if err != nil { - return nil, fmt.Errorf("grpc rate limiter: %w", err) - } - serverOpts = append(serverOpts, - grpc.ChainUnaryInterceptor(UnaryRateLimitInterceptor(rateLimitRegistry)), - grpc.ChainStreamInterceptor(StreamRateLimitInterceptor(rateLimitRegistry)), - ) - } + }, rateLimitOpts...) grpcSrv := grpc.NewServer(serverOpts...) app.RegisterGRPCServer(grpcSrv) // reflection allows consumers to build dynamic clients that can write // to any cosmos-sdk application without relying on application packages at compile time - err := reflection.Register(grpcSrv, reflection.Config{ + err = reflection.Register(grpcSrv, reflection.Config{ SigningModes: func() map[string]int32 { modes := make(map[string]int32, len(clientCtx.TxConfig.SignModeHandler().Modes())) for _, m := range clientCtx.TxConfig.SignModeHandler().Modes() { diff --git a/sei-cosmos/server/grpc/server_rate_limit_test.go b/sei-cosmos/server/grpc/server_rate_limit_test.go new file mode 100644 index 0000000000..56ba971bae --- /dev/null +++ b/sei-cosmos/server/grpc/server_rate_limit_test.go @@ -0,0 +1,132 @@ +package grpc + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + "github.com/sei-protocol/sei-chain/sei-cosmos/baseapp" + "github.com/sei-protocol/sei-chain/sei-cosmos/client" + cryptotypes "github.com/sei-protocol/sei-chain/sei-cosmos/crypto/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/server/api" + "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" + "github.com/sei-protocol/sei-chain/sei-cosmos/testutil" + "github.com/sei-protocol/sei-chain/sei-cosmos/testutil/testdata" + sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + moduletestutil "github.com/sei-protocol/sei-chain/sei-cosmos/types/module/testutil" +) + +// startGRPCServerApp is the minimal types.Application StartGRPCServer needs: a +// BaseApp carrying the query router, plus stubs for the bootstrap hooks the +// gRPC path never calls. +type startGRPCServerApp struct { + *baseapp.BaseApp +} + +func (startGRPCServerApp) RegisterAPIRoutes(*api.Server, config.APIConfig) {} +func (startGRPCServerApp) RegisterLocalServices(client.LocalClient, client.TxConfig) {} +func (startGRPCServerApp) InplaceTestnetInitialize(cryptotypes.PubKey) {} + +// startRateLimitedGRPCServer starts a real server through StartGRPCServer with +// cfg's rate-limit settings and returns a client dialled to it. +func startRateLimitedGRPCServer(t *testing.T, cfg config.GRPCConfig) testdata.QueryClient { + t.Helper() + + srv, addr := startGRPCServerOnFreePort(t, cfg) + t.Cleanup(srv.Stop) + + conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + return testdata.NewQueryClient(conn) +} + +func startGRPCServerOnFreePort(t *testing.T, cfg config.GRPCConfig) (*grpc.Server, string) { + t.Helper() + + app := baseapp.NewBaseApp(t.Name(), dbm.NewMemDB(), nil, nil, &testutil.TestAppOpts{}) + app.MountStores(sdk.NewKVStoreKey("test")) + require.NoError(t, app.LoadLatestVersion()) + + encCfg := moduletestutil.MakeTestEncodingConfig() + testdata.RegisterInterfaces(encCfg.InterfaceRegistry) + app.SetInterfaceRegistry(encCfg.InterfaceRegistry) + testdata.RegisterQueryServer(app.GRPCQueryRouter(), testdata.QueryImpl{}) + + clientCtx := client.Context{}. + WithChainID("test-chain"). + WithTxConfig(encCfg.TxConfig). + WithInterfaceRegistry(encCfg.InterfaceRegistry) + + cfg.Address = freeTCPAddr(t) + srv, err := StartGRPCServer(clientCtx, startGRPCServerApp{app}, cfg) + require.NoError(t, err) + + return srv, cfg.Address +} + +func freeTCPAddr(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := ln.Addr().String() + require.NoError(t, ln.Close()) + return addr +} + +func rateLimitedGRPCConfig(rps float64, burst int) config.GRPCConfig { + return config.GRPCConfig{ + Enable: true, + RateLimitingEnabled: true, + IPRateLimitRPS: rps, + IPRateLimitBurst: burst, + } +} + +// TestStartGRPCServer_RateLimitingEnabledInstallsInterceptors covers the wiring +// in StartGRPCServer itself: without the interceptor options it appends, a +// second query from the same IP would be served rather than rejected. +func TestStartGRPCServer_RateLimitingEnabledInstallsInterceptors(t *testing.T) { + client := startRateLimitedGRPCServer(t, rateLimitedGRPCConfig(0.001, 1)) + + res, err := client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.NoError(t, err) + require.Equal(t, "hello", res.Message) + + _, err = client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.Error(t, err) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +// TestStartGRPCServer_RateLimitingDisabledSkipsInterceptors pins the master +// switch: a bucket that would reject the second query never sees it. +func TestStartGRPCServer_RateLimitingDisabledSkipsInterceptors(t *testing.T) { + cfg := rateLimitedGRPCConfig(0.001, 1) + cfg.RateLimitingEnabled = false + client := startRateLimitedGRPCServer(t, cfg) + + for i := 0; i < 3; i++ { + res, err := client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.NoError(t, err) + require.Equal(t, "hello", res.Message) + } +} + +// TestStartGRPCServer_MalformedTrustedProxyCIDR pins that a bad CIDR fails +// startup rather than silently starting an unprotected server. +func TestStartGRPCServer_MalformedTrustedProxyCIDR(t *testing.T) { + cfg := rateLimitedGRPCConfig(100, 100) + cfg.TrustedProxyCIDRs = []string{"not-a-cidr"} + cfg.Address = freeTCPAddr(t) + + _, err := StartGRPCServer(client.Context{}, startGRPCServerApp{}, cfg) + require.ErrorContains(t, err, "grpc rate limiter") +} From d6edf040286c5c57dac92293a35abe2a79a1ee1a Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Mon, 31 Aug 2026 11:38:38 +0200 Subject: [PATCH 06/12] PLT-1072: Charge gRPC streams per message and label metrics by method Streams now cost one token to establish and one per inbound message, so an admitted reflection stream can no longer serve unbounded descriptor lookups. PlaneGRPC rejection metrics record the full "service/Method" instead of the service name, bounded by the methods actually registered on the server (read from GetServiceInfo at startup) rather than a hand-maintained allowlist. Unregistered methods still record "other". Co-Authored-By: Claude Opus 5 (1M context) --- ratelimiter/method_bucket.go | 54 +++------------- ratelimiter/method_bucket_test.go | 63 ++++++++++--------- ratelimiter/registry.go | 24 ++++++- ratelimiter/registry_test.go | 36 +++++++++++ sei-cosmos/server/grpc/rate_limit.go | 37 ++++++++--- .../server/grpc/rate_limit_dispatch_test.go | 35 +++++++++++ sei-cosmos/server/grpc/rate_limit_test.go | 33 +++++++++- sei-cosmos/server/grpc/server.go | 27 ++++++-- .../server/grpc/server_rate_limit_test.go | 50 +++++++++++++++ 9 files changed, 269 insertions(+), 90 deletions(-) diff --git a/ratelimiter/method_bucket.go b/ratelimiter/method_bucket.go index ed02c1120d..a164f6454f 100644 --- a/ratelimiter/method_bucket.go +++ b/ratelimiter/method_bucket.go @@ -77,37 +77,11 @@ var knownCometBFTRPCMethods = map[string]struct{}{ "websocket": {}, } -// knownGRPCServices lists protobuf service names registered on the native gRPC -// server. Rejection metrics on PlaneGRPC record the service name rather than -// the full /service/Method path, keeping OTel attribute cardinality bounded. -var knownGRPCServices = map[string]struct{}{ - "cosmos.auth.v1beta1.Query": {}, - "cosmos.authz.v1beta1.Query": {}, - "cosmos.bank.v1beta1.Query": {}, - "cosmos.base.reflection.v1beta1.ReflectionService": {}, - "cosmos.base.reflection.v2alpha1.ReflectionService": {}, - "cosmos.base.tendermint.v1beta1.Service": {}, - "cosmos.distribution.v1beta1.Query": {}, - "cosmos.evidence.v1beta1.Query": {}, - "cosmos.gov.v1beta1.Query": {}, - "cosmos.params.v1beta1.Query": {}, - "cosmos.slashing.v1beta1.Query": {}, - "cosmos.staking.v1beta1.Query": {}, - "cosmos.tx.v1beta1.Service": {}, - "cosmos.upgrade.v1beta1.Query": {}, - "cosmwasm.wasm.v1.Query": {}, - "grpc.reflection.v1.ServerReflection": {}, - "seiprotocol.seichain.epoch.Query": {}, - "seiprotocol.seichain.evm.Query": {}, - "seiprotocol.seichain.mint.Query": {}, - "seiprotocol.seichain.oracle.Query": {}, - "seiprotocol.seichain.tokenfactory.Query": {}, -} - // bucketRPCMethod maps a raw JSON-RPC method name to a low-cardinality label // suitable for OTel/Prometheus metrics. Attacker-controlled method strings -// collapse to rpcMethodBucketOther. -func bucketRPCMethod(plane, method string) string { +// collapse to rpcMethodBucketOther. knownGRPCMethods bounds the PlaneGRPC label +// set; the other planes ignore it. +func bucketRPCMethod(plane, method string, knownGRPCMethods map[string]struct{}) string { if method == MethodInvalid { return MethodInvalid } @@ -115,34 +89,22 @@ func bucketRPCMethod(plane, method string) string { return bucketCometBFTRPCMethod(method) } if plane == PlaneGRPC { - return bucketGRPCMethod(method) + return bucketGRPCMethod(method, knownGRPCMethods) } return bucketNamespacedRPCMethod(method) } -func bucketGRPCMethod(fullMethod string) string { +func bucketGRPCMethod(fullMethod string, known map[string]struct{}) string { if fullMethod == "" || len(fullMethod) > maxRPCMethodLen { return rpcMethodBucketOther } - svc := grpcServiceFromFullMethod(fullMethod) - if svc == "" || len(svc) > maxRPCMethodLen { - return rpcMethodBucketOther - } - if _, ok := knownGRPCServices[svc]; ok { - return svc + name := strings.TrimPrefix(fullMethod, "/") + if _, ok := known[name]; ok { + return name } return rpcMethodBucketOther } -func grpcServiceFromFullMethod(fullMethod string) string { - method := strings.TrimPrefix(fullMethod, "/") - slash := strings.LastIndexByte(method, '/') - if slash <= 0 { - return "" - } - return method[:slash] -} - func bucketCometBFTRPCMethod(method string) string { if method == "" || len(method) > maxRPCMethodLen { return rpcMethodBucketOther diff --git a/ratelimiter/method_bucket_test.go b/ratelimiter/method_bucket_test.go index 9a2778b2fd..cd7388adc9 100644 --- a/ratelimiter/method_bucket_test.go +++ b/ratelimiter/method_bucket_test.go @@ -8,21 +8,21 @@ import ( ) func TestBucketRPCMethod_KnownNamespaces(t *testing.T) { - require.Equal(t, "eth", bucketRPCMethod("evm", "eth_call")) - require.Equal(t, "eth", bucketRPCMethod("evm", "eth_getBalance")) - require.Equal(t, "debug", bucketRPCMethod("evm", "debug_traceTransaction")) - require.Equal(t, "web3", bucketRPCMethod("evm", "web3_clientVersion")) + require.Equal(t, "eth", bucketRPCMethod("evm", "eth_call", nil)) + require.Equal(t, "eth", bucketRPCMethod("evm", "eth_getBalance", nil)) + require.Equal(t, "debug", bucketRPCMethod("evm", "debug_traceTransaction", nil)) + require.Equal(t, "web3", bucketRPCMethod("evm", "web3_clientVersion", nil)) } func TestBucketRPCMethod_UnknownOrMalformed(t *testing.T) { - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "notnamespaced")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "eth")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "_eth_call")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "eth_")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "bogus_method")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "eth_BAD-chars")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", strings.Repeat("a", maxRPCMethodLen+1))) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "", nil)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "notnamespaced", nil)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "eth", nil)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "_eth_call", nil)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "eth_", nil)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "bogus_method", nil)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", "eth_BAD-chars", nil)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod("evm", strings.Repeat("a", maxRPCMethodLen+1), nil)) } func TestBucketRPCMethod_ValidAndInvalidMethods(t *testing.T) { @@ -32,7 +32,7 @@ func TestBucketRPCMethod_ValidAndInvalidMethods(t *testing.T) { "eth_random-uuid-1", "eth_random-uuid-2", } { - seen[bucketRPCMethod("evm", method)] = struct{}{} + seen[bucketRPCMethod("evm", method, nil)] = struct{}{} } require.Len(t, seen, 2) require.Contains(t, seen, "eth") @@ -48,30 +48,37 @@ func TestBucketRPCMethod_CometBFTKnownMethods(t *testing.T) { "catalog", "websocket", } { - require.Equal(t, method, bucketRPCMethod(PlaneCometBFT, method), method) + require.Equal(t, method, bucketRPCMethod(PlaneCometBFT, method, nil), method) } } func TestBucketRPCMethod_CometBFTUnknownMethods(t *testing.T) { - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneCometBFT, "")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneCometBFT, "bogus")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneCometBFT, strings.Repeat("a", maxRPCMethodLen+1))) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneCometBFT, "", nil)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneCometBFT, "bogus", nil)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneCometBFT, strings.Repeat("a", maxRPCMethodLen+1), nil)) } func TestBucketRPCMethod_Invalid(t *testing.T) { - require.Equal(t, MethodInvalid, bucketRPCMethod("evm", MethodInvalid)) - require.Equal(t, MethodInvalid, bucketRPCMethod(PlaneCometBFT, MethodInvalid)) - require.Equal(t, MethodInvalid, bucketRPCMethod(PlaneGRPC, MethodInvalid)) + require.Equal(t, MethodInvalid, bucketRPCMethod("evm", MethodInvalid, nil)) + require.Equal(t, MethodInvalid, bucketRPCMethod(PlaneCometBFT, MethodInvalid, nil)) + require.Equal(t, MethodInvalid, bucketRPCMethod(PlaneGRPC, MethodInvalid, nil)) } -func TestBucketRPCMethod_GrpcknownServices(t *testing.T) { - require.Equal(t, "cosmos.bank.v1beta1.Query", bucketRPCMethod(PlaneGRPC, "/cosmos.bank.v1beta1.Query/Balance")) - require.Equal(t, "cosmos.tx.v1beta1.Service", bucketRPCMethod(PlaneGRPC, "/cosmos.tx.v1beta1.Service/Simulate")) +func TestBucketRPCMethod_GrpcKnownMethods(t *testing.T) { + known := map[string]struct{}{ + "cosmos.bank.v1beta1.Query/Balance": {}, + "cosmos.tx.v1beta1.Service/Simulate": {}, + } + require.Equal(t, "cosmos.bank.v1beta1.Query/Balance", bucketRPCMethod(PlaneGRPC, "/cosmos.bank.v1beta1.Query/Balance", known)) + require.Equal(t, "cosmos.tx.v1beta1.Service/Simulate", bucketRPCMethod(PlaneGRPC, "/cosmos.tx.v1beta1.Service/Simulate", known)) } -func TestBucketRPCMethod_GrpcUnknownServices(t *testing.T) { - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "/bogus.Service/Call")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "not-a-grpc-path")) - require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, strings.Repeat("a", maxRPCMethodLen+1))) +func TestBucketRPCMethod_GrpcUnknownMethods(t *testing.T) { + known := map[string]struct{}{"cosmos.bank.v1beta1.Query/Balance": {}} + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "", known)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "/cosmos.bank.v1beta1.Query/AllBalances", known)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "/bogus.Service/Call", known)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "not-a-grpc-path", known)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, strings.Repeat("a", maxRPCMethodLen+1), known)) + require.Equal(t, rpcMethodBucketOther, bucketRPCMethod(PlaneGRPC, "/cosmos.bank.v1beta1.Query/Balance", nil)) } diff --git a/ratelimiter/registry.go b/ratelimiter/registry.go index 7923f6e45c..31b259aee6 100644 --- a/ratelimiter/registry.go +++ b/ratelimiter/registry.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" "sync" + "sync/atomic" "time" "github.com/hashicorp/golang-lru/v2/expirable" @@ -72,6 +73,25 @@ type Registry struct { trustedProxies []*net.IPNet lru *expirable.LRU[string, *rate.Limiter] mu sync.Mutex + grpcMethods atomic.Pointer[map[string]struct{}] +} + +// SetKnownGRPCMethods bounds the method label recorded for PlaneGRPC rejections +// to the given "service/Method" names. Call it before serving; unlisted methods +// are recorded as "other". +func (r *Registry) SetKnownGRPCMethods(methods []string) { + known := make(map[string]struct{}, len(methods)) + for _, m := range methods { + known[strings.TrimPrefix(m, "/")] = struct{}{} + } + r.grpcMethods.Store(&known) +} + +func (r *Registry) knownGRPCMethods() map[string]struct{} { + if known := r.grpcMethods.Load(); known != nil { + return *known + } + return nil } // New creates a Registry from cfg. Returns an error if any CIDR in TrustedProxyCIDRs is invalid. @@ -101,7 +121,7 @@ func (r *Registry) Allow(ctx context.Context, ip, plane, method string) bool { 1, metric.WithAttributes( attribute.String("plane", plane), - attribute.String("method_namespace", bucketRPCMethod(plane, method)), + attribute.String("method_namespace", bucketRPCMethod(plane, method, r.knownGRPCMethods())), ), ) return false @@ -121,7 +141,7 @@ func (r *Registry) AllowN(ctx context.Context, ip, plane, method string, n int) 1, metric.WithAttributes( attribute.String("plane", plane), - attribute.String("method_namespace", bucketRPCMethod(plane, method)), + attribute.String("method_namespace", bucketRPCMethod(plane, method, r.knownGRPCMethods())), ), ) return false diff --git a/ratelimiter/registry_test.go b/ratelimiter/registry_test.go index 59831685db..12bd0ca804 100644 --- a/ratelimiter/registry_test.go +++ b/ratelimiter/registry_test.go @@ -368,3 +368,39 @@ func TestNew_InvalidCIDR_ReturnsError(t *testing.T) { }) require.ErrorContains(t, err, "bad") } + +func TestAllow_GRPCRejectionRecordsFullMethodMetric(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + prev := otel.GetMeterProvider() + otel.SetMeterProvider(provider) + t.Cleanup(func() { otel.SetMeterProvider(prev) }) + + registryMetrics.rejectedCounter = must(provider.Meter("ratelimiter").Int64Counter( + "rpc_rate_limit_rejected_total", + )) + + r := mustNew(t, cfg(0.001, 1)) + r.SetKnownGRPCMethods([]string{"/cosmos.bank.v1beta1.Query/Balance"}) + ip := "10.0.0.42" + require.True(t, r.Allow(t.Context(), ip, PlaneGRPC, "/cosmos.bank.v1beta1.Query/Balance")) + require.False(t, r.Allow(t.Context(), ip, PlaneGRPC, "/cosmos.bank.v1beta1.Query/Balance")) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + found := false + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != "rpc_rate_limit_rejected_total" { + continue + } + sum := m.Data.(metricdata.Sum[int64]) + require.Equal(t, int64(1), sum.DataPoints[0].Value) + attrs := sum.DataPoints[0].Attributes.ToSlice() + require.Contains(t, attrs, attribute.String("plane", PlaneGRPC)) + require.Contains(t, attrs, attribute.String("method_namespace", "cosmos.bank.v1beta1.Query/Balance")) + found = true + } + } + require.True(t, found, "expected rpc_rate_limit_rejected_total metric") +} diff --git a/sei-cosmos/server/grpc/rate_limit.go b/sei-cosmos/server/grpc/rate_limit.go index 515e903a8f..930e1da06c 100644 --- a/sei-cosmos/server/grpc/rate_limit.go +++ b/sei-cosmos/server/grpc/rate_limit.go @@ -25,13 +25,10 @@ func UnaryRateLimitInterceptor(registry *ratelimiter.Registry) grpc.UnaryServerI } // StreamRateLimitInterceptor returns a server interceptor that applies per-IP -// token-bucket rate limiting at stream establishment. registry must be non-nil. +// token-bucket rate limiting to client streams. registry must be non-nil. // -// Admission is per stream, not per message: one token is spent when the stream -// opens and messages sent on an admitted stream are not accounted. The only -// streaming surface on :9090 is server reflection, whose bidirectional -// ServerReflectionInfo can therefore serve unbounded descriptor lookups over a -// single admitted stream. +// A stream costs one token to establish and one more per inbound message, so an +// admitted stream cannot serve unbounded work. func StreamRateLimitInterceptor(registry *ratelimiter.Registry) grpc.StreamServerInterceptor { return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { ctx := ss.Context() @@ -39,6 +36,32 @@ func StreamRateLimitInterceptor(registry *ratelimiter.Registry) grpc.StreamServe if !registry.Allow(ctx, ip, ratelimiter.PlaneGRPC, info.FullMethod) { return errRateLimited } - return handler(srv, ss) + return handler(srv, rateLimitedServerStream{ + ServerStream: ss, + registry: registry, + ip: ip, + method: info.FullMethod, + }) } } + +// rateLimitedServerStream charges the per-IP bucket for each message the client +// sends on an admitted stream. +type rateLimitedServerStream struct { + grpc.ServerStream + + registry *ratelimiter.Registry + ip string + method string +} + +func (s rateLimitedServerStream) RecvMsg(m interface{}) error { + // Charge after the read so a blocked stream holds no token and EOF is free. + if err := s.ServerStream.RecvMsg(m); err != nil { + return err + } + if !s.registry.Allow(s.Context(), s.ip, ratelimiter.PlaneGRPC, s.method) { + return errRateLimited + } + return nil +} diff --git a/sei-cosmos/server/grpc/rate_limit_dispatch_test.go b/sei-cosmos/server/grpc/rate_limit_dispatch_test.go index 18e92b0e79..041ce6ae58 100644 --- a/sei-cosmos/server/grpc/rate_limit_dispatch_test.go +++ b/sei-cosmos/server/grpc/rate_limit_dispatch_test.go @@ -10,11 +10,13 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" + rpb "google.golang.org/grpc/reflection/grpc_reflection_v1" "google.golang.org/grpc/status" "github.com/sei-protocol/sei-chain/ratelimiter" "github.com/sei-protocol/sei-chain/sei-cosmos/baseapp" codectypes "github.com/sei-protocol/sei-chain/sei-cosmos/codec/types" + "github.com/sei-protocol/sei-chain/sei-cosmos/server/grpc/gogoreflection" "github.com/sei-protocol/sei-chain/sei-cosmos/testutil" "github.com/sei-protocol/sei-chain/sei-cosmos/testutil/testdata" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" @@ -74,3 +76,36 @@ func TestUnaryRateLimitInterceptor_RegisteredQueryServiceUnlimited(t *testing.T) require.Equal(t, "hello", res.Message) } } + +// TestStreamRateLimitInterceptor_ReflectionChargesPerMessage exercises the only +// streaming surface on :9090 over a real server: a burst of 2 covers the stream +// establishment and one lookup, so the second lookup is rejected. +func TestStreamRateLimitInterceptor_ReflectionChargesPerMessage(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 2)) + srv := grpc.NewServer(grpc.ChainStreamInterceptor(StreamRateLimitInterceptor(reg))) + gogoreflection.Register(srv) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = srv.Serve(listener) }() + t.Cleanup(srv.Stop) + + conn, err := grpc.Dial(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + stream, err := rpb.NewServerReflectionClient(conn).ServerReflectionInfo(context.Background()) + require.NoError(t, err) + + listServices := &rpb.ServerReflectionRequest{ + MessageRequest: &rpb.ServerReflectionRequest_ListServices{}, + } + require.NoError(t, stream.Send(listServices)) + _, err = stream.Recv() + require.NoError(t, err) + + require.NoError(t, stream.Send(listServices)) + _, err = stream.Recv() + require.Error(t, err) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} diff --git a/sei-cosmos/server/grpc/rate_limit_test.go b/sei-cosmos/server/grpc/rate_limit_test.go index 56c9e8656b..2234e4865c 100644 --- a/sei-cosmos/server/grpc/rate_limit_test.go +++ b/sei-cosmos/server/grpc/rate_limit_test.go @@ -2,6 +2,7 @@ package grpc import ( "context" + "io" "net" "testing" @@ -91,10 +92,12 @@ func TestUnaryRateLimitInterceptor_TrustedProxyXFF(t *testing.T) { type mockServerStream struct { grpc.ServerStream - ctx context.Context + ctx context.Context + recvErr error } -func (m mockServerStream) Context() context.Context { return m.ctx } +func (m mockServerStream) Context() context.Context { return m.ctx } +func (m mockServerStream) RecvMsg(interface{}) error { return m.recvErr } func TestStreamRateLimitInterceptor_AllowThenReject(t *testing.T) { reg := mustNewRegistry(t, cfg(0.001, 1)) @@ -160,3 +163,29 @@ func TestUnaryRateLimitInterceptor_RealTCPAddr(t *testing.T) { _, err = ic(ctx, nil, info, handler) require.Equal(t, codes.ResourceExhausted, status.Code(err)) } + +func TestStreamRateLimitInterceptor_ChargesPerInboundMessage(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 2)) + ic := StreamRateLimitInterceptor(reg) + handler := func(_ interface{}, stream grpc.ServerStream) error { + require.NoError(t, stream.RecvMsg(nil)) + return stream.RecvMsg(nil) + } + info := &grpc.StreamServerInfo{FullMethod: "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo"} + stream := mockServerStream{ctx: grpcCtx(t.Context(), "10.0.0.1:9000")} + + err := ic(nil, stream, info, handler) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +func TestStreamRateLimitInterceptor_RecvErrorIsNotCharged(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1)) + ic := StreamRateLimitInterceptor(reg) + handler := func(_ interface{}, stream grpc.ServerStream) error { + return stream.RecvMsg(nil) + } + info := &grpc.StreamServerInfo{FullMethod: "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo"} + stream := mockServerStream{ctx: grpcCtx(t.Context(), "10.0.0.1:9000"), recvErr: io.EOF} + + require.Equal(t, io.EOF, ic(nil, stream, info, handler)) +} diff --git a/sei-cosmos/server/grpc/server.go b/sei-cosmos/server/grpc/server.go index 4468381e5e..05bb4c1fc1 100644 --- a/sei-cosmos/server/grpc/server.go +++ b/sei-cosmos/server/grpc/server.go @@ -25,13 +25,13 @@ var logger = seilog.NewLogger("cosmos", "server", "grpc") // rateLimitServerOptions returns the unary and stream interceptor options that // apply per-IP admission to the gRPC plane. It returns nil when cfg leaves // rate limiting disabled. -func rateLimitServerOptions(cfg config.GRPCConfig) ([]grpc.ServerOption, error) { +func rateLimitServerOptions(cfg config.GRPCConfig) ([]grpc.ServerOption, *ratelimiter.Registry, error) { if !cfg.RateLimitingEnabled { - return nil, nil + return nil, nil, nil } registry, err := ratelimiter.New(cfg.RateLimiterConfig()) if err != nil { - return nil, fmt.Errorf("grpc rate limiter: %w", err) + return nil, nil, fmt.Errorf("grpc rate limiter: %w", err) } // A zeroed bucket makes Allow admit unconditionally, so the interceptors // install and throttle nothing. The CometBFT RPC plane logs the same @@ -47,7 +47,20 @@ func rateLimitServerOptions(cfg config.GRPCConfig) ([]grpc.ServerOption, error) return []grpc.ServerOption{ grpc.ChainUnaryInterceptor(UnaryRateLimitInterceptor(registry)), grpc.ChainStreamInterceptor(StreamRateLimitInterceptor(registry)), - }, nil + }, registry, nil +} + +// registeredMethods returns the "service/Method" name of every method served by +// srv. +func registeredMethods(srv *grpc.Server) []string { + info := srv.GetServiceInfo() + methods := make([]string, 0, len(info)) + for service, svcInfo := range info { + for _, m := range svcInfo.Methods { + methods = append(methods, service+"/"+m.Name) + } + } + return methods } // StartGRPCServer starts a gRPC server on the address given by cfg. @@ -57,7 +70,7 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config maxRecvMsgSize = config.DefaultGRPCMaxRecvMsgSize } - rateLimitOpts, err := rateLimitServerOptions(cfg) + rateLimitOpts, rateLimitRegistry, err := rateLimitServerOptions(cfg) if err != nil { return nil, err } @@ -102,6 +115,10 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config // Reflection allows external clients to see what services and methods // the gRPC server exposes. gogoreflection.Register(grpcSrv) + if rateLimitRegistry != nil { + rateLimitRegistry.SetKnownGRPCMethods(registeredMethods(grpcSrv)) + } + listener, err := net.Listen("tcp", cfg.Address) if err != nil { return nil, err diff --git a/sei-cosmos/server/grpc/server_rate_limit_test.go b/sei-cosmos/server/grpc/server_rate_limit_test.go index 56ba971bae..6a4c53c1f4 100644 --- a/sei-cosmos/server/grpc/server_rate_limit_test.go +++ b/sei-cosmos/server/grpc/server_rate_limit_test.go @@ -7,6 +7,10 @@ import ( "github.com/stretchr/testify/require" dbm "github.com/tendermint/tm-db" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" @@ -17,6 +21,7 @@ import ( cryptotypes "github.com/sei-protocol/sei-chain/sei-cosmos/crypto/types" "github.com/sei-protocol/sei-chain/sei-cosmos/server/api" "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" + "github.com/sei-protocol/sei-chain/sei-cosmos/server/grpc/gogoreflection" "github.com/sei-protocol/sei-chain/sei-cosmos/testutil" "github.com/sei-protocol/sei-chain/sei-cosmos/testutil/testdata" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" @@ -95,6 +100,7 @@ func rateLimitedGRPCConfig(rps float64, burst int) config.GRPCConfig { // in StartGRPCServer itself: without the interceptor options it appends, a // second query from the same IP would be served rather than rejected. func TestStartGRPCServer_RateLimitingEnabledInstallsInterceptors(t *testing.T) { + reader := collectRejectionMetrics(t) client := startRateLimitedGRPCServer(t, rateLimitedGRPCConfig(0.001, 1)) res, err := client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) @@ -104,6 +110,40 @@ func TestStartGRPCServer_RateLimitingEnabledInstallsInterceptors(t *testing.T) { _, err = client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) require.Error(t, err) require.Equal(t, codes.ResourceExhausted, status.Code(err)) + + // StartGRPCServer hands the registry its registered methods, so the + // rejection is labelled with the method rather than "other". + require.Contains(t, rejectionMethodLabels(t, reader), "testdata.Query/Echo") +} + +func collectRejectionMetrics(t *testing.T) *sdkmetric.ManualReader { + t.Helper() + reader := sdkmetric.NewManualReader() + prev := otel.GetMeterProvider() + otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))) + t.Cleanup(func() { otel.SetMeterProvider(prev) }) + return reader +} + +func rejectionMethodLabels(t *testing.T, reader *sdkmetric.ManualReader) []string { + t.Helper() + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + + labels := []string{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != "rpc_rate_limit_rejected_total" { + continue + } + for _, dp := range m.Data.(metricdata.Sum[int64]).DataPoints { + if v, ok := dp.Attributes.Value(attribute.Key("method_namespace")); ok { + labels = append(labels, v.AsString()) + } + } + } + } + return labels } // TestStartGRPCServer_RateLimitingDisabledSkipsInterceptors pins the master @@ -130,3 +170,13 @@ func TestStartGRPCServer_MalformedTrustedProxyCIDR(t *testing.T) { _, err := StartGRPCServer(client.Context{}, startGRPCServerApp{}, cfg) require.ErrorContains(t, err, "grpc rate limiter") } + +func TestRegisteredMethods(t *testing.T) { + srv := grpc.NewServer() + testdata.RegisterQueryServer(srv, testdata.QueryImpl{}) + gogoreflection.Register(srv) + + methods := registeredMethods(srv) + require.Contains(t, methods, "testdata.Query/Echo") + require.Contains(t, methods, "grpc.reflection.v1.ServerReflection/ServerReflectionInfo") +} From 63c3e0c6bc9bed58893e359a6829d4b30f8b87d0 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Mon, 31 Aug 2026 18:51:14 +0200 Subject: [PATCH 07/12] PLT-1072: Admit gRPC requests before they are decoded gRPC decodes the request message before the interceptor chain runs, so the unary interceptor could shed a throttled caller's handler work but never the decoder's. Methods with expensive decodes, such as GetTxsEvent, were still reachable by a caller over its budget. - Add RateLimitTapHandle, a grpc.InTapHandle that charges the per-IP bucket when a stream's headers arrive, before any message is read off the wire or unmarshalled. It admits native gRPC (:9090). - Add RateLimitHTTPMiddleware for gRPC-Web (:9091), which reaches the same server through ServeHTTP and so never runs the tap handler. Over-budget callers get HTTP 429 there; CORS preflight is exempt so a browser does not pay two tokens per call. - Record the admission on the request context, which becomes the stream context, so the interceptors do not charge an RPC twice. They now cover the per-message stream charge and any RPC that arrived uncharged. - Return the registry from StartGRPCServer and pass it to StartGRPCWeb. A registry built separately for gRPC-Web would give each IP a second bucket, doubling a client's budget across the two ports. The stream per-message charge stays after RecvMsg: charging before the read would throttle a caller for a message it never sent, killing an idle stream whose client is waiting rather than sending. At most one message per stream is decoded past the budget, bounded by MaxRecvMsgSize. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 1 + sei-cosmos/server/config/config.go | 19 +- sei-cosmos/server/config/toml.go | 8 +- sei-cosmos/server/grpc/grpc_web.go | 15 +- .../server/grpc/grpc_web_timeout_test.go | 2 +- sei-cosmos/server/grpc/rate_limit.go | 88 ++++++++- .../server/grpc/rate_limit_dispatch_test.go | 48 +++++ sei-cosmos/server/grpc/rate_limit_test.go | 169 ++++++++++++++++++ sei-cosmos/server/grpc/server.go | 34 ++-- .../server/grpc/server_rate_limit_test.go | 106 ++++++++++- sei-cosmos/server/start.go | 4 +- sei-cosmos/testutil/network/util.go | 4 +- 12 files changed, 452 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ae3225bf9..82d81c31e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#3984](https://github.com/sei-protocol/sei-chain/pull/3984) feat(query): origin-aware pagination limits for ABCI queries. Untrusted callers on the ABCI/gRPC query path get configurable `max-limit`, `max-offset`, and flat `max-iterations` (defaults: 1000 / 10000 / 11000); requests above the caps are rejected upfront, and an exhausted iteration budget returns a partial page with `next_key` instead of failing. Trusted origins (new `[query] trusted-cidrs`) and the `[query] disable-limits` kill switch bypass the caps; the consensus/EVM precompile path is unaffected. * [#3990](https://github.com/sei-protocol/sei-chain/pull/3990) Freeze mode is limited to full nodes and disables transaction and evidence submission, mempool gossip, and state sync from startup while preserving query RPC and mempool-backed reads. Frozen and Autobahn nodes no longer advertise the unused mempool P2P channel. * [#4009](https://github.com/sei-protocol/sei-chain/pull/4009) Bound `/store/*/subspace` ABCI queries with pair/byte caps, empty-prefix rejection, SS-path concurrency limits, and context-aware iteration to prevent memory-exhaustion DoS. +* [#4021](https://github.com/sei-protocol/sei-chain/pull/4021) feat(grpc): per-IP rate-limit admission for the gRPC plane, off by default behind `[grpc] rate-limiting-enabled` (new `ip-rate-limit-rps` / `ip-rate-limit-burst` / `trusted-proxy-cidrs`, defaults 200 rps / 400 burst / trust no proxy). Native gRPC (:9090) is admitted by a tap handler and gRPC-Web (:9091) by HTTP middleware, both before the request is protobuf-decoded, so a throttled caller cannot spend the decoder; streams pay one token to establish and one per inbound message. Both planes draw from the same per-IP buckets. Over-budget callers get `ResourceExhausted` on :9090 and HTTP 429 on :9091, counted by `rpc_rate_limit_rejected_total{plane="grpc", method_namespace}`. ### Upgrade guide * **IBC core removal.** Removes the retired IBC core source, protobufs, light clients, CLI, and simulation support. Retired IBC stores remain mounted but are omitted from `export-genesis`; preserve the state database or use v6.6 freeze nodes for historical IBC data. diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index ad0ec1ea72..bce723b8f9 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -260,20 +260,19 @@ type GRPCConfig struct { // pings even when there are no active streams. KeepalivePermitWithoutStream bool `mapstructure:"keepalive-permit-without-stream"` - // IPRateLimitRPS is the per-IP sustained request rate in requests/second for - // native gRPC (:9090). Zero disables the token bucket (Allow always returns - // true) and does not bypass the admission interceptor when - // rate-limiting-enabled is true. + // IPRateLimitRPS is the per-IP sustained request rate in requests/second. + // Zero disables the token bucket (Allow always returns true) and does not + // bypass admission when rate-limiting-enabled is true. IPRateLimitRPS float64 `mapstructure:"ip-rate-limit-rps"` // IPRateLimitBurst is the maximum per-IP burst size. Zero disables the token - // bucket (same effect as ip-rate-limit-rps = 0) and does not bypass the - // admission interceptor when rate-limiting-enabled is true. + // bucket (same effect as ip-rate-limit-rps = 0) and does not bypass + // admission when rate-limiting-enabled is true. IPRateLimitBurst int `mapstructure:"ip-rate-limit-burst"` - // RateLimitingEnabled is the master switch for gRPC rate-limit admission. - // The interceptors it installs also govern gRPC-Web (:9091), which wraps the - // same *grpc.Server, and its traffic draws from the same per-IP buckets. + // RateLimitingEnabled is the master switch for gRPC rate-limit admission. It + // governs gRPC-Web (:9091) as well as native gRPC (:9090), and both planes + // draw from the same per-IP buckets. RateLimitingEnabled bool `mapstructure:"rate-limiting-enabled"` // TrustedProxyCIDRs lists CIDRs whose x-forwarded-for metadata is trusted @@ -282,7 +281,7 @@ type GRPCConfig struct { TrustedProxyCIDRs []string `mapstructure:"trusted-proxy-cidrs"` } -// RateLimiterConfig builds the ratelimiter.Config used by native gRPC admission. +// RateLimiterConfig builds the ratelimiter.Config used by gRPC admission. func (c GRPCConfig) RateLimiterConfig() ratelimiter.Config { return ratelimiter.Config{ RPS: c.IPRateLimitRPS, diff --git a/sei-cosmos/server/config/toml.go b/sei-cosmos/server/config/toml.go index 47aab9d055..e17e5e56cc 100644 --- a/sei-cosmos/server/config/toml.go +++ b/sei-cosmos/server/config/toml.go @@ -260,7 +260,7 @@ keepalive-min-time = "{{ .GRPC.KeepaliveMinTime }}" # KeepalivePermitWithoutStream defines whether the server allows keepalive pings even when there are no active streams. keepalive-permit-without-stream = {{ .GRPC.KeepalivePermitWithoutStream }} -# ip-rate-limit-rps is the per-IP sustained request rate in requests/second for native gRPC (:9090). +# ip-rate-limit-rps is the per-IP sustained request rate in requests/second. # Zero disables per-IP throttling; set rate-limiting-enabled = false for a full bypass. ip-rate-limit-rps = {{ .GRPC.IPRateLimitRPS }} @@ -268,9 +268,9 @@ ip-rate-limit-rps = {{ .GRPC.IPRateLimitRPS }} # Zero disables per-IP throttling (same effect as ip-rate-limit-rps = 0). ip-rate-limit-burst = {{ .GRPC.IPRateLimitBurst }} -# rate-limiting-enabled is the master switch for gRPC rate-limit admission interceptors. -# These interceptors also govern gRPC-Web (:9091), which wraps the same server, and its -# traffic draws from the same per-IP buckets as native gRPC (:9090). +# rate-limiting-enabled is the master switch for gRPC rate-limit admission. +# It governs gRPC-Web (:9091) as well as native gRPC (:9090), and both planes draw +# from the same per-IP buckets. rate-limiting-enabled = {{ .GRPC.RateLimitingEnabled }} # trusted-proxy-cidrs lists CIDRs whose x-forwarded-for metadata is trusted when diff --git a/sei-cosmos/server/grpc/grpc_web.go b/sei-cosmos/server/grpc/grpc_web.go index e80e042751..2b90504c19 100644 --- a/sei-cosmos/server/grpc/grpc_web.go +++ b/sei-cosmos/server/grpc/grpc_web.go @@ -11,12 +11,16 @@ import ( "golang.org/x/net/netutil" "google.golang.org/grpc" + "github.com/sei-protocol/sei-chain/ratelimiter" "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" "github.com/sei-protocol/sei-chain/sei-cosmos/server/types" ) -// StartGRPCWeb starts a gRPC-Web server on the given address. -func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, error) { +// StartGRPCWeb starts a gRPC-Web server on the given address. registry is the +// rate-limit registry StartGRPCServer returned; when it is non-nil the two +// planes admit against the same per-IP buckets, and when it is nil gRPC-Web +// serves without per-IP admission. +func StartGRPCWeb(grpcSrv *grpc.Server, registry *ratelimiter.Registry, config config.Config) (*http.Server, error) { var options []grpcweb.Option if config.GRPCWeb.EnableUnsafeCORS { options = append(options, @@ -26,9 +30,12 @@ func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, err ) } - wrappedServer := grpcweb.WrapServer(grpcSrv, options...) + var handler http.Handler = grpcweb.WrapServer(grpcSrv, options...) + if registry != nil { + handler = RateLimitHTTPMiddleware(registry, handler) + } grpcWebSrv := &http.Server{ - Handler: wrappedServer, + Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 2 * time.Minute, diff --git a/sei-cosmos/server/grpc/grpc_web_timeout_test.go b/sei-cosmos/server/grpc/grpc_web_timeout_test.go index 7b2e3437a0..743ff29c93 100644 --- a/sei-cosmos/server/grpc/grpc_web_timeout_test.go +++ b/sei-cosmos/server/grpc/grpc_web_timeout_test.go @@ -29,7 +29,7 @@ func TestStartGRPCWebTimeouts(t *testing.T) { }, } - srv, err := srvgrpc.StartGRPCWeb(grpcSrv, cfg) + srv, err := srvgrpc.StartGRPCWeb(grpcSrv, nil, cfg) require.NoError(t, err) require.NotNil(t, srv) defer func() { diff --git a/sei-cosmos/server/grpc/rate_limit.go b/sei-cosmos/server/grpc/rate_limit.go index 930e1da06c..8e70f52708 100644 --- a/sei-cosmos/server/grpc/rate_limit.go +++ b/sei-cosmos/server/grpc/rate_limit.go @@ -2,23 +2,86 @@ package grpc import ( "context" + "net/http" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/grpc/tap" "github.com/sei-protocol/sei-chain/ratelimiter" ) var errRateLimited = status.Error(codes.ResourceExhausted, "too many requests") +// admittedIPKey addresses the client IP recorded when the tap handler charges an RPC. +type admittedIPKey struct{} + +// admittedIP returns the client IP the tap handler charged for this RPC and +// whether the tap handler ran at all. +func admittedIP(ctx context.Context) (string, bool) { + ip, ok := ctx.Value(admittedIPKey{}).(string) + return ip, ok +} + +// RateLimitTapHandle returns a tap handler that charges the per-IP bucket when a +// stream's headers arrive. registry must be non-nil. +// +// This is the earliest admission point gRPC offers: it runs before the request +// message is read off the wire and unmarshalled, so a throttled caller cannot +// spend the server's decoder. Interceptors run after that decode and so cannot. +// It executes on the connection's I/O goroutine, which is why it does nothing +// beyond the bucket check. +// +// Only the HTTP/2 transport calls it. gRPC-Web reaches the same server through +// ServeHTTP, where RateLimitHTTPMiddleware is the admission point instead. +func RateLimitTapHandle(registry *ratelimiter.Registry) tap.ServerInHandle { + return func(ctx context.Context, info *tap.Info) (context.Context, error) { + ip := registry.IPFromGRPCContext(ctx) + if !registry.Allow(ctx, ip, ratelimiter.PlaneGRPC, info.FullMethodName) { + return ctx, errRateLimited + } + return context.WithValue(ctx, admittedIPKey{}, ip), nil + } +} + +// RateLimitHTTPMiddleware returns a handler that charges the per-IP bucket +// before next sees the request. registry must be non-nil. +// +// This is the admission point for gRPC-Web: it reaches the gRPC server through +// ServeHTTP rather than the HTTP/2 transport, so the tap handler never runs and +// the interceptors would otherwise admit it only after the body was read and +// decoded. The admission is recorded on the request context, which becomes the +// stream context, so the interceptors do not charge the RPC a second time. +func RateLimitHTTPMiddleware(registry *ratelimiter.Registry, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // A CORS preflight carries no RPC payload, and charging it would cost a + // browser two tokens per call. + if r.Method == http.MethodOptions { + next.ServeHTTP(w, r) + return + } + ip := registry.IPFromHTTPRequest(r) + if !registry.Allow(r.Context(), ip, ratelimiter.PlaneGRPC, r.URL.Path) { + http.Error(w, "too many requests", http.StatusTooManyRequests) + return + } + next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), admittedIPKey{}, ip))) + }) +} + // UnaryRateLimitInterceptor returns a server interceptor that applies per-IP // token-bucket rate limiting before the handler runs. registry must be non-nil. +// +// An RPC the tap handler or RateLimitHTTPMiddleware already charged passes +// through untouched, so a request costs one token however it reached the server. func UnaryRateLimitInterceptor(registry *ratelimiter.Registry) grpc.UnaryServerInterceptor { return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - ip := registry.IPFromGRPCContext(ctx) - if !registry.Allow(ctx, ip, ratelimiter.PlaneGRPC, info.FullMethod) { - return nil, errRateLimited + if _, admitted := admittedIP(ctx); !admitted { + ip := registry.IPFromGRPCContext(ctx) + if !registry.Allow(ctx, ip, ratelimiter.PlaneGRPC, info.FullMethod) { + return nil, errRateLimited + } } return handler(ctx, req) } @@ -28,13 +91,18 @@ func UnaryRateLimitInterceptor(registry *ratelimiter.Registry) grpc.UnaryServerI // token-bucket rate limiting to client streams. registry must be non-nil. // // A stream costs one token to establish and one more per inbound message, so an -// admitted stream cannot serve unbounded work. +// admitted stream cannot serve unbounded work. The establishment token is +// charged here only when neither the tap handler nor RateLimitHTTPMiddleware +// already charged it. func StreamRateLimitInterceptor(registry *ratelimiter.Registry) grpc.StreamServerInterceptor { return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { ctx := ss.Context() - ip := registry.IPFromGRPCContext(ctx) - if !registry.Allow(ctx, ip, ratelimiter.PlaneGRPC, info.FullMethod) { - return errRateLimited + ip, admitted := admittedIP(ctx) + if !admitted { + ip = registry.IPFromGRPCContext(ctx) + if !registry.Allow(ctx, ip, ratelimiter.PlaneGRPC, info.FullMethod) { + return errRateLimited + } } return handler(srv, rateLimitedServerStream{ ServerStream: ss, @@ -56,7 +124,11 @@ type rateLimitedServerStream struct { } func (s rateLimitedServerStream) RecvMsg(m interface{}) error { - // Charge after the read so a blocked stream holds no token and EOF is free. + // Charge after the read so a caller is only ever throttled for a message it + // actually sent: charging before would kill an idle stream whose client is + // waiting rather than sending, and would spend a token on EOF. The cost is + // that one message per stream may be decoded past the budget, bounded by + // MaxRecvMsgSize. if err := s.ServerStream.RecvMsg(m); err != nil { return err } diff --git a/sei-cosmos/server/grpc/rate_limit_dispatch_test.go b/sei-cosmos/server/grpc/rate_limit_dispatch_test.go index 041ce6ae58..b89cdcb237 100644 --- a/sei-cosmos/server/grpc/rate_limit_dispatch_test.go +++ b/sei-cosmos/server/grpc/rate_limit_dispatch_test.go @@ -3,6 +3,7 @@ package grpc import ( "context" "net" + "sync/atomic" "testing" "github.com/stretchr/testify/require" @@ -10,6 +11,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/encoding/proto" rpb "google.golang.org/grpc/reflection/grpc_reflection_v1" "google.golang.org/grpc/status" @@ -109,3 +112,48 @@ func TestStreamRateLimitInterceptor_ReflectionChargesPerMessage(t *testing.T) { require.Error(t, err) require.Equal(t, codes.ResourceExhausted, status.Code(err)) } + +// countingCodec delegates to the server's proto codec and counts the request +// messages it decodes. +type countingCodec struct { + encoding.Codec + unmarshals atomic.Int64 +} + +func (c *countingCodec) Unmarshal(data []byte, v interface{}) error { + c.unmarshals.Add(1) + return c.Codec.Unmarshal(data, v) +} + +// TestRateLimitTapHandle_RejectsBeforeRequestIsDecoded is the reason the tap +// handler exists: a throttled caller must not reach the protobuf decoder, which +// an interceptor cannot prevent because gRPC decodes the request before the +// interceptor chain runs. +func TestRateLimitTapHandle_RejectsBeforeRequestIsDecoded(t *testing.T) { + registry := mustNewRegistry(t, cfg(0.001, 1)) + codec := &countingCodec{Codec: encoding.GetCodec(proto.Name)} + + srv := grpc.NewServer( + grpc.InTapHandle(RateLimitTapHandle(registry)), + grpc.ForceServerCodec(codec), + ) + testdata.RegisterQueryServer(srv, testdata.QueryImpl{}) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go func() { _ = srv.Serve(listener) }() + t.Cleanup(srv.Stop) + + conn, err := grpc.Dial(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + client := testdata.NewQueryClient(conn) + + _, err = client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.NoError(t, err) + require.Equal(t, int64(1), codec.unmarshals.Load()) + + _, err = client.Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + require.Equal(t, int64(1), codec.unmarshals.Load(), "rejected request must not reach the decoder") +} diff --git a/sei-cosmos/server/grpc/rate_limit_test.go b/sei-cosmos/server/grpc/rate_limit_test.go index 2234e4865c..b5fa5dc0e8 100644 --- a/sei-cosmos/server/grpc/rate_limit_test.go +++ b/sei-cosmos/server/grpc/rate_limit_test.go @@ -4,6 +4,8 @@ import ( "context" "io" "net" + "net/http" + "net/http/httptest" "testing" "github.com/stretchr/testify/require" @@ -12,6 +14,7 @@ import ( "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" "google.golang.org/grpc/status" + "google.golang.org/grpc/tap" "github.com/sei-protocol/sei-chain/ratelimiter" ) @@ -189,3 +192,169 @@ func TestStreamRateLimitInterceptor_RecvErrorIsNotCharged(t *testing.T) { require.Equal(t, io.EOF, ic(nil, stream, info, handler)) } + +func TestRateLimitTapHandle_AllowThenReject(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1)) + tapHandle := RateLimitTapHandle(reg) + info := &tap.Info{FullMethodName: "/cosmos.bank.v1beta1.Query/Balance"} + ctx := grpcCtx(t.Context(), "10.0.0.1:9000") + + admittedCtx, err := tapHandle(ctx, info) + require.NoError(t, err) + ip, admitted := admittedIP(admittedCtx) + require.True(t, admitted) + require.Equal(t, "10.0.0.1", ip) + + _, err = tapHandle(ctx, info) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +func TestRateLimitTapHandle_TrustedProxyXFF(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1, "10.0.0.0/8")) + tapHandle := RateLimitTapHandle(reg) + info := &tap.Info{FullMethodName: "/cosmos.bank.v1beta1.Query/Balance"} + ctx := grpcCtx(t.Context(), "10.0.0.2:9000", "203.0.113.5, 10.0.0.2") + + admittedCtx, err := tapHandle(ctx, info) + require.NoError(t, err) + ip, _ := admittedIP(admittedCtx) + require.Equal(t, "203.0.113.5", ip) + + _, err = tapHandle(ctx, info) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +// TestUnaryRateLimitInterceptor_TapAdmissionIsNotChargedTwice pins that stacking +// the tap handler and the interceptor costs one token per RPC: with a burst of +// 1, the interceptor must not spend the token the tap handler left behind. +func TestUnaryRateLimitInterceptor_TapAdmissionIsNotChargedTwice(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 2)) + tapHandle := RateLimitTapHandle(reg) + ic := UnaryRateLimitInterceptor(reg) + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return "ok", nil + } + info := &grpc.UnaryServerInfo{FullMethod: "/cosmos.bank.v1beta1.Query/Balance"} + ctx := grpcCtx(t.Context(), "10.0.0.1:9000") + + for range 2 { + admittedCtx, err := tapHandle(ctx, &tap.Info{FullMethodName: info.FullMethod}) + require.NoError(t, err) + _, err = ic(admittedCtx, nil, info, handler) + require.NoError(t, err) + } + + _, err := tapHandle(ctx, &tap.Info{FullMethodName: info.FullMethod}) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +// TestStreamRateLimitInterceptor_TapAdmissionSkipsEstablishmentCharge pins that +// a tapped stream pays for establishment once: a burst of 2 covers the tap +// admission and one message, and the second message is rejected. +func TestStreamRateLimitInterceptor_TapAdmissionSkipsEstablishmentCharge(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 2)) + tapHandle := RateLimitTapHandle(reg) + ic := StreamRateLimitInterceptor(reg) + info := &grpc.StreamServerInfo{FullMethod: "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo"} + + admittedCtx, err := tapHandle(grpcCtx(t.Context(), "10.0.0.1:9000"), &tap.Info{FullMethodName: info.FullMethod}) + require.NoError(t, err) + + handler := func(_ interface{}, stream grpc.ServerStream) error { + require.NoError(t, stream.RecvMsg(nil)) + return stream.RecvMsg(nil) + } + err = ic(nil, mockServerStream{ctx: admittedCtx}, info, handler) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +func TestRateLimitHTTPMiddleware_AllowThenReject(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1)) + var served int + handler := RateLimitHTTPMiddleware(reg, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + served++ + })) + + req := func() *http.Request { + r := httptest.NewRequest(http.MethodPost, "/testdata.Query/Echo", nil) + r.RemoteAddr = "10.0.0.1:9000" + return r + } + + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req()) + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, 1, served) + + rec = httptest.NewRecorder() + handler.ServeHTTP(rec, req()) + require.Equal(t, http.StatusTooManyRequests, rec.Code) + require.Equal(t, 1, served, "rejected request must not reach the wrapped server") +} + +// TestRateLimitHTTPMiddleware_RecordsAdmissionOnRequestContext pins the handoff +// to the interceptors: the request context becomes the stream context, so the +// admission recorded here is what stops the RPC being charged twice. +func TestRateLimitHTTPMiddleware_RecordsAdmissionOnRequestContext(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1)) + var got string + var admitted bool + handler := RateLimitHTTPMiddleware(reg, http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + got, admitted = admittedIP(r.Context()) + })) + + r := httptest.NewRequest(http.MethodPost, "/testdata.Query/Echo", nil) + r.RemoteAddr = "10.0.0.1:9000" + handler.ServeHTTP(httptest.NewRecorder(), r) + + require.True(t, admitted) + require.Equal(t, "10.0.0.1", got) +} + +// TestRateLimitHTTPMiddleware_PreflightIsNotCharged keeps a browser's CORS +// preflight from costing the same token as the call it precedes. +func TestRateLimitHTTPMiddleware_PreflightIsNotCharged(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1)) + handler := RateLimitHTTPMiddleware(reg, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + + for range 3 { + r := httptest.NewRequest(http.MethodOptions, "/testdata.Query/Echo", nil) + r.RemoteAddr = "10.0.0.1:9000" + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, r) + require.Equal(t, http.StatusOK, rec.Code) + } + + r := httptest.NewRequest(http.MethodPost, "/testdata.Query/Echo", nil) + r.RemoteAddr = "10.0.0.1:9000" + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, r) + require.Equal(t, http.StatusOK, rec.Code) +} + +func TestRateLimitHTTPMiddleware_TrustedProxyXFF(t *testing.T) { + reg := mustNewRegistry(t, cfg(0.001, 1, "10.0.0.0/8")) + handler := RateLimitHTTPMiddleware(reg, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + + req := func() *http.Request { + r := httptest.NewRequest(http.MethodPost, "/testdata.Query/Echo", nil) + r.RemoteAddr = "10.0.0.2:9000" + r.Header.Set("X-Forwarded-For", "203.0.113.5, 10.0.0.2") + return r + } + + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req()) + require.Equal(t, http.StatusOK, rec.Code) + + rec = httptest.NewRecorder() + handler.ServeHTTP(rec, req()) + require.Equal(t, http.StatusTooManyRequests, rec.Code) + + // A different client behind the same proxy keeps its own bucket. + r := req() + r.Header.Set("X-Forwarded-For", "203.0.113.9, 10.0.0.2") + rec = httptest.NewRecorder() + handler.ServeHTTP(rec, r) + require.Equal(t, http.StatusOK, rec.Code) +} diff --git a/sei-cosmos/server/grpc/server.go b/sei-cosmos/server/grpc/server.go index 05bb4c1fc1..72754e0f4f 100644 --- a/sei-cosmos/server/grpc/server.go +++ b/sei-cosmos/server/grpc/server.go @@ -22,7 +22,7 @@ import ( var logger = seilog.NewLogger("cosmos", "server", "grpc") -// rateLimitServerOptions returns the unary and stream interceptor options that +// rateLimitServerOptions returns the tap-handler and interceptor options that // apply per-IP admission to the gRPC plane. It returns nil when cfg leaves // rate limiting disabled. func rateLimitServerOptions(cfg config.GRPCConfig) ([]grpc.ServerOption, *ratelimiter.Registry, error) { @@ -33,9 +33,9 @@ func rateLimitServerOptions(cfg config.GRPCConfig) ([]grpc.ServerOption, *rateli if err != nil { return nil, nil, fmt.Errorf("grpc rate limiter: %w", err) } - // A zeroed bucket makes Allow admit unconditionally, so the interceptors - // install and throttle nothing. The CometBFT RPC plane logs the same - // combination; without this the gRPC plane would fail open silently. + // A zeroed bucket makes Allow admit unconditionally, so admission installs and + // throttles nothing. The CometBFT RPC plane logs the same combination; without + // this the gRPC plane would fail open silently. if cfg.IPRateLimitRPS <= 0 || cfg.IPRateLimitBurst <= 0 { logger.Info( "gRPC rate-limit admission is enabled but the token bucket is disabled "+ @@ -45,6 +45,12 @@ func rateLimitServerOptions(cfg config.GRPCConfig) ([]grpc.ServerOption, *rateli ) } return []grpc.ServerOption{ + // Admission happens before the request is decoded: the tap handler + // covers native gRPC, and RateLimitHTTPMiddleware covers gRPC-Web, + // which reaches this server through ServeHTTP. The interceptors charge + // what neither can see, each message on an established stream, and + // admit any RPC that arrived uncharged. + grpc.InTapHandle(RateLimitTapHandle(registry)), grpc.ChainUnaryInterceptor(UnaryRateLimitInterceptor(registry)), grpc.ChainStreamInterceptor(StreamRateLimitInterceptor(registry)), }, registry, nil @@ -63,8 +69,10 @@ func registeredMethods(srv *grpc.Server) []string { return methods } -// StartGRPCServer starts a gRPC server on the address given by cfg. -func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config.GRPCConfig) (*grpc.Server, error) { +// StartGRPCServer starts a gRPC server on the address given by cfg. It returns +// the rate-limit registry the server admits against, or nil when cfg leaves rate +// limiting disabled, so the gRPC-Web server can share the same per-IP buckets. +func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config.GRPCConfig) (*grpc.Server, *ratelimiter.Registry, error) { maxRecvMsgSize := cfg.MaxRecvMsgSize if maxRecvMsgSize <= 0 { maxRecvMsgSize = config.DefaultGRPCMaxRecvMsgSize @@ -72,13 +80,13 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config rateLimitOpts, rateLimitRegistry, err := rateLimitServerOptions(cfg) if err != nil { - return nil, err + return nil, nil, err } serverOpts := append([]grpc.ServerOption{ grpc.MaxConcurrentStreams(100), - // MaxRecvMsgSize bounds per-request memory allocation before the rate - // limiter fires, preventing an oversized request from exhausting memory. + // MaxRecvMsgSize bounds the memory an admitted request may allocate, + // preventing an oversized request from exhausting memory. grpc.MaxRecvMsgSize(maxRecvMsgSize), grpc.KeepaliveParams(keepalive.ServerParameters{ MaxConnectionIdle: cfg.MaxConnectionIdle, @@ -110,7 +118,7 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config InterfaceRegistry: clientCtx.InterfaceRegistry, }) if err != nil { - return nil, err + return nil, nil, err } // Reflection allows external clients to see what services and methods // the gRPC server exposes. @@ -121,7 +129,7 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config listener, err := net.Listen("tcp", cfg.Address) if err != nil { - return nil, err + return nil, nil, err } if cfg.MaxOpenConnections > 0 { maxConn := cfg.MaxOpenConnections @@ -141,8 +149,8 @@ func StartGRPCServer(clientCtx client.Context, app types.Application, cfg config select { case err := <-errCh: - return nil, err + return nil, nil, err case <-time.After(types.ServerStartTime): // assume server started successfully - return grpcSrv, nil + return grpcSrv, rateLimitRegistry, nil } } diff --git a/sei-cosmos/server/grpc/server_rate_limit_test.go b/sei-cosmos/server/grpc/server_rate_limit_test.go index 6a4c53c1f4..c62970522a 100644 --- a/sei-cosmos/server/grpc/server_rate_limit_test.go +++ b/sei-cosmos/server/grpc/server_rate_limit_test.go @@ -1,8 +1,10 @@ package grpc import ( + "bytes" "context" "net" + "net/http" "testing" "github.com/stretchr/testify/require" @@ -16,6 +18,8 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" + "github.com/sei-protocol/sei-chain/ratelimiter" + "github.com/sei-protocol/sei-chain/sei-cosmos/baseapp" "github.com/sei-protocol/sei-chain/sei-cosmos/client" cryptotypes "github.com/sei-protocol/sei-chain/sei-cosmos/crypto/types" @@ -72,7 +76,7 @@ func startGRPCServerOnFreePort(t *testing.T, cfg config.GRPCConfig) (*grpc.Serve WithInterfaceRegistry(encCfg.InterfaceRegistry) cfg.Address = freeTCPAddr(t) - srv, err := StartGRPCServer(clientCtx, startGRPCServerApp{app}, cfg) + srv, _, err := StartGRPCServer(clientCtx, startGRPCServerApp{app}, cfg) require.NoError(t, err) return srv, cfg.Address @@ -167,7 +171,7 @@ func TestStartGRPCServer_MalformedTrustedProxyCIDR(t *testing.T) { cfg.TrustedProxyCIDRs = []string{"not-a-cidr"} cfg.Address = freeTCPAddr(t) - _, err := StartGRPCServer(client.Context{}, startGRPCServerApp{}, cfg) + _, _, err := StartGRPCServer(client.Context{}, startGRPCServerApp{}, cfg) require.ErrorContains(t, err, "grpc rate limiter") } @@ -180,3 +184,101 @@ func TestRegisteredMethods(t *testing.T) { require.Contains(t, methods, "testdata.Query/Echo") require.Contains(t, methods, "grpc.reflection.v1.ServerReflection/ServerReflectionInfo") } + +// TestStartGRPCWeb_SharesRateLimitBucketsWithGRPC covers the plumbing that hands +// StartGRPCWeb the registry StartGRPCServer built: gRPC-Web is admitted, and it +// spends from the same per-IP bucket as :9090 rather than a second one that +// would double a client's budget. +func TestStartGRPCWeb_SharesRateLimitBucketsWithGRPC(t *testing.T) { + grpcCfg := rateLimitedGRPCConfig(0.001, 2) + grpcCfg.Address = freeTCPAddr(t) + + srv, registry := startGRPCServerWithRegistry(t, grpcCfg) + require.NotNil(t, registry) + + webAddr := freeTCPAddr(t) + webSrv, err := StartGRPCWeb(srv, registry, config.Config{ + GRPCWeb: config.GRPCWebConfig{Enable: true, Address: webAddr}, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = webSrv.Close() }) + + conn, err := grpc.Dial(grpcCfg.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + // The first token goes to the native plane. + _, err = testdata.NewQueryClient(conn).Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.NoError(t, err) + + // The second goes to gRPC-Web, which leaves the shared bucket empty. + require.NotEqual(t, http.StatusTooManyRequests, postGRPCWeb(t, webAddr)) + require.Equal(t, http.StatusTooManyRequests, postGRPCWeb(t, webAddr)) + + // The native plane is throttled by the tokens gRPC-Web spent. + _, err = testdata.NewQueryClient(conn).Echo(context.Background(), &testdata.EchoRequest{Message: "hello"}) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +// TestStartGRPCWeb_RateLimitingDisabledServesUnthrottled pins that a nil +// registry leaves gRPC-Web serving without admission. +func TestStartGRPCWeb_RateLimitingDisabledServesUnthrottled(t *testing.T) { + grpcCfg := rateLimitedGRPCConfig(0.001, 1) + grpcCfg.RateLimitingEnabled = false + grpcCfg.Address = freeTCPAddr(t) + + srv, registry := startGRPCServerWithRegistry(t, grpcCfg) + require.Nil(t, registry) + + webAddr := freeTCPAddr(t) + webSrv, err := StartGRPCWeb(srv, registry, config.Config{ + GRPCWeb: config.GRPCWebConfig{Enable: true, Address: webAddr}, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = webSrv.Close() }) + + for range 3 { + require.NotEqual(t, http.StatusTooManyRequests, postGRPCWeb(t, webAddr)) + } +} + +func startGRPCServerWithRegistry(t *testing.T, cfg config.GRPCConfig) (*grpc.Server, *ratelimiter.Registry) { + t.Helper() + + app := baseapp.NewBaseApp(t.Name(), dbm.NewMemDB(), nil, nil, &testutil.TestAppOpts{}) + app.MountStores(sdk.NewKVStoreKey("test")) + require.NoError(t, app.LoadLatestVersion()) + + encCfg := moduletestutil.MakeTestEncodingConfig() + testdata.RegisterInterfaces(encCfg.InterfaceRegistry) + app.SetInterfaceRegistry(encCfg.InterfaceRegistry) + testdata.RegisterQueryServer(app.GRPCQueryRouter(), testdata.QueryImpl{}) + + clientCtx := client.Context{}. + WithChainID("test-chain"). + WithTxConfig(encCfg.TxConfig). + WithInterfaceRegistry(encCfg.InterfaceRegistry) + + srv, registry, err := StartGRPCServer(clientCtx, startGRPCServerApp{app}, cfg) + require.NoError(t, err) + t.Cleanup(srv.Stop) + + return srv, registry +} + +// postGRPCWeb sends a gRPC-Web request to addr and returns its HTTP status. The +// body is not a valid request frame: only whether admission let it through to +// the wrapped server is under test. +func postGRPCWeb(t *testing.T, addr string) int { + t.Helper() + + req, err := http.NewRequest(http.MethodPost, "http://"+addr+"/testdata.Query/Echo", bytes.NewReader([]byte{0, 0, 0, 0, 0})) + require.NoError(t, err) + req.Header.Set("content-type", "application/grpc-web+proto") + + res, err := http.DefaultClient.Do(req) + require.NoError(t, err) + t.Cleanup(func() { _ = res.Body.Close() }) + + return res.StatusCode +} diff --git a/sei-cosmos/server/start.go b/sei-cosmos/server/start.go index ff9e416b75..b8f58bdae2 100644 --- a/sei-cosmos/server/start.go +++ b/sei-cosmos/server/start.go @@ -418,14 +418,14 @@ func startInProcess( } if config.GRPC.Enable { - grpcSrv, err := servergrpc.StartGRPCServer(clientCtx, app, config.GRPC) + grpcSrv, rateLimitRegistry, err := servergrpc.StartGRPCServer(clientCtx, app, config.GRPC) if err != nil { return err } defer grpcSrv.Stop() if config.GRPCWeb.Enable { - grpcWebSrv, err := servergrpc.StartGRPCWeb(grpcSrv, config) + grpcWebSrv, err := servergrpc.StartGRPCWeb(grpcSrv, rateLimitRegistry, config) if err != nil { logger.Error("failed to start grpc-web http server", "err", err) return err diff --git a/sei-cosmos/testutil/network/util.go b/sei-cosmos/testutil/network/util.go index 49b8f91557..65cca9354c 100644 --- a/sei-cosmos/testutil/network/util.go +++ b/sei-cosmos/testutil/network/util.go @@ -102,7 +102,7 @@ func startInProcess(cfg Config, val *Validator) error { } if val.AppConfig.GRPC.Enable { - grpcSrv, err := servergrpc.StartGRPCServer(val.ClientCtx, app, val.AppConfig.GRPC) + grpcSrv, rateLimitRegistry, err := servergrpc.StartGRPCServer(val.ClientCtx, app, val.AppConfig.GRPC) if err != nil { return err } @@ -110,7 +110,7 @@ func startInProcess(cfg Config, val *Validator) error { val.grpc = grpcSrv if val.AppConfig.GRPCWeb.Enable { - val.grpcWeb, err = servergrpc.StartGRPCWeb(grpcSrv, *val.AppConfig) + val.grpcWeb, err = servergrpc.StartGRPCWeb(grpcSrv, rateLimitRegistry, *val.AppConfig) if err != nil { return err } From e6aebcf452cc1a72b0b7d7b465200598b3590579 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Mon, 31 Aug 2026 18:58:55 +0200 Subject: [PATCH 08/12] PLT-1072: Lower default gRPC rate limit to 10 rps / 20 burst The gRPC plane read its per-IP defaults from ratelimiter.DefaultRPS and DefaultBurst, whose names claim a scope wider than the one plane that consumes them. Declare the numbers in the gRPC config package alongside the other DefaultGRPC* knobs, matching how the EVM and CometBFT planes already carry their own, and leave the ratelimiter constants at 200/400. The stream path charges a token per inbound message, so a burst of 20 is low enough to cut off a streaming client mid-call. Admission is still off by default; note the sizing consideration in the upgrade guide. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 3 ++- sei-cosmos/server/config/config.go | 16 ++++++++++++---- sei-cosmos/server/config/config_test.go | 5 ++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82d81c31e3..a1823b7efc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#3984](https://github.com/sei-protocol/sei-chain/pull/3984) feat(query): origin-aware pagination limits for ABCI queries. Untrusted callers on the ABCI/gRPC query path get configurable `max-limit`, `max-offset`, and flat `max-iterations` (defaults: 1000 / 10000 / 11000); requests above the caps are rejected upfront, and an exhausted iteration budget returns a partial page with `next_key` instead of failing. Trusted origins (new `[query] trusted-cidrs`) and the `[query] disable-limits` kill switch bypass the caps; the consensus/EVM precompile path is unaffected. * [#3990](https://github.com/sei-protocol/sei-chain/pull/3990) Freeze mode is limited to full nodes and disables transaction and evidence submission, mempool gossip, and state sync from startup while preserving query RPC and mempool-backed reads. Frozen and Autobahn nodes no longer advertise the unused mempool P2P channel. * [#4009](https://github.com/sei-protocol/sei-chain/pull/4009) Bound `/store/*/subspace` ABCI queries with pair/byte caps, empty-prefix rejection, SS-path concurrency limits, and context-aware iteration to prevent memory-exhaustion DoS. -* [#4021](https://github.com/sei-protocol/sei-chain/pull/4021) feat(grpc): per-IP rate-limit admission for the gRPC plane, off by default behind `[grpc] rate-limiting-enabled` (new `ip-rate-limit-rps` / `ip-rate-limit-burst` / `trusted-proxy-cidrs`, defaults 200 rps / 400 burst / trust no proxy). Native gRPC (:9090) is admitted by a tap handler and gRPC-Web (:9091) by HTTP middleware, both before the request is protobuf-decoded, so a throttled caller cannot spend the decoder; streams pay one token to establish and one per inbound message. Both planes draw from the same per-IP buckets. Over-budget callers get `ResourceExhausted` on :9090 and HTTP 429 on :9091, counted by `rpc_rate_limit_rejected_total{plane="grpc", method_namespace}`. +* [#4021](https://github.com/sei-protocol/sei-chain/pull/4021) feat(grpc): per-IP rate-limit admission for the gRPC plane, off by default behind `[grpc] rate-limiting-enabled` (new `ip-rate-limit-rps` / `ip-rate-limit-burst` / `trusted-proxy-cidrs`, defaults 10 rps / 20 burst / trust no proxy). Native gRPC (:9090) is admitted by a tap handler and gRPC-Web (:9091) by HTTP middleware, both before the request is protobuf-decoded, so a throttled caller cannot spend the decoder; streams pay one token to establish and one per inbound message. Both planes draw from the same per-IP buckets. Over-budget callers get `ResourceExhausted` on :9090 and HTTP 429 on :9091, counted by `rpc_rate_limit_rejected_total{plane="grpc", method_namespace}`. ### Upgrade guide * **IBC core removal.** Removes the retired IBC core source, protobufs, light clients, CLI, and simulation support. Retired IBC stores remain mounted but are omitted from `export-genesis`; preserve the state database or use v6.6 freeze nodes for historical IBC data. @@ -45,6 +45,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#3958](https://github.com/sei-protocol/sei-chain/pull/3958) **Feegrant removal.** Removes feegrant execution, module APIs, and the unreleased feegrant EVM precompile. The feegrant store remains mounted for historical state access. Transactions with a fee granter different from the payer are rejected. * **WebSocket frame size default drops from 10 MiB to 5 MiB.** Before this release, :8546 used a hardcoded 10 MiB frame cap. Both HTTP and WebSocket now share `[evm].max_request_body_bytes`, whose default is 5 MiB (`5242880`). WS clients that send frames in the 5-10 MiB range (large `eth_sendRawTransaction` batches, wide filter payloads, etc.) will be disconnected after upgrade unless the limit is raised. **Operators who relied on the old 10 MiB WS cap should set `max_request_body_bytes = 10485760` in `app.toml` before upgrading.** This also raises the HTTP body limit to 10 MiB. The exported `DefaultWebsocketMaxMessageSize` constant was removed; use the config knob instead. * [#3984](https://github.com/sei-protocol/sei-chain/pull/3984) **ABCI/gRPC pagination is now capped by default.** Untrusted callers requesting `limit` above 1000, `offset` above 10000, or a scan that exceeds 11000 total iterations now get `InvalidArgument` (over-cap) or a partial page with `next_key` (budget exhausted) instead of the previously unbounded scan. Clients that page with large limits/offsets, or trusted internal indexers, should either follow `next_key` for resumption or be added to the new `[query] trusted-cidrs` allowlist (or set `[query] disable-limits = true`) before upgrading. +* [#4021](https://github.com/sei-protocol/sei-chain/pull/4021) **gRPC per-IP rate limiting defaults are deliberately conservative.** Admission stays off unless `[grpc] rate-limiting-enabled = true`, but the defaults it enables are 10 rps / 20 burst per IP. Streams pay one token to establish and one per inbound message, so at the default burst a client that sends more than ~20 messages in a burst is cut off mid-stream with `ResourceExhausted`. Operators enabling admission should size `ip-rate-limit-rps` / `ip-rate-limit-burst` against their heaviest streaming client, and set `trusted-proxy-cidrs` to their ingress CIDRs so callers are not all bucketed under the proxy's IP. * [#4009](https://github.com/sei-protocol/sei-chain/pull/4009) **`/store/*/subspace` scans are now capped.** Wide prefix scans that previously returned unbounded KV pairs now fail with `subspace result exceeds limit` once they would exceed the default caps of 1,000 pairs or 4 MiB of accumulated key+value bytes. Empty prefixes are rejected. Indexers and tooling that issue wide `/subspace` queries must narrow their prefixes, shard by sub-prefix, or raise `[state-commit] sc-subspace-max-pairs` and `sc-subspace-max-bytes` before upgrading. Values `<= 0` resolve to these defaults; there is no unlimited setting. * [#3927](https://github.com/sei-protocol/sei-chain/pull/3927) **Legacy Sei JSON-RPC and CLI removal.** Removes `sei_associate`, `sei_getBlockByHash`, `sei_getBlockByHashExcludeTraceFail`, `sei_getBlockTransactionCountByHash`, `sei_getBlockTransactionCountByNumber`, `sei_getEvmTx`, `sei_getFilterChanges`, `sei_getFilterLogs`, `sei_getLogs`, `sei_getTransactionByBlockHashAndIndex`, `sei_getTransactionByBlockNumberAndIndex`, `sei_getTransactionByHash`, `sei_getTransactionCount`, `sei_getTransactionErrorByHash`, `sei_getTransactionReceiptExcludeTraceFail`, `sei_getVMError`, `sei_newBlockFilter`, `sei_newFilter`, `sei_sign`, and `sei_uninstallFilter`. Use standard `eth_*` methods for EVM-originated data and `seid tx evm native-associate -y` for address association. There is no block- or filter-level replacement for discovering Cosmos-originated synthetic logs; clients that know the synthetic transaction hash can enable `sei_getTransactionReceipt`. diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index bce723b8f9..657d00a26e 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -75,6 +75,14 @@ const ( // keepalive pings even when there are no active streams. DefaultGRPCKeepalivePermitWithoutStream = false + // DefaultGRPCIPRateLimitRPS is the default per-IP sustained request rate in + // requests/second for the gRPC plane. + DefaultGRPCIPRateLimitRPS = 10.0 + + // DefaultGRPCIPRateLimitBurst is the default maximum per-IP burst size for + // the gRPC plane. + DefaultGRPCIPRateLimitBurst = 20 + // DefaultOccEanbled defines whether to use OCC for tx processing DefaultOccEnabled = true ) @@ -416,8 +424,8 @@ func DefaultConfig() *Config { KeepaliveTimeout: DefaultGRPCKeepaliveTimeout, KeepaliveMinTime: DefaultGRPCKeepaliveMinTime, KeepalivePermitWithoutStream: DefaultGRPCKeepalivePermitWithoutStream, - IPRateLimitRPS: ratelimiter.DefaultRPS, - IPRateLimitBurst: ratelimiter.DefaultBurst, + IPRateLimitRPS: DefaultGRPCIPRateLimitRPS, + IPRateLimitBurst: DefaultGRPCIPRateLimitBurst, RateLimitingEnabled: false, TrustedProxyCIDRs: nil, }, @@ -609,11 +617,11 @@ func GetConfig(v *viper.Viper) (Config, error) { grpcMaxConnectionAge := clampNonNegativeDuration(v.GetDuration("grpc.max-connection-age"), DefaultGRPCMaxConnectionAge) grpcMaxConnectionAgeGrace := clampNonNegativeDuration(v.GetDuration("grpc.max-connection-age-grace"), DefaultGRPCMaxConnectionAgeGrace) - grpcIPRateLimitRPS := ratelimiter.DefaultRPS + grpcIPRateLimitRPS := DefaultGRPCIPRateLimitRPS if v.IsSet("grpc.ip-rate-limit-rps") { grpcIPRateLimitRPS = v.GetFloat64("grpc.ip-rate-limit-rps") } - grpcIPRateLimitBurst := ratelimiter.DefaultBurst + grpcIPRateLimitBurst := DefaultGRPCIPRateLimitBurst if v.IsSet("grpc.ip-rate-limit-burst") { grpcIPRateLimitBurst = v.GetInt("grpc.ip-rate-limit-burst") } diff --git a/sei-cosmos/server/config/config_test.go b/sei-cosmos/server/config/config_test.go index 8cebbfeac3..9f7926dd36 100644 --- a/sei-cosmos/server/config/config_test.go +++ b/sei-cosmos/server/config/config_test.go @@ -11,7 +11,6 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/require" - "github.com/sei-protocol/sei-chain/ratelimiter" storetypes "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config" @@ -82,8 +81,8 @@ func TestDefaultGRPCConfig(t *testing.T) { require.Equal(t, DefaultGRPCKeepaliveTimeout, cfg.GRPC.KeepaliveTimeout) require.Equal(t, DefaultGRPCKeepaliveMinTime, cfg.GRPC.KeepaliveMinTime) require.Equal(t, DefaultGRPCKeepalivePermitWithoutStream, cfg.GRPC.KeepalivePermitWithoutStream) - require.Equal(t, ratelimiter.DefaultRPS, cfg.GRPC.IPRateLimitRPS) - require.Equal(t, ratelimiter.DefaultBurst, cfg.GRPC.IPRateLimitBurst) + require.Equal(t, DefaultGRPCIPRateLimitRPS, cfg.GRPC.IPRateLimitRPS) + require.Equal(t, DefaultGRPCIPRateLimitBurst, cfg.GRPC.IPRateLimitBurst) require.False(t, cfg.GRPC.RateLimitingEnabled) require.Nil(t, cfg.GRPC.TrustedProxyCIDRs) } From 084c4cc4c11f08542de48e9d59d284a89fae2ba9 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Tue, 1 Sep 2026 13:06:58 +0200 Subject: [PATCH 09/12] Removed extra golden files --- sei-cosmos/server/config/testdata/grpc.keys.golden | 0 sei-cosmos/server/config/testdata/server_config.golden | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 sei-cosmos/server/config/testdata/grpc.keys.golden delete mode 100644 sei-cosmos/server/config/testdata/server_config.golden diff --git a/sei-cosmos/server/config/testdata/grpc.keys.golden b/sei-cosmos/server/config/testdata/grpc.keys.golden deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/sei-cosmos/server/config/testdata/server_config.golden b/sei-cosmos/server/config/testdata/server_config.golden deleted file mode 100644 index e69de29bb2..0000000000 From 1a420832a8923d9e489c9ebc03c93f34404edea9 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Tue, 1 Sep 2026 13:29:27 +0200 Subject: [PATCH 10/12] Added comments for loop back bucket --- ratelimiter/registry.go | 3 +++ sei-cosmos/server/config/toml.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ratelimiter/registry.go b/ratelimiter/registry.go index 31b259aee6..dc2c2c9c2c 100644 --- a/ratelimiter/registry.go +++ b/ratelimiter/registry.go @@ -216,6 +216,9 @@ func (r *Registry) getOrCreate(ip string) *rate.Limiter { } // bucketKey returns the LRU key for ip. +// The key is the address alone, so every client sharing an address shares one +// bucket: all processes reaching the node over 127.0.0.1, and any population +// behind a single egress address that TrustedProxyCIDRs does not cover. // IPv6 addresses are masked to /64: a client rotating within a /64 prefix // would otherwise get a fresh bucket per address. func bucketKey(ip string) string { diff --git a/sei-cosmos/server/config/toml.go b/sei-cosmos/server/config/toml.go index e17e5e56cc..ead31582bf 100644 --- a/sei-cosmos/server/config/toml.go +++ b/sei-cosmos/server/config/toml.go @@ -261,6 +261,9 @@ keepalive-min-time = "{{ .GRPC.KeepaliveMinTime }}" keepalive-permit-without-stream = {{ .GRPC.KeepalivePermitWithoutStream }} # ip-rate-limit-rps is the per-IP sustained request rate in requests/second. +# Buckets are keyed by client address alone, so all local clients reaching the +# node over 127.0.0.1 share one bucket, as does anything behind a single egress +# address not covered by trusted-proxy-cidrs. # Zero disables per-IP throttling; set rate-limiting-enabled = false for a full bypass. ip-rate-limit-rps = {{ .GRPC.IPRateLimitRPS }} From c622e14e352809bca35a006ddc7b0b19d0bf5d98 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Tue, 1 Sep 2026 14:26:40 +0200 Subject: [PATCH 11/12] Updating comments --- sei-cosmos/server/config/config_fuzz_test.go | 248 ++++++++----------- 1 file changed, 109 insertions(+), 139 deletions(-) diff --git a/sei-cosmos/server/config/config_fuzz_test.go b/sei-cosmos/server/config/config_fuzz_test.go index c7718cd92c..6206e427e4 100644 --- a/sei-cosmos/server/config/config_fuzz_test.go +++ b/sei-cosmos/server/config/config_fuzz_test.go @@ -13,24 +13,21 @@ import ( "github.com/spf13/viper" ) -// GetConfig is the second parser of app.toml, and the one that feeds api, grpc, -// grpc-web, rosetta, telemetry and state-sync. app/seidb.go parses [state-commit] -// and [state-store] out of the same viper for the store; GetConfig parses them -// again, by a different mechanism (viper.IsSet plus typed getters rather than -// appOpts.Get plus cast), into a Config nobody hands to the store. +// GetConfig is the second parser of app.toml, feeding api, grpc, grpc-web, rosetta, +// telemetry and state-sync. app/seidb.go parses [state-commit] and [state-store] out +// of the same viper for the store; GetConfig parses them again through a different +// mechanism (viper.IsSet plus typed getters) into a Config the store never sees. // -// Two parsers of one section is the drift risk, and the reason this file exists: it -// pins GetConfig's own resolution rules so a change that unified the parsers would -// show up as a diff here rather than as two components disagreeing about the same -// key at runtime. +// Two parsers of one section is the drift risk this file exists to pin: a change that +// unified them shows up as a diff here rather than as two components disagreeing at +// runtime. // -// Everything below drives a bare viper.New, which is what GetConfig takes. That -// deliberately excludes flag binding and env resolution — those belong to Apply and -// are exercised in cmd/seid/cmd. What is left is the parse itself. +// Everything below drives a bare viper.New, which is what GetConfig takes. Flag +// binding and env resolution belong to Apply and are exercised in cmd/seid/cmd. // newAppViper returns a viper holding the one key GetConfig unconditionally // requires, plus whatever the caller adds. telemetry.global-labels has no default -// and no presence guard, so nothing can be parsed without it. +// and no presence guard, so nothing parses without it. func newAppViper(t testing.TB, keys map[string]any) *viper.Viper { t.Helper() v := viper.New() @@ -41,24 +38,19 @@ func newAppViper(t testing.TB, keys map[string]any) *viper.Viper { return v } -// FuzzGetConfigGlobalLabels pins the one key that can stop a node booting by being -// absent rather than wrong. +// FuzzGetConfigGlobalLabels pins telemetry.global-labels, the key that stops a node +// booting by being absent rather than wrong. // -// telemetry.global-labels is read as a bare type assertion to []interface{} with no -// presence check, so an app.toml that omits the key entirely fails GetConfig -// outright — a node provisioned before the key existed does not start. Inside, each -// label is asserted to []interface{} and then its two elements to string with no -// checked assertion, so a label list holding a non-string panics rather than -// erroring. -// -// The shape rules are otherwise permissive in a way no operator would guess: a -// label whose length is not exactly 2 is silently dropped, not rejected. +// It is read as a bare type assertion with no presence check, so an app.toml that +// omits it fails GetConfig outright. Each label's elements are asserted to string +// unchecked, so a non-string label panics rather than erroring, and a label whose +// length is not exactly 2 is dropped silently rather than rejected. func FuzzGetConfigGlobalLabels(f *testing.F) { f.Add(0, 0, "chain") // no labels f.Add(1, 2, "chain") // one well-formed pair f.Add(3, 2, "chain") // several pairs - f.Add(1, 1, "chain") // one element: silently dropped - f.Add(1, 3, "chain") // three elements: silently dropped + f.Add(1, 1, "chain") // too few elements: silently dropped + f.Add(1, 3, "chain") // too many elements: silently dropped f.Add(1, 0, "chain") // empty label f.Add(2, 2, "") // empty strings are legal label content f.Add(1, 2, "a=b,c=d") // punctuation is not special @@ -98,8 +90,8 @@ func FuzzGetConfigGlobalLabels(f *testing.F) { }) } -// TestGetConfigRequiresGlobalLabels pins the absent-key failure on its own. This is -// what turns a missing telemetry section into a node that will not start. +// TestGetConfigRequiresGlobalLabels pins the absent-key failure on its own: a missing +// telemetry section is a node that will not start. func TestGetConfigRequiresGlobalLabels(t *testing.T) { _, err := GetConfig(viper.New()) if err == nil { @@ -112,9 +104,8 @@ func TestGetConfigRequiresGlobalLabels(t *testing.T) { } // TestGetConfigPanicsOnNonStringLabel records that the inner element assertions are -// unchecked. A label list of the right shape but the wrong element type takes the -// node down with a raw interface-conversion panic rather than an error naming -// telemetry. +// unchecked: a label of the right shape but the wrong element type takes the node down +// with a raw interface-conversion panic rather than an error naming telemetry. func TestGetConfigPanicsOnNonStringLabel(t *testing.T) { v := viper.New() v.Set("telemetry.global-labels", []any{[]any{1, 2}}) @@ -152,15 +143,12 @@ var grpcClamps = []grpcClamp{ // // gRPC accepts a negative keepalive verbatim and behaves pathologically, so GetConfig // substitutes the in-code default instead of passing it through. Only a negative is -// clamped, uniformly across all six keys: zero passes through everywhere, which matters -// most on the two age keys where gRPC reads zero as "no limit". Distinguishing negative -// from zero rather than treating both as unset is what this target holds in place. +// clamped, uniformly across the keys in grpcClamps: zero passes through, which matters +// on the age keys where gRPC reads zero as "no limit". // -// Each value is driven in two shapes, because a typed time.Duration is not a shape any -// app.toml can produce. A file gives "30s" or a bare integer and an environment variable -// always gives a string, so the typed form skips the cast.ToDuration step that sits between -// the file layer and this comparison. The string spelling is the one an operator actually -// writes, and "-1s" is how they would express the boundary that matters here. +// Each value is driven both typed and as its own String() spelling, because app.toml and +// the environment only ever produce text; the typed form skips the cast.ToDuration step +// that sits between the file layer and this comparison. func FuzzGetConfigGRPCDurationClamps(f *testing.F) { f.Add(uint(0), int64(0), false) f.Add(uint(0), int64(-1), false) @@ -180,15 +168,12 @@ func FuzzGetConfigGRPCDurationClamps(f *testing.F) { d := time.Duration(nanos) // A typed duration and its own String() spelling must resolve identically, since - // viper.GetDuration casts the text back. Any divergence is in cast, not in the clamp, - // and it belongs to this key because the file layer only ever produces the text. + // viper.GetDuration casts the text back. Any divergence is in cast, not in the clamp. var raw any = d if asString { - // Only a spelling that parses back to the same duration is a faithful stand-in for - // the typed value. Duration.String has not always round-tripped at the int64 - // boundary, and a spelling the parser rejects would resolve to zero and fail this - // case with a message about the clamp rather than about the encoding, so a value - // with no faithful text form is declined instead. + // Duration.String has not always round-tripped at the int64 boundary, and a + // spelling the parser rejects would resolve to zero and fail with a message about + // the clamp rather than about the encoding, so such a value is declined instead. spelled := d.String() if back, perr := time.ParseDuration(spelled); perr != nil || back != d { return // no faithful text spelling; the typed shape already covers this value @@ -222,10 +207,10 @@ func FuzzGetConfigGRPCDurationClamps(f *testing.F) { // // The rules match app/seidb.go — always parse, then let sc-write-mode-enable-auto // (default true, flipped only by an explicit key) decide whether the parsed mode is -// honored — but the mechanism differs: GetConfig returns an error where seidb.go -// panics. Both parsers must agree on the resolved mode for a node's store choice -// and its reported config to describe the same thing, so the agreement is asserted -// against the shared helpers rather than restated. +// honored — but GetConfig returns an error where seidb.go panics. Both parsers must +// agree on the resolved mode for a node's store choice and its reported config to +// describe the same thing, so the agreement is asserted against the shared helpers +// rather than restated. func FuzzGetConfigWriteMode(f *testing.F) { f.Add("memiavl_only", true, true) f.Add("memiavl_only", true, false) @@ -289,15 +274,14 @@ type guardedKey struct { // Set is a value distinguishable from the default, used to prove the guard // admits an explicit value as well as protecting an absent one. Set any - // DefaultIsZero marks a key whose in-code default is already the zero value. - // The guard still matters there — it is what lets an operator set a non-zero - // value — but an absent key resolving to zero is correct rather than a clobber, - // so the two cases need different assertions. + // DefaultIsZero marks a key whose in-code default is already the zero value, so an + // absent key resolving to zero is correct rather than a clobber. The guard still + // matters there — it is what lets an operator set a non-zero value. DefaultIsZero bool } -// guardedKeys are the GetConfig reads wrapped in viper.IsSet. They are the same -// zero-clobber class app/seidb.go guards, expressed through a different mechanism. +// guardedKeys are GetConfig reads wrapped in viper.IsSet: the same zero-clobber class +// app/seidb.go guards, expressed through a different mechanism. var guardedKeys = []guardedKey{ {Key: "state-commit.sc-async-commit-buffer", Path: "StateCommit.MemIAVLConfig.AsyncCommitBuffer", Set: 7}, {Key: "state-commit.sc-keep-recent", Path: "StateCommit.MemIAVLConfig.SnapshotKeepRecent", Set: 9}, @@ -329,11 +313,10 @@ var guardedKeys = []guardedKey{ // absent key resolves to the in-code default rather than to the zero value viper's // typed getters would otherwise return. // -// The keys that matter most are the bounded ones. grpc.max-recv-msg-size, -// grpc.max-open-connections and grpc-web.max-open-connections all default to a -// finite limit, and an unguarded read of an absent key would resolve 0 — which -// gRPC reads as unlimited. A node upgrading with an older app.toml would go from -// bounded to unbounded connections and message sizes with nothing said about it. +// The bounded gRPC keys matter most. They default to a finite limit, and an unguarded +// read of an absent key would resolve 0 — which gRPC reads as unlimited, so a node +// upgrading with an older app.toml would go from bounded to unbounded connections and +// message sizes with nothing said about it. func FuzzGetConfigGuardedKeysPreserveDefaults(f *testing.F) { for i := range len(guardedKeys) { f.Add(uint(i), false) @@ -353,13 +336,10 @@ func FuzzGetConfigGuardedKeysPreserveDefaults(f *testing.F) { } if !present { - // Whether the guard is doing anything is decided by comparing an absent key - // against an explicit zero, not against a synthesized zero literal. A - // synthesized one has to guess the field's Go type, and guessing wrong makes - // the comparison unsatisfiable and the assertion vacuous — which is exactly - // what an int-typed literal did for every uint, uint32 and float64 row here, - // including both gRPC connection bounds. Reading the reader twice needs no - // type knowledge at all. + // An absent key is compared against an explicit zero rather than a synthesized + // zero literal: a literal has to guess the field's Go type, and guessing wrong + // makes the comparison unsatisfiable and the assertion vacuous. Reading the + // reader twice needs no type knowledge at all. explicitZero, zeroErr := GetConfig(newAppViper(t, map[string]any{row.Key: 0})) if zeroErr != nil { t.Fatalf("%s = 0 must parse, got %v", row.Key, zeroErr) @@ -398,17 +378,15 @@ func FuzzGetConfigGuardedKeysPreserveDefaults(f *testing.F) { }) } -// TestGetConfigGuardedKeyDefaultsMatchTheTable keeps guardedKeys' DefaultIsZero -// column honest in both directions. -// -// It is what stops the guard assertions above from going vacuous. A key marked -// non-zero whose default moves to zero would make its clobber check meaningless; -// a key marked zero whose default becomes non-zero would leave a real clobber -// unchecked. Either way the table, not the assertion, is what needs updating. +// TestGetConfigGuardedKeyDefaultsMatchTheTable keeps guardedKeys' DefaultIsZero column +// honest in both directions, which is what stops the guard assertions above from going +// vacuous: a key marked non-zero whose default moves to zero makes its clobber check +// meaningless, and a key marked zero whose default becomes non-zero leaves a real +// clobber unchecked. // -// "Is the default zero" is answered by resolving the key explicitly as 0 and -// comparing, for the same reason the target above does it that way: a literal 0 -// carries Go's int type and would never compare equal to a uint or float64 leaf. +// "Is the default zero" is answered by resolving the key explicitly as 0 and comparing, +// for the same reason the target above does it that way: a literal 0 carries Go's int +// type and would never compare equal to a uint or float64 leaf. func TestGetConfigGuardedKeyDefaultsMatchTheTable(t *testing.T) { absent, err := GetConfig(newAppViper(t, nil)) if err != nil { @@ -442,11 +420,10 @@ func TestGetConfigGuardedKeyDefaultsMatchTheTable(t *testing.T) { } } -// TestGetConfigGenesisKeyDivergesFromTheAppSideKey records that the two genesis -// parsers read different keys for the same value. GetConfig reads -// genesis.genesis-stream-file; app/genesis.go reads genesis.import-file. Setting -// one leaves the other empty, so a stream-import node configured through the key -// the template renders streams from "". +// TestGetConfigGenesisKeyDivergesFromTheAppSideKey records that the two genesis parsers +// read different keys for the same value: GetConfig reads genesis.genesis-stream-file +// while app/genesis.go reads genesis.import-file. Setting one leaves the other empty, so +// a stream-import node configured through the key the template renders streams from "". func TestGetConfigGenesisKeyDivergesFromTheAppSideKey(t *testing.T) { cfg, err := GetConfig(newAppViper(t, map[string]any{ "genesis.stream-import": true, @@ -475,11 +452,10 @@ func TestGetConfigGenesisKeyDivergesFromTheAppSideKey(t *testing.T) { } } -// TestGetConfigStateStoreReadsAreUnguarded records that GetConfig's [state-store] -// parse has no presence checks, matching app/seidb.go's parseSSConfigs. An absent -// section resolves every field to its zero value, so the reported config says -// ss-enable false and an empty backend on a node whose app.toml simply predates the -// section. +// TestGetConfigStateStoreReadsAreUnguarded records that GetConfig's [state-store] parse +// has no presence checks, matching app/seidb.go's parseSSConfigs. An absent section +// resolves every field to its zero value, so the reported config says ss-enable false +// and an empty backend on a node whose app.toml predates the section. func TestGetConfigStateStoreReadsAreUnguarded(t *testing.T) { cfg, err := GetConfig(newAppViper(t, nil)) if err != nil { @@ -502,16 +478,16 @@ func TestGetConfigStateStoreReadsAreUnguarded(t *testing.T) { // TestGetConfigStateSyncReadsAreUnguarded records the [state-sync] clobber as a divergence // rather than as two facts that happen to disagree. // -// snapshot-keep-recent is a registered flag defaulting to 2 (server/start.go:234) and -// DefaultConfig declares 2, but GetConfig reads it with a bare v.GetUint32, so a viper that -// never saw the key resolves 0 — which toml.go:78 documents as "keep all". That viper is this -// file's layer and not a booted node's: start.go:117 binds the flag in PreRunE, ahead of both -// production calls (start.go:168 and :303), so there the same read takes the flag's 2 whenever -// app.toml is silent. +// snapshot-keep-recent is a registered flag with a non-zero default and DefaultConfig +// declares the same, but GetConfig reads it with a bare v.GetUint32, so a viper that never +// saw the key resolves 0 — which toml.go documents as "keep all". That viper is this file's +// layer and not a booted node's: server/start.go binds the flag in PreRunE ahead of both +// production calls, so there the same read takes the flag's value whenever app.toml is +// silent. // -// Either side moving fails this assertion — a guard makes the absent read return 2, and a -// default moved to 0 makes the clobber stop being one — so the divergence can neither close -// nor widen without a diff. +// Either side moving fails this assertion — a guard makes the absent read return the +// default, and a default moved to 0 makes the clobber stop being one — so the divergence +// can neither close nor widen without a diff. func TestGetConfigStateSyncReadsAreUnguarded(t *testing.T) { cfg, err := GetConfig(newAppViper(t, nil)) if err != nil { @@ -526,8 +502,8 @@ func TestGetConfigStateSyncReadsAreUnguarded(t *testing.T) { if cfg.StateSync.SnapshotKeepRecent != 0 { t.Fatalf("an absent state-sync.snapshot-keep-recent resolved to %d rather than 0, so the "+ "read is no longer unguarded. That is a fine end state and a no-op for a booted node, "+ - "which takes 2 from the bound flag either way (start.go:117 and :234); what it changes "+ - "is this assertion, so update it in the PR that adds the guard", + "which takes the bound flag's value either way; what it changes is this assertion, so "+ + "update it in the PR that adds the guard", cfg.StateSync.SnapshotKeepRecent) } if cfg.StateSync.SnapshotInterval != 0 || cfg.StateSync.SnapshotDirectory != "" { @@ -540,13 +516,12 @@ func TestGetConfigStateSyncReadsAreUnguarded(t *testing.T) { // readers of [state-sync], which is what a guard on GetConfig would close. // // Handed the same flagless viper, ParseConfig unmarshals over a DefaultConfig base and keeps -// snapshot-keep-recent at 2 while GetConfig's bare v.GetUint32 resolves 0. Nothing else in the -// tree states that — TestParseConfig (config_test.go:358) asserts only MinGasPrices. +// snapshot-keep-recent at its declared default while GetConfig's bare v.GetUint32 resolves 0. +// Nothing else in the tree states that — TestParseConfig asserts only MinGasPrices. // // The disagreement is between the two readers and not between two nodes. GetConfig's production -// calls (start.go:168 and :303) run with the flag bound, where it takes the same 2, and the one -// non-test caller of ParseConfig is server/util.go:308's empty-template branch, which no binary in -// this tree reaches. +// calls run with the flag bound, where it takes the declared value, and the one non-test caller +// of ParseConfig is server/util.go's empty-template branch, which no binary in this tree reaches. func TestParseConfigAndGetConfigDisagree(t *testing.T) { v := newAppViper(t, nil) parsed, err := ParseConfig(v) @@ -562,10 +537,8 @@ func TestParseConfigAndGetConfigDisagree(t *testing.T) { "%d through GetConfig, want 2 and 0. If GetConfig gained the guard the two readers now "+ "agree, which is the end state — update this test and "+ "TestGetConfigStateSyncReadsAreUnguarded together. If ParseConfig stopped resolving the "+ - "section over DefaultConfig, there is no implemented guard left to copy, and whoever "+ - "adds one is choosing a retention rather than matching one. A third cause is that the "+ - "in-code default moved off 2, which this literal does not name: a default of 3 reaches "+ - "here rather than the unguarded-read check, whose guard fires only at exactly 0", + "section over DefaultConfig, there is no implemented guard left to copy. If the in-code "+ + "default moved off 2, update this literal to match it", parsed.StateSync.SnapshotKeepRecent, direct.StateSync.SnapshotKeepRecent) } } @@ -573,11 +546,11 @@ func TestParseConfigAndGetConfigDisagree(t *testing.T) { // FuzzConfigValidateBasic pins the two conditions that reject an otherwise // parseable app.toml. // -// An empty minimum-gas-prices fails, because a validator accepting zero-fee -// transactions is a misconfiguration rather than a choice. And pruning -// "everything" with state-sync snapshots enabled fails, because a node cannot -// serve a snapshot of state it has already pruned. Both are the rare case in this -// surface where a bad combination is refused rather than absorbed. +// An empty minimum-gas-prices fails, because a validator accepting zero-fee transactions +// is a misconfiguration rather than a choice, and pruning "everything" with state-sync +// snapshots enabled fails, because a node cannot serve a snapshot of state it has already +// pruned. Both are the rare case here where a bad combination is refused rather than +// absorbed. func FuzzConfigValidateBasic(f *testing.F) { f.Add("0.01usei", "default", uint64(0)) f.Add("", "default", uint64(0)) @@ -613,17 +586,16 @@ func FuzzConfigValidateBasic(f *testing.F) { // declared default when the section is absent from app.toml. // // One table for all of them, because the divergence is one property and a reader comparing sections -// wants them side by side. It puts api.max-open-connections and api.rpc-max-body-bytes beside the -// guarded grpc-web.max-open-connections, which is the contrast worth seeing. +// wants them side by side: the unguarded api.max-open-connections sits beside the guarded +// grpc-web.max-open-connections, which is the contrast worth seeing. // -// The diverges column asserts both directions, so a key is anchored whether or not it moves today. A -// declared default later shifting onto the getter's zero, or off it, fails here rather than changing -// the divergence set quietly. The rows set false are the keys whose declared default already equals -// that zero. +// The diverges column asserts both directions, so a key is anchored whether or not it moves today; a +// declared default shifting onto the getter's zero, or off it, fails here rather than changing the +// divergence set quietly. Rows set false are the keys whose declared default already equals that +// zero. // -// Compared with reflect.DeepEqual rather than !=, because != on two any values panics rather than -// reporting when either side holds a slice or a map. index-events already holds a []string, and a -// field type changing to one later would otherwise turn this table into a panic. +// Compared with reflect.DeepEqual rather than !=, because != on two any values panics when either +// side holds a slice or a map, as index-events does. func TestGetConfigAbsentSectionDivergences(t *testing.T) { cfg, err := GetConfig(newAppViper(t, nil)) if err != nil { @@ -642,9 +614,8 @@ func TestGetConfigAbsentSectionDivergences(t *testing.T) { {"rosetta.retries", cfg.Rosetta.Retries, def.Rosetta.Retries, true}, {"grpc-web.enable", cfg.GRPCWeb.Enable, def.GRPCWeb.Enable, true}, {"grpc-web.address", cfg.GRPCWeb.Address, def.GRPCWeb.Address, true}, - // The [grpc] keys read as plain casts. keepalive-permit-without-stream is the third, further - // down with the other rows whose default is already the zero. Its remaining eight are guarded - // or clamped and held by TestGetConfigGRPCAbsentReads. + // The [grpc] keys read as plain casts. The rest of the section is guarded or clamped, and + // held by TestGetConfigGRPCAbsentReads. {"grpc.enable", cfg.GRPC.Enable, def.GRPC.Enable, true}, {"grpc.address", cfg.GRPC.Address, def.GRPC.Address, true}, {"telemetry.enabled", cfg.Telemetry.Enabled, def.Telemetry.Enabled, true}, @@ -653,8 +624,8 @@ func TestGetConfigAbsentSectionDivergences(t *testing.T) { cfg.Telemetry.PrometheusRetentionTime, def.Telemetry.PrometheusRetentionTime, true, }, - // [api]. Five diverge. The three set false have a declared default that is already the - // getter's zero, so nothing about the resolved value distinguishes a guard from its absence. + // [api]. The rows set false have a declared default that is already the getter's zero, so + // nothing about the resolved value distinguishes a guard from its absence. {"api.swagger", cfg.API.Swagger, def.API.Swagger, true}, {"api.address", cfg.API.Address, def.API.Address, true}, {"api.max-open-connections", cfg.API.MaxOpenConnections, def.API.MaxOpenConnections, true}, @@ -680,10 +651,9 @@ func TestGetConfigAbsentSectionDivergences(t *testing.T) { {"min-retain-blocks", cfg.MinRetainBlocks, def.MinRetainBlocks, false}, {"compaction-interval", cfg.CompactionInterval, def.CompactionInterval, false}, - // The remaining plain casts. Every row from here down has a declared default equal to its - // getter's zero, so none diverges today, and each is here for the reason api.enable is. This - // table is the only thing tying these sections' absent-key resolution to their declared - // defaults. + // The remaining plain casts, each here for the reason api.enable is: a declared default equal + // to its getter's zero, anchored so that a default moving off it fails here. This table is + // the only thing tying these sections' absent-key resolution to their declared defaults. {"rosetta.enable", cfg.Rosetta.Enable, def.Rosetta.Enable, false}, {"rosetta.offline", cfg.Rosetta.Offline, def.Rosetta.Offline, false}, {"grpc-web.enable-unsafe-cors", cfg.GRPCWeb.EnableUnsafeCORS, def.GRPCWeb.EnableUnsafeCORS, false}, @@ -706,8 +676,8 @@ func TestGetConfigAbsentSectionDivergences(t *testing.T) { cfg.Telemetry.EnableServiceLabel, def.Telemetry.EnableServiceLabel, false, }, - // index-events resolves to a []string, which is why the comparison is reflect.DeepEqual rather - // than !=. Both sides are nil today, so it is a false row. + // index-events resolves to a []string, which is why the comparison is reflect.DeepEqual. Both + // sides are nil today, so it is a false row. {"index-events", cfg.IndexEvents, def.IndexEvents, false}, // The guarded read. Its absent value is the declared default, which is the property the // guard exists to provide. @@ -732,10 +702,10 @@ func TestGetConfigAbsentSectionDivergences(t *testing.T) { // reads apart from the two that only look guarded. // // The split matters because the two groups fail for different reasons and a reader needs the right -// one. Six keys are read behind v.IsSet, so a guard is what returns the declared default and losing -// it is the failure. max-connection-age and max-connection-age-grace have no guard at all: they are -// read unconditionally and clamped, so an absent key resolves 0 and that happens to equal their -// declared default. Moving either default off 0 turns them into a visible clobber, which is a +// one. The guarded keys are read behind v.IsSet, so a guard is what returns the declared default and +// losing it is the failure. max-connection-age and max-connection-age-grace have no guard at all: +// they are read unconditionally and clamped, so an absent key resolves 0 and that happens to equal +// their declared default. Moving either default off 0 turns them into a visible clobber, which is a // different event from a guard disappearing. func TestGetConfigGRPCAbsentReads(t *testing.T) { cfg, err := GetConfig(newAppViper(t, nil)) @@ -761,8 +731,8 @@ func TestGetConfigGRPCAbsentReads(t *testing.T) { } { if c.absent != c.declared { t.Errorf("an absent %s resolved to %v rather than the declared %v, so its v.IsSet guard "+ - "is gone. That is the failure the guard exists to prevent, and config.go:519-521 says "+ - "why: a node upgrading with an older app.toml stays bounded", c.key, c.absent, c.declared) + "is gone. That is the failure the guard exists to prevent: a node upgrading with an "+ + "older app.toml stays bounded", c.key, c.absent, c.declared) } } From dd4efaede499d36f9f90ca2dc6b74925ff49e68d Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Tue, 1 Sep 2026 14:38:10 +0200 Subject: [PATCH 12/12] Used correct capacity for registeredMethods --- sei-cosmos/server/grpc/server.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sei-cosmos/server/grpc/server.go b/sei-cosmos/server/grpc/server.go index 72754e0f4f..668177941f 100644 --- a/sei-cosmos/server/grpc/server.go +++ b/sei-cosmos/server/grpc/server.go @@ -60,7 +60,12 @@ func rateLimitServerOptions(cfg config.GRPCConfig) ([]grpc.ServerOption, *rateli // srv. func registeredMethods(srv *grpc.Server) []string { info := srv.GetServiceInfo() - methods := make([]string, 0, len(info)) + total := 0 + for _, svcInfo := range info { + total += len(svcInfo.Methods) + } + + methods := make([]string, 0, total) for service, svcInfo := range info { for _, m := range svcInfo.Methods { methods = append(methods, service+"/"+m.Name)