diff --git a/sei-tendermint/internal/mempool/metrics.go b/sei-tendermint/internal/mempool/metrics.go index d313e746ec..d55e09131b 100644 --- a/sei-tendermint/internal/mempool/metrics.go +++ b/sei-tendermint/internal/mempool/metrics.go @@ -1,6 +1,7 @@ package mempool import ( + "context" "math" "strconv" @@ -27,6 +28,7 @@ var ( otelMetrics = struct { compactTotal metric.Int64Counter compactDurationSeconds metric.Float64Histogram + pendingNonce metric.Int64Counter }{ compactTotal: utils.OrPanic1(mempoolMeter.Int64Counter( "tendermint_mempool_compact_total", @@ -38,13 +40,39 @@ var ( metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(prometheus.ExponentialBucketsRange(0.001, 30, 14)...), )), + pendingNonce: utils.OrPanic1(mempoolMeter.Int64Counter( + "tendermint_mempool_pending_nonce", + metric.WithDescription("EVM pending nonce events by type (added, expired, rejected, accepted)"), + metric.WithUnit("{count}"), + )), } + pendingNonceEventAdded = metric.WithAttributes(attribute.String("event", "added")) + pendingNonceEventExpired = metric.WithAttributes(attribute.String("event", "expired")) + pendingNonceEventRejected = metric.WithAttributes(attribute.String("event", "rejected")) + pendingNonceEventAccepted = metric.WithAttributes(attribute.String("event", "accepted")) + triggerInsertOverflowAttr = metric.WithAttributes(attribute.String("trigger", "insert_overflow")) triggerUpdateAttr = metric.WithAttributes(attribute.String("trigger", "update")) triggerReapAttr = metric.WithAttributes(attribute.String("trigger", "reap")) ) +func recordPendingNonceAdded() { + otelMetrics.pendingNonce.Add(context.Background(), 1, pendingNonceEventAdded) +} + +func recordPendingNonceExpired() { + otelMetrics.pendingNonce.Add(context.Background(), 1, pendingNonceEventExpired) +} + +func recordPendingNonceRejected() { + otelMetrics.pendingNonce.Add(context.Background(), 1, pendingNonceEventRejected) +} + +func recordPendingNonceAccepted() { + otelMetrics.pendingNonce.Add(context.Background(), 1, pendingNonceEventAccepted) +} + //go:generate go run ../../scripts/metricsgen -struct=Metrics // Metrics contains metrics exposed by this package. diff --git a/sei-tendermint/internal/mempool/tx.go b/sei-tendermint/internal/mempool/tx.go index 5a728bde6e..7b53d436ba 100644 --- a/sei-tendermint/internal/mempool/tx.go +++ b/sei-tendermint/internal/mempool/tx.go @@ -353,7 +353,7 @@ func (inner *txStoreInner) shouldReject(txHash types.TxHash) bool { return false } -func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { +func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx, recordAdded bool) error { if _, ok := inner.byHash[wtx.Hash()]; ok { return errDuplicateTx } @@ -371,8 +371,11 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { // Reject transactions with old nonces. if evm.nonce < account.firstNonce { inner.cache.Push(wtx.Hash(), utils.None[cacheEvm]()) + recordPendingNonceRejected() return errOldNonce } + insertedAtNextNonce := evm.nonce == account.nextNonce + newTx := wtx inner.cache.Push(wtx.Hash(), utils.Some(cacheEvm{ priority: wtx.priority, address: evm.address, @@ -429,6 +432,9 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { if !wtx.readyEl.IsPresent() { s.priorityReservoir.Add(wtx.priority) wtx.readyEl = utils.Some(s.readyTxs.PushBack(wtx.Tx())) + if !recordAdded || wtx != newTx || !insertedAtNextNonce { + recordPendingNonceAccepted() + } } } } else { @@ -442,6 +448,9 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error { } inner.byHash[wtx.Hash()] = wtx inner.state.Store(state) + if recordAdded && wtx.evm.IsPresent() { + recordPendingNonceAdded() + } return nil } @@ -501,7 +510,7 @@ func (inner *txStoreInner) inInclusionOrder() []*WrappedTx { // txStore takes ownership of wtx. func (s *txStore) Insert(wtx *WrappedTx) error { for inner := range s.inner.Lock() { - if err := s.insert(inner, wtx); err != nil { + if err := s.insert(inner, wtx, true); err != nil { return err } if total := inner.state.Load().total; !total.LessEqual(&inner.hardLimit) { @@ -538,7 +547,7 @@ func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) { total.Inc(wtx.Size()) limitOk := total.LessEqual(&inner.softLimit) // NOTE: insertion is lazily evaluated here. - if !limitOk || s.insert(inner, wtx) != nil { + if !limitOk || s.insert(inner, wtx, false) != nil { Global.RemovedTxsAt().Add(1) Global.EvictedTxsAt().Add(1) if el, ok := wtx.readyEl.Get(); ok { @@ -594,8 +603,12 @@ func (s *txStore) Update(spec updateSpec) { } invalid := spec.InvalidTxs[wtx.Hash()] || wtx.check(spec.Constraints) != nil _, executed := spec.TxResults[wtx.Hash()] - remove := invalid || executed || (expired && (s.config.RemoveExpiredTxsFromQueue || !inner.isReady(wtx))) + removedByExpiry := expired && (s.config.RemoveExpiredTxsFromQueue || !inner.isReady(wtx)) + remove := invalid || executed || removedByExpiry if remove { + if removedByExpiry && wtx.evm.IsPresent() { + recordPendingNonceExpired() + } // KeepInvalidTxsInCache decides whether we invalidate txs removed from mempool for any reason. // Executed txs invalidation was handled earlier. if s.config.KeepInvalidTxsInCache && !executed { diff --git a/x/evm/ante/metrics.go b/x/evm/ante/metrics.go index 8c08336236..3b62e025f4 100644 --- a/x/evm/ante/metrics.go +++ b/x/evm/ante/metrics.go @@ -17,7 +17,6 @@ var ( } evmAnteMetrics = struct { - pendingNonce metric.Int64Counter nonceMismatch metric.Int64Counter // Gas price histogram @@ -26,12 +25,6 @@ var ( // Association errors associationError metric.Int64Counter }{ - pendingNonce: must(meter.Int64Counter( - "evm_pending_nonce", - metric.WithDescription("EVM pending nonce events by type (added, expired, rejected, accepted)"), - metric.WithUnit("{count}"), - )), - nonceMismatch: must(meter.Int64Counter( "evm_nonce_mismatch", metric.WithDescription("EVM nonce mismatches by cause (too_high, too_low)"),