Skip to content

fix(metrics): wire pending nonce events in mempool OTel counter (PLT-1017) #3991

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
amir-deris merged 4 commits into from
Aug 24, 2026
Merged

fix(metrics): wire pending nonce events in mempool OTel counter (PLT-1017) #3991

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
464e9b8
fix(metrics): wire pending nonce events in mempool OTel counter (PLT-…
amir-deris Aug 24, 2026
668bd1d
Merge branch 'main' into amir/plt-1017-fix-pending-nonce-events-metric
amir-deris Aug 24, 2026
4360826
fix(lint): rewrite pending nonce guard to satisfy staticcheck QF1001
amir-deris Aug 24, 2026
56d97bb
fix(metrics): count pending nonce expired only on TTL removal
amir-deris Aug 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions sei-tendermint/internal/mempool/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package mempool

import (
"context"
"math"
"strconv"

Expand All @@ -27,6 +28,7 @@ var (
otelMetrics = struct {
compactTotal metric.Int64Counter
compactDurationSeconds metric.Float64Histogram
pendingNonce metric.Int64Counter
}{
compactTotal: utils.OrPanic1(mempoolMeter.Int64Counter(
"tendermint_mempool_compact_total",
Expand All @@ -38,13 +40,39 @@ var (
metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(prometheus.ExponentialBucketsRange(0.001, 30, 14)...),
)),
pendingNonce: utils.OrPanic1(mempoolMeter.Int64Counter(
"tendermint_mempool_pending_nonce",
metric.WithDescription("EVM pending nonce events by type (added, expired, rejected, accepted)"),
metric.WithUnit("{count}"),
)),
}

pendingNonceEventAdded = metric.WithAttributes(attribute.String("event", "added"))
pendingNonceEventExpired = metric.WithAttributes(attribute.String("event", "expired"))
pendingNonceEventRejected = metric.WithAttributes(attribute.String("event", "rejected"))
pendingNonceEventAccepted = metric.WithAttributes(attribute.String("event", "accepted"))

triggerInsertOverflowAttr = metric.WithAttributes(attribute.String("trigger", "insert_overflow"))
triggerUpdateAttr = metric.WithAttributes(attribute.String("trigger", "update"))
triggerReapAttr = metric.WithAttributes(attribute.String("trigger", "reap"))
)

func recordPendingNonceAdded() {
otelMetrics.pendingNonce.Add(context.Background(), 1, pendingNonceEventAdded)
}

func recordPendingNonceExpired() {
otelMetrics.pendingNonce.Add(context.Background(), 1, pendingNonceEventExpired)
}

func recordPendingNonceRejected() {
otelMetrics.pendingNonce.Add(context.Background(), 1, pendingNonceEventRejected)
}

func recordPendingNonceAccepted() {
otelMetrics.pendingNonce.Add(context.Background(), 1, pendingNonceEventAccepted)
}

//go:generate go run ../../scripts/metricsgen -struct=Metrics

// Metrics contains metrics exposed by this package.
Expand Down
21 changes: 17 additions & 4 deletions sei-tendermint/internal/mempool/tx.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,7 @@ func (inner *txStoreInner) shouldReject(txHash types.TxHash) bool {
return false
}

func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error {
func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx, recordAdded bool) error {
if _, ok := inner.byHash[wtx.Hash()]; ok {
return errDuplicateTx
}
Expand All @@ -371,8 +371,11 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error {
// Reject transactions with old nonces.
if evm.nonce < account.firstNonce {
inner.cache.Push(wtx.Hash(), utils.None[cacheEvm]())
recordPendingNonceRejected()

@seidroid seidroid Bot Aug 24, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[suggestion] Unlike added and accepted, rejected is not gated by recordAdded, so it also fires from compact() — which Update runs every block with clearAccounts: true, re-reading firstNonce from app state. A queued transaction whose nonce was consumed on chain by some other means is then re-inserted, hits errOldNonce, and is counted as a rejection even though it is really an eviction of a stale entry. Gating this on recordAdded (or adding a distinct label for the compaction path) would keep the series meaning "incoming transaction rejected".

Conversely — and as Codex noted — this is the only rejection path counted: errDuplicateTx (duplicate EVM hash) and errSameNonce (replacement lost on balance or priority) return without incrementing anything, so same-nonce replacement rejections, arguably the most interesting pending-nonce rejection, never appear on the dashboard. Worth either covering them with the same label or documenting that rejected means "nonce below account nonce" only.

return errOldNonce
}
insertedAtNextNonce := evm.nonce == account.nextNonce
newTx := wtx
inner.cache.Push(wtx.Hash(), utils.Some(cacheEvm{
priority: wtx.priority,
address: evm.address,
Expand Down Expand Up @@ -429,6 +432,9 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error {
if !wtx.readyEl.IsPresent() {
s.priorityReservoir.Add(wtx.priority)
wtx.readyEl = utils.Some(s.readyTxs.PushBack(wtx.Tx()))
if !recordAdded || wtx != newTx || !insertedAtNextNonce {
recordPendingNonceAccepted()
}
}
}
} else {
Expand All @@ -442,6 +448,9 @@ func (s *txStore) insert(inner *txStoreInner, wtx *WrappedTx) error {
}
inner.byHash[wtx.Hash()] = wtx
inner.state.Store(state)
if recordAdded && wtx.evm.IsPresent() {

@seidroid seidroid Bot Aug 24, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[suggestion] added is recorded here, before Insert checks the hard limit and runs the overflow compact() (lines 516-523). When that compaction evicts the transaction just inserted, Insert returns errMempoolFull but the added increment has already landed, so the series counts transactions that never entered the mempool. Recording added in Insert after the hard-limit check (rather than inside insert) would avoid that; it would also drop the need to thread recordAdded through for this event.

More generally, the four labels are not a closed lifecycle: added fires for every EVM insert including ones that go straight to ready, while accepted deliberately excludes those, so added - accepted - expired - rejected is not pending-queue depth. Since the linked dashboard PR graphs this series, please state the per-label meaning in the metric description in metrics.go so the panels are not derived from an assumed conservation law.

recordPendingNonceAdded()
}
return nil
}

Expand Down Expand Up @@ -501,7 +510,7 @@ func (inner *txStoreInner) inInclusionOrder() []*WrappedTx {
// txStore takes ownership of wtx.
func (s *txStore) Insert(wtx *WrappedTx) error {
for inner := range s.inner.Lock() {
if err := s.insert(inner, wtx); err != nil {
if err := s.insert(inner, wtx, true); err != nil {
return err
}
if total := inner.state.Load().total; !total.LessEqual(&inner.hardLimit) {
Expand Down Expand Up @@ -538,7 +547,7 @@ func (s *txStore) compact(inner *txStoreInner, clearAccounts bool) {
total.Inc(wtx.Size())
limitOk := total.LessEqual(&inner.softLimit)
// NOTE: insertion is lazily evaluated here.
if !limitOk || s.insert(inner, wtx) != nil {
if !limitOk || s.insert(inner, wtx, false) != nil {
Global.RemovedTxsAt().Add(1)
Global.EvictedTxsAt().Add(1)
if el, ok := wtx.readyEl.Get(); ok {
Expand Down Expand Up @@ -594,8 +603,12 @@ func (s *txStore) Update(spec updateSpec) {
}
invalid := spec.InvalidTxs[wtx.Hash()] || wtx.check(spec.Constraints) != nil
_, executed := spec.TxResults[wtx.Hash()]
remove := invalid || executed || (expired && (s.config.RemoveExpiredTxsFromQueue || !inner.isReady(wtx)))
removedByExpiry := expired && (s.config.RemoveExpiredTxsFromQueue || !inner.isReady(wtx))
remove := invalid || executed || removedByExpiry
if remove {
if removedByExpiry && wtx.evm.IsPresent() {
recordPendingNonceExpired()
}
Comment thread
amir-deris marked this conversation as resolved.
// KeepInvalidTxsInCache decides whether we invalidate txs removed from mempool for any reason.
// Executed txs invalidation was handled earlier.
if s.config.KeepInvalidTxsInCache && !executed {
Expand Down
7 changes: 0 additions & 7 deletions x/evm/ante/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ var (
}

evmAnteMetrics = struct {
pendingNonce metric.Int64Counter
nonceMismatch metric.Int64Counter

// Gas price histogram
Expand All @@ -26,12 +25,6 @@ var (
// Association errors
associationError metric.Int64Counter
}{
pendingNonce: must(meter.Int64Counter(
"evm_pending_nonce",

@bdchatham bdchatham Aug 24, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any concern with this going away? Any alerts?

@amir-deris amir-deris Aug 24, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe there shouldn't be any concern, as this metric was already not being emitted.

The metric was already dead
On main, evm_pending_nonce in x/evm/ante/metrics.go is defined but never incremented. Nothing in x/evm calls evmAnteMetrics.pendingNonce:

sig.go → nonceMismatch
fee.go → effectiveGasPrice
preprocess.go → associationError
Instrumentation moved out of ante when #3410 removed mempool callbacks. At the same time, app/ante/evm_checktx.go stopped emitting app_pending_nonce (that metric definition was removed too). Pending-nonce lifecycle now lives entirely in the Tendermint mempool (tx.go), which PR 3991 wires up.

So removing the dead definition is cleanup, not a behavioral change — production has not been emitting either old series since #3410.

metric.WithDescription("EVM pending nonce events by type (added, expired, rejected, accepted)"),
metric.WithUnit("{count}"),
)),

nonceMismatch: must(meter.Int64Counter(
"evm_nonce_mismatch",
metric.WithDescription("EVM nonce mismatches by cause (too_high, too_low)"),
Expand Down
Loading