core/node/crypto: refetch tx nonce in case chain rpc node reports non…

…ce issues

core/node/crypto/chain_txpool.go

@@ -5,6 +5,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"math/big"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -79,9 +80,9 @@ type (
 		walletBalanceLastTimeChecked time.Time
 		walletBalance                prometheus.Gauge
 
-		// mu guards lastPendingTx that is used to determine the tx nonce
-		mu            sync.Mutex
-		lastPendingTx *txPoolPendingTransaction
+		// mu guards lastNonce that is used to determine the tx nonce
+		mu        sync.Mutex
+		lastNonce *uint64
 	}
 
 	// txPoolPendingTransaction represents a transaction that is submitted to the chain but no receipt was retrieved.
@@ -478,8 +479,8 @@ func (tx *txPoolPendingTransaction) TransactionHash() common.Hash {
 
 // caller is expected to hold a lock on r.mu
 func (r *transactionPool) nextNonce(ctx context.Context) (uint64, error) {
-	if r.lastPendingTx != nil {
-		return r.lastPendingTx.tx.Nonce() + 1, nil
+	if r.lastNonce != nil {
+		return *r.lastNonce + 1, nil
 	}
 	return r.client.PendingNonceAt(ctx, r.wallet.Address)
 }
@@ -507,17 +508,24 @@ func (r *transactionPool) Submit(
 	name string,
 	createTx CreateTransaction,
 ) (TransactionPoolPendingTransaction, error) {
-	var span trace.Span
+	// lock to prevent tx.Nonce collisions
+	r.mu.Lock()
+	defer r.mu.Unlock()
 
+	return r.submitNoLock(ctx, name, createTx, true)
+}
+func (r *transactionPool) submitNoLock(
+	ctx context.Context,
+	name string,
+	createTx CreateTransaction,
+	canRetry bool,
+) (TransactionPoolPendingTransaction, error) {
+	var span trace.Span
 	if r.tracer != nil {
 		ctx, span = r.tracer.Start(ctx, "txpool_submit")
 		defer span.End()
 	}
 
-	// lock to prevent tx.Nonce collisions
-	r.mu.Lock()
-	defer r.mu.Unlock()
-
 	nonce, err := r.nextNonce(ctx)
 	if err != nil {
 		return nil, err
@@ -548,6 +556,13 @@ func (r *transactionPool) Submit(
 	}
 
 	if err := r.client.SendTransaction(ctx, tx); err != nil {
+		// force fetching the latest nonce from the rpc node again when it was reported to be too low. This can be
+		// caused by the chain rpc node lagging behind when the tx pool fetched the nonce. When the chain rpc node
+		// caught up the fetched nonce can be too low. Fetch the nonce again recovers from this scenario.
+		if canRetry && strings.Contains(strings.ToLower(err.Error()), "nonce too low") {
+			r.lastNonce = nil
+			return r.submitNoLock(ctx, name, createTx, false)
+		}
 		return nil, err
 	}
 
@@ -564,7 +579,11 @@ func (r *transactionPool) Submit(
 		listener:    make(chan *types.Receipt, 1),
 	}
 
-	r.lastPendingTx = pendingTx
+	if r.lastNonce == nil {
+		r.lastNonce = new(uint64)
+	}
+	*r.lastNonce = pendingTx.tx.Nonce()
+
 	r.pendingTransactionPool.addPendingTx <- pendingTx
 
 	// metrics