Skip to content

Commit ac044ca

Browse files
0xmountaintopomerfirmakThegaram
authored
feat(miner): account fetch limit (#939)
* feat(worker): try to limit the number of txns miner has to deal with (#745) to reduce the effect of having a huge backlog on performance * fix(worker): set default account fetch limit (#756) * fix * update miner/worker.go * fix --------- Co-authored-by: Ömer Faruk Irmak <omerfirmak@gmail.com> Co-authored-by: Péter Garamvölgyi <peter@scroll.io>
1 parent c86a215 commit ac044ca

12 files changed

+102
-9
lines changed

cmd/geth/main.go

+1
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ var (
124124
utils.MinerRecommitIntervalFlag,
125125
utils.MinerNewPayloadTimeout,
126126
utils.MinerStoreSkippedTxTracesFlag,
127+
utils.MinerMaxAccountsNumFlag,
127128
utils.NATFlag,
128129
utils.NoDiscoverFlag,
129130
utils.DiscoveryV4Flag,

cmd/utils/flags.go

+12-2
Original file line numberDiff line numberDiff line change
@@ -524,8 +524,15 @@ var (
524524
Category: flags.MinerCategory,
525525
}
526526
MinerStoreSkippedTxTracesFlag = &cli.BoolFlag{
527-
Name: "miner.storeskippedtxtraces",
528-
Usage: "Store the wrapped traces when storing a skipped tx",
527+
Name: "miner.storeskippedtxtraces",
528+
Usage: "Store the wrapped traces when storing a skipped tx",
529+
Category: flags.MinerCategory,
530+
}
531+
MinerMaxAccountsNumFlag = &cli.IntFlag{
532+
Name: "miner.maxaccountsnum",
533+
Usage: "Maximum number of accounts that miner will fetch the pending transactions of when building a new block",
534+
Value: math.MaxInt,
535+
Category: flags.MinerCategory,
529536
}
530537

531538
// Account settings
@@ -1697,6 +1704,9 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) {
16971704
if ctx.IsSet(MinerStoreSkippedTxTracesFlag.Name) {
16981705
cfg.StoreSkippedTxTraces = ctx.Bool(MinerStoreSkippedTxTracesFlag.Name)
16991706
}
1707+
if ctx.IsSet(MinerMaxAccountsNumFlag.Name) {
1708+
cfg.MaxAccountsNum = ctx.Int(MinerMaxAccountsNumFlag.Name)
1709+
}
17001710
}
17011711

17021712
func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) {

core/txpool/blobpool/blobpool.go

+15-2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ import (
2929
"sync"
3030
"time"
3131

32+
"github.com/holiman/billy"
33+
"github.com/holiman/uint256"
3234
"github.com/scroll-tech/go-ethereum/common"
3335
"github.com/scroll-tech/go-ethereum/consensus/misc/eip1559"
3436
"github.com/scroll-tech/go-ethereum/consensus/misc/eip4844"
@@ -42,8 +44,6 @@ import (
4244
"github.com/scroll-tech/go-ethereum/params"
4345
"github.com/scroll-tech/go-ethereum/rlp"
4446
"github.com/scroll-tech/go-ethereum/rollup/fees"
45-
"github.com/holiman/billy"
46-
"github.com/holiman/uint256"
4747
)
4848

4949
const (
@@ -1381,6 +1381,16 @@ func (p *BlobPool) drop() {
13811381
// Pending retrieves all currently processable transactions, grouped by origin
13821382
// account and sorted by nonce.
13831383
func (p *BlobPool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTransaction {
1384+
return p.pendingWithMax(enforceTips, math.MaxInt)
1385+
}
1386+
1387+
// PendingWithMax works similar to Pending but allows setting an upper limit on how many
1388+
// accounts to return
1389+
func (p *BlobPool) PendingWithMax(enforceTips bool, maxAccountsNum int) map[common.Address][]*txpool.LazyTransaction {
1390+
return p.pendingWithMax(enforceTips, maxAccountsNum)
1391+
}
1392+
1393+
func (p *BlobPool) pendingWithMax(enforceTips bool, maxAccountsNum int) map[common.Address][]*txpool.LazyTransaction {
13841394
// Track the amount of time waiting to retrieve the list of pending blob txs
13851395
// from the pool and the amount of time actually spent on assembling the data.
13861396
// The latter will be pretty much moot, but we've kept it to have symmetric
@@ -1411,6 +1421,9 @@ func (p *BlobPool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTr
14111421
if len(lazies) > 0 {
14121422
pending[addr] = lazies
14131423
}
1424+
if len(pending) >= maxAccountsNum {
1425+
break
1426+
}
14141427
}
14151428
return pending
14161429
}

core/txpool/legacypool/legacypool.go

+13
Original file line numberDiff line numberDiff line change
@@ -523,6 +523,16 @@ func (pool *LegacyPool) ContentFrom(addr common.Address) ([]*types.Transaction,
523523
// transactions and only return those whose **effective** tip is large enough in
524524
// the next pending execution environment.
525525
func (pool *LegacyPool) Pending(enforceTips bool) map[common.Address][]*txpool.LazyTransaction {
526+
return pool.pendingWithMax(enforceTips, math.MaxInt)
527+
}
528+
529+
// PendingWithMax works similar to Pending but allows setting an upper limit on how many
530+
// accounts to return
531+
func (pool *LegacyPool) PendingWithMax(enforceTips bool, maxAccountsNum int) map[common.Address][]*txpool.LazyTransaction {
532+
return pool.pendingWithMax(enforceTips, maxAccountsNum)
533+
}
534+
535+
func (pool *LegacyPool) pendingWithMax(enforceTips bool, maxAccountsNum int) map[common.Address][]*txpool.LazyTransaction {
526536
pool.mu.Lock()
527537
defer pool.mu.Unlock()
528538

@@ -554,6 +564,9 @@ func (pool *LegacyPool) Pending(enforceTips bool) map[common.Address][]*txpool.L
554564
}
555565
}
556566
pending[addr] = lazies
567+
if len(pending) >= maxAccountsNum {
568+
break
569+
}
557570
}
558571
}
559572
return pending

core/txpool/legacypool/legacypool_test.go

+26
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ import (
2929
"testing"
3030
"time"
3131

32+
"github.com/stretchr/testify/assert"
33+
3234
"github.com/scroll-tech/go-ethereum/common"
3335
"github.com/scroll-tech/go-ethereum/core"
3436
"github.com/scroll-tech/go-ethereum/core/rawdb"
@@ -2628,3 +2630,27 @@ func BenchmarkMultiAccountBatchInsert(b *testing.B) {
26282630
pool.addRemotesSync([]*types.Transaction{tx})
26292631
}
26302632
}
2633+
2634+
func TestPoolPending(t *testing.T) {
2635+
// Generate a batch of transactions to enqueue into the pool
2636+
pool, _ := setupPool()
2637+
defer pool.Close()
2638+
numTxns := 100
2639+
batches := make(types.Transactions, numTxns)
2640+
for i := 0; i < numTxns; i++ {
2641+
key, _ := crypto.GenerateKey()
2642+
account := crypto.PubkeyToAddress(key.PublicKey)
2643+
pool.currentState.AddBalance(account, big.NewInt(1000000))
2644+
tx := transaction(uint64(0), 100000, key)
2645+
batches[i] = tx
2646+
}
2647+
// Benchmark importing the transactions into the queue
2648+
for _, tx := range batches {
2649+
pool.addRemotesSync([]*types.Transaction{tx})
2650+
}
2651+
2652+
assert.Len(t, pool.Pending(false), numTxns)
2653+
2654+
maxAccounts := 10
2655+
assert.Len(t, pool.PendingWithMax(false, maxAccounts), maxAccounts)
2656+
}

core/txpool/subpool.go

+2
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,8 @@ type SubPool interface {
110110
// account and sorted by nonce.
111111
Pending(enforceTips bool) map[common.Address][]*LazyTransaction
112112

113+
PendingWithMax(enforceTips bool, maxAccountsNum int) map[common.Address][]*LazyTransaction
114+
113115
// SubscribeTransactions subscribes to new transaction events. The subscriber
114116
// can decide whether to receive notifications only for newly seen transactions
115117
// or also for reorged out ones.

core/txpool/txpool.go

+10
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,16 @@ func (p *TxPool) Pending(enforceTips bool) map[common.Address][]*LazyTransaction
318318
return txs
319319
}
320320

321+
func (p *TxPool) PendingWithMax(enforceTips bool, maxAccountsNum int) map[common.Address][]*LazyTransaction {
322+
txs := make(map[common.Address][]*LazyTransaction)
323+
for _, subpool := range p.subpools {
324+
for addr, set := range subpool.PendingWithMax(enforceTips, maxAccountsNum) {
325+
txs[addr] = set
326+
}
327+
}
328+
return txs
329+
}
330+
321331
// SubscribeTransactions registers a subscription for new transaction events,
322332
// supporting feeding only newly seen or also resurrected transactions.
323333
func (p *TxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool) event.Subscription {

miner/miner.go

+1
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ type Config struct {
5959
NewPayloadTimeout time.Duration // The maximum time allowance for creating a new payload
6060

6161
StoreSkippedTxTraces bool // Whether store the wrapped traces when storing a skipped tx
62+
MaxAccountsNum int // Maximum number of accounts that miner will fetch the pending transactions of when building a new block
6263
}
6364

6465
// DefaultConfig contains default settings for miner.

miner/miner_test.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package miner
1919

2020
import (
2121
"errors"
22+
"math"
2223
"math/big"
2324
"testing"
2425
"time"
@@ -301,7 +302,8 @@ func minerTestGenesisBlock(period uint64, gasLimit uint64, faucet common.Address
301302
func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) {
302303
// Create Ethash config
303304
config := Config{
304-
Etherbase: common.HexToAddress("123456789"),
305+
Etherbase: common.HexToAddress("123456789"),
306+
MaxAccountsNum: math.MaxInt,
305307
}
306308
// Create chainConfig
307309
chainDB := rawdb.NewMemoryDatabase()

miner/scroll_worker.go

+8-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package miner
1919
import (
2020
"bytes"
2121
"errors"
22+
"math"
2223
"math/big"
2324
"sync"
2425
"sync/atomic"
@@ -162,6 +163,12 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus
162163
// Subscribe events for blockchain
163164
worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh)
164165

166+
// Sanitize account fetch limit.
167+
if worker.config.MaxAccountsNum == 0 {
168+
log.Warn("Sanitizing miner account fetch limit", "provided", worker.config.MaxAccountsNum, "updated", math.MaxInt)
169+
worker.config.MaxAccountsNum = math.MaxInt
170+
}
171+
165172
worker.wg.Add(1)
166173
go worker.mainLoop()
167174

@@ -382,7 +389,7 @@ func (w *worker) startNewPipeline(timestamp int64) {
382389

383390
tidyPendingStart := time.Now()
384391
// Fill the block with all available pending transactions.
385-
pending := w.eth.TxPool().Pending(false)
392+
pending := w.eth.TxPool().PendingWithMax(false, w.config.MaxAccountsNum)
386393
// Split the pending transactions into locals and remotes
387394
localTxs, remoteTxs := make(map[common.Address][]*txpool.LazyTransaction), pending
388395
for _, account := range w.eth.TxPool().Locals() {

miner/worker.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,12 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus
353353
}
354354
worker.newpayloadTimeout = newpayloadTimeout
355355
356+
// Sanitize account fetch limit.
357+
if worker.config.MaxAccountsNum == 0 {
358+
log.Warn("Sanitizing miner account fetch limit", "provided", worker.config.MaxAccountsNum, "updated", math.MaxInt)
359+
worker.config.MaxAccountsNum = math.MaxInt
360+
}
361+
356362
worker.wg.Add(4)
357363
go worker.mainLoop()
358364
go worker.newWorkLoop(recommit)
@@ -1420,7 +1426,7 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err
14201426
}
14211427
14221428
tidyPendingStart := time.Now()
1423-
pending := w.eth.TxPool().Pending(true)
1429+
pending := w.eth.TxPool().PendingWithMax(true, w.config.MaxAccountsNum)
14241430
14251431
// Split the pending transactions into locals and remotes.
14261432
localTxs, remoteTxs := make(map[common.Address][]*txpool.LazyTransaction), pending

miner/worker_test.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package miner
1818

1919
import (
20+
"math"
2021
"math/big"
2122
"testing"
2223
"time"
@@ -67,8 +68,9 @@ var (
6768
newTxs []*types.Transaction
6869

6970
testConfig = &Config{
70-
Recommit: time.Second,
71-
GasCeil: params.GenesisGasLimit,
71+
Recommit: time.Second,
72+
GasCeil: params.GenesisGasLimit,
73+
MaxAccountsNum: math.MaxInt,
7274
}
7375
)
7476

0 commit comments

Comments
 (0)