Skip to content

Commit edda31c

Browse files
cabradorcabrador
committed
Lower memory consumption for testnet and integration tests.
1 parent 18d9c82 commit edda31c

File tree

14 files changed

+71
-23
lines changed

14 files changed

+71
-23
lines changed

‎cmd/sonicd/app/launcher.go

+1
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ func initFlags() {
6666
flags.CacheFlag,
6767
flags.LiveDbCacheFlag,
6868
flags.ArchiveCacheFlag,
69+
flags.StateDbCacheCapacityFlag,
6970
}
7071
networkingFlags = []cli.Flag{
7172
flags.BootnodesFlag,

‎cmd/sonicd/app/run_test.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,12 @@ func tmpdir(t *testing.T) string {
2323
}
2424

2525
func initFakenetDatadir(dataDir string, validatorsNum idx.Validator) {
26-
genesisStore := makefakegenesis.FakeGenesisStore(validatorsNum, futils.ToFtm(1000000000), futils.ToFtm(5000000))
26+
genesisStore := makefakegenesis.FakeGenesisStore(
27+
validatorsNum,
28+
futils.ToFtm(1000000000),
29+
futils.ToFtm(5000000),
30+
1024, // 1024 bytes is both small and safe value used by tests
31+
)
2732
defer genesisStore.Close()
2833

2934
if err := genesis.ImportGenesisStore(genesis.ImportParams{

‎cmd/sonictool/app/genesis.go

+9-2
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,9 @@ func jsonGenesisImport(ctx *cli.Context) error {
100100
if err != nil {
101101
return fmt.Errorf("failed to load JSON genesis: %w", err)
102102
}
103-
genesisStore, err := makefakegenesis.ApplyGenesisJson(genesisJson)
103+
104+
stateDbCacheCapacity := ctx.Int(flags.StateDbCacheCapacityFlag.Name)
105+
genesisStore, err := makefakegenesis.ApplyGenesisJson(genesisJson, stateDbCacheCapacity)
104106
if err != nil {
105107
return fmt.Errorf("failed to prepare JSON genesis: %w", err)
106108
}
@@ -139,7 +141,12 @@ func fakeGenesisImport(ctx *cli.Context) error {
139141
return err
140142
}
141143

142-
genesisStore := makefakegenesis.FakeGenesisStore(idx.Validator(validatorsNumber), futils.ToFtm(1000000000), futils.ToFtm(5000000))
144+
genesisStore := makefakegenesis.FakeGenesisStore(
145+
idx.Validator(validatorsNumber),
146+
futils.ToFtm(1000000000),
147+
futils.ToFtm(5000000),
148+
ctx.Int(flags.StateDbCacheCapacityFlag.Name),
149+
)
143150
defer genesisStore.Close()
144151
return genesis.ImportGenesisStore(genesis.ImportParams{
145152
GenesisStore: genesisStore,

‎cmd/sonictool/app/main.go

+3
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ Initialize the database using data from the experimental genesis file.
6262
ArgsUsage: "<validators>",
6363
Action: fakeGenesisImport,
6464
Flags: []cli.Flag{
65+
flags.LiveDbCacheFlag,
66+
flags.ArchiveCacheFlag,
67+
flags.StateDbCacheCapacityFlag,
6568
ModeFlag,
6669
},
6770
Description: `

‎config/config.go

+4
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,10 @@ func MakeAllConfigsFromFile(ctx *cli.Context, configFile string) (*Config, error
341341
cfg.Lachesis.SuppressFramePanic = true
342342
}
343343

344+
if ctx.IsSet(flags.StateDbCacheCapacityFlag.Name) {
345+
cfg.OperaStore.EVM.Cache.StateDbCapacity = ctx.Int(flags.StateDbCacheCapacityFlag.Name)
346+
}
347+
344348
return &cfg, nil
345349
}
346350

‎config/flags/flags.go

+6
Original file line numberDiff line numberDiff line change
@@ -365,4 +365,10 @@ var (
365365
"Setting this value to <=2000 will result in pre-confugired cache capacity of 2KB", CacheFlag.Name),
366366
Value: 0,
367367
}
368+
StateDbCacheCapacityFlag = cli.IntFlag{
369+
Name: "statedb.cache",
370+
Usage: "Size of StateDb instances cache in bytes. Leaving this blank (which is generally recommended), or setting" +
371+
"this to <1 will automatically set the cache capacity a hard-coded constant.",
372+
Value: 0,
373+
}
368374
)

‎gossip/common_test.go

+9-1
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,15 @@ func newTestEnv(firstEpoch idx.Epoch, validatorsNum idx.Validator, tb testing.TB
143143
rules.Blocks.MaxEmptyBlockSkipPeriod = 0
144144
rules.Emitter.Interval = 0
145145

146-
genStore := makefakegenesis.FakeGenesisStoreWithRulesAndStart(validatorsNum, utils.ToFtm(genesisBalance), utils.ToFtm(genesisStake), rules, firstEpoch, 2)
146+
genStore := makefakegenesis.FakeGenesisStoreWithRulesAndStart(
147+
validatorsNum,
148+
utils.ToFtm(genesisBalance),
149+
utils.ToFtm(genesisStake),
150+
rules,
151+
firstEpoch,
152+
2,
153+
1024,
154+
)
147155
genesis := genStore.Genesis()
148156

149157
store, err := NewMemStore(tb)

‎gossip/evmstore/config.go

+7-5
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ type (
1919
EvmBlocksNum int
2020
// Cache size for EvmBlock (size in bytes).
2121
EvmBlocksSize uint
22+
// Cache size for StateDb instances (0 for DB-selected default)
23+
StateDbCapacity int
2224
}
2325
// StoreConfig is a config for store db.
2426
StoreConfig struct {
@@ -38,11 +40,11 @@ type (
3840
func DefaultStoreConfig(scale cachescale.Func) StoreConfig {
3941
return StoreConfig{
4042
Cache: StoreCacheConfig{
41-
ReceiptsSize: scale.U(4 * opt.MiB),
42-
ReceiptsBlocks: scale.I(4000),
43-
TxPositions: scale.I(20000),
44-
EvmBlocksNum: scale.I(5000),
45-
EvmBlocksSize: scale.U(6 * opt.MiB),
43+
ReceiptsSize: scale.U(4 * opt.MiB),
44+
ReceiptsBlocks: scale.I(4000),
45+
TxPositions: scale.I(20000),
46+
EvmBlocksNum: scale.I(5000),
47+
EvmBlocksSize: scale.U(6 * opt.MiB),
4648
},
4749
StateDb: carmen.Parameters{
4850
Variant: "go-file",

‎gossip/evmstore/store.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ func (s *Store) Open() error {
7878
if err != nil {
7979
return fmt.Errorf("failed to create carmen state; %s", err)
8080
}
81-
s.liveStateDb = carmen.CreateStateDBUsing(s.carmenState)
81+
s.liveStateDb = carmen.CreateCustomStateDBUsing(s.carmenState, s.cfg.Cache.StateDbCapacity)
8282
return nil
8383
}
8484

‎gossip/handler_fuzz.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,12 @@ func makeFuzzedHandler() (h *handler, err error) {
6868
genesisStake = 2 * 4e6
6969
)
7070

71-
genStore := makefakegenesis.FakeGenesisStore(genesisStakers, utils.ToFtm(genesisBalance), utils.ToFtm(genesisStake))
71+
genStore := makefakegenesis.FakeGenesisStore(
72+
genesisStakers,
73+
utils.ToFtm(genesisBalance),
74+
utils.ToFtm(genesisStake),
75+
0, // 0 sets default value
76+
)
7277
genesis := genStore.Genesis()
7378

7479
config := DefaultConfig(cachescale.Identity)

‎integration/makefakegenesis/genesis.go

+13-6
Original file line numberDiff line numberDiff line change
@@ -42,16 +42,23 @@ func FakeKey(n idx.ValidatorID) *ecdsa.PrivateKey {
4242
return evmcore.FakeKey(uint32(n))
4343
}
4444

45-
func FakeGenesisStore(num idx.Validator, balance, stake *big.Int) *genesisstore.Store {
46-
return FakeGenesisStoreWithRules(num, balance, stake, opera.FakeNetRules())
45+
func FakeGenesisStore(num idx.Validator, balance, stake *big.Int, stateDbCacheCapacity int) *genesisstore.Store {
46+
return FakeGenesisStoreWithRules(num, balance, stake, opera.FakeNetRules(), stateDbCacheCapacity)
4747
}
4848

49-
func FakeGenesisStoreWithRules(num idx.Validator, balance, stake *big.Int, rules opera.Rules) *genesisstore.Store {
50-
return FakeGenesisStoreWithRulesAndStart(num, balance, stake, rules, 2, 1)
49+
func FakeGenesisStoreWithRules(num idx.Validator, balance, stake *big.Int, rules opera.Rules, stateDbCacheCapacity int) *genesisstore.Store {
50+
return FakeGenesisStoreWithRulesAndStart(num, balance, stake, rules, 2, 1, stateDbCacheCapacity)
5151
}
5252

53-
func FakeGenesisStoreWithRulesAndStart(num idx.Validator, balance, stake *big.Int, rules opera.Rules, epoch idx.Epoch, block idx.Block) *genesisstore.Store {
54-
builder := makegenesis.NewGenesisBuilder()
53+
func FakeGenesisStoreWithRulesAndStart(
54+
num idx.Validator,
55+
balance, stake *big.Int,
56+
rules opera.Rules,
57+
epoch idx.Epoch,
58+
block idx.Block,
59+
stateDbCacheCapacity int,
60+
) *genesisstore.Store {
61+
builder := makegenesis.NewGenesisBuilder(stateDbCacheCapacity)
5562

5663
validators := GetFakeValidators(num)
5764

‎integration/makefakegenesis/json.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@ func LoadGenesisJson(filename string) (*GenesisJson, error) {
6161
return &decoded, nil
6262
}
6363

64-
func ApplyGenesisJson(json *GenesisJson) (*genesisstore.Store, error) {
64+
func ApplyGenesisJson(json *GenesisJson, stateDbCacheCapacity int) (*genesisstore.Store, error) {
6565
if json.BlockZeroTime.IsZero() {
6666
return nil, fmt.Errorf("block zero time must be set")
6767
}
6868

69-
builder := makegenesis.NewGenesisBuilder()
69+
builder := makegenesis.NewGenesisBuilder(stateDbCacheCapacity)
7070

7171
fmt.Printf("Building genesis file - rules: %+v\n", json.Rules)
7272

‎integration/makegenesis/genesis.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ func (b *GenesisBuilder) CurrentHash() hash.Hash {
113113
return er.Hash()
114114
}
115115

116-
func NewGenesisBuilder() *GenesisBuilder {
116+
func NewGenesisBuilder(stateDbCacheCapacity int) *GenesisBuilder {
117117
carmenDir, err := os.MkdirTemp("", "opera-tmp-genesis")
118118
if err != nil {
119119
panic(fmt.Errorf("failed to create temporary dir for GenesisBuilder: %v", err))
@@ -128,7 +128,7 @@ func NewGenesisBuilder() *GenesisBuilder {
128128
if err != nil {
129129
panic(fmt.Errorf("failed to create carmen state; %s", err))
130130
}
131-
carmenStateDb := carmen.CreateStateDBUsing(carmenState)
131+
carmenStateDb := carmen.CreateCustomStateDBUsing(carmenState, stateDbCacheCapacity)
132132
tmpStateDB := evmstore.CreateCarmenStateDb(carmenStateDb)
133133
return &GenesisBuilder{
134134
tmpStateDB: tmpStateDB,

‎tests/integration_test_net.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -200,15 +200,14 @@ func startIntegrationTestNet(directory string, args []string) (*IntegrationTestN
200200
"--datadir", result.stateDir(),
201201
"--statedb.livecache", "1",
202202
"--statedb.archivecache", "1",
203+
"--statedb.cache", "1024",
203204
}, args...)
204205
if err := sonictool.Run(); err != nil {
205206
os.Args = originalArgs
206207
return nil, fmt.Errorf("failed to initialize the test network: %w", err)
207208
}
208209
os.Args = originalArgs
209210

210-
os.Args = originalArgs
211-
212211
if err := result.start(); err != nil {
213212
return nil, fmt.Errorf("failed to start the test network: %w", err)
214213
}
@@ -273,6 +272,7 @@ func (n *IntegrationTestNet) start() error {
273272
// database memory usage options
274273
"--statedb.livecache", "1",
275274
"--statedb.archivecache", "1",
275+
"--statedb.cache", "1024",
276276
}
277277

278278
err := sonicd.Run()

0 commit comments

Comments
 (0)