Skip to content

Commit 179ec05

Browse files
cabradorcabrador
authored
Lower integration tests memory usage (#327)
1 parent cee952b commit 179ec05

File tree

11 files changed

+149
-29
lines changed

11 files changed

+149
-29
lines changed

‎cmd/sonicd/app/launcher.go

+11
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ func initFlags() {
6565
}
6666
performanceFlags = []cli.Flag{
6767
flags.CacheFlag,
68+
flags.LiveDbCacheFlag,
69+
flags.ArchiveCacheFlag,
6870
}
6971
networkingFlags = []cli.Flag{
7072
flags.BootnodesFlag,
@@ -222,6 +224,15 @@ func lachesisMain(ctx *cli.Context) error {
222224
}
223225

224226
metrics.SetDataDir(cfg.Node.DataDir) // report disk space usage into metrics
227+
liveCache := ctx.GlobalInt64(flags.LiveDbCacheFlag.Name)
228+
if liveCache > 0 {
229+
cfg.OperaStore.EVM.StateDb.LiveCache = liveCache
230+
}
231+
232+
archiveCache := ctx.GlobalInt64(flags.ArchiveCacheFlag.Name)
233+
if archiveCache > 0 {
234+
cfg.OperaStore.EVM.StateDb.ArchiveCache = archiveCache
235+
}
225236

226237
node, _, nodeClose, err := config.MakeNode(ctx, cfg)
227238
if err != nil {

‎cmd/sonicd/app/run_test.go

+9-3
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package app
22

33
import (
44
"fmt"
5+
"github.com/Fantom-foundation/lachesis-base/utils/cachescale"
56
"os"
67
"strings"
78
"testing"
@@ -11,8 +12,6 @@ import (
1112
"github.com/Fantom-foundation/go-opera/integration/makefakegenesis"
1213
futils "github.com/Fantom-foundation/go-opera/utils"
1314
"github.com/Fantom-foundation/lachesis-base/inter/idx"
14-
"github.com/Fantom-foundation/lachesis-base/utils/cachescale"
15-
1615
"github.com/docker/docker/pkg/reexec"
1716
"github.com/ethereum/go-ethereum/common"
1817

@@ -26,7 +25,14 @@ func tmpdir(t *testing.T) string {
2625
func initFakenetDatadir(dataDir string, validatorsNum idx.Validator) {
2726
genesisStore := makefakegenesis.FakeGenesisStore(validatorsNum, futils.ToFtm(1000000000), futils.ToFtm(5000000))
2827
defer genesisStore.Close()
29-
if err := genesis.ImportGenesisStore(genesisStore, dataDir, false, cachescale.Identity); err != nil {
28+
29+
if err := genesis.ImportGenesisStore(genesis.ImportParams{
30+
GenesisStore: genesisStore,
31+
DataDir: dataDir,
32+
CacheRatio: cachescale.Identity,
33+
LiveDbCache: 1, // Set lowest cache
34+
ArchiveCache: 1, // Set lowest cache
35+
}); err != nil {
3036
panic(err)
3137
}
3238
}

‎cmd/sonictool/app/chain.go

+8-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package app
33
import (
44
"compress/gzip"
55
"fmt"
6+
"github.com/Fantom-foundation/go-opera/cmd/sonictool/db"
67
"io"
78
"os"
89
"strconv"
@@ -57,8 +58,14 @@ func exportEvents(ctx *cli.Context) error {
5758
to = idx.Epoch(n)
5859
}
5960

61+
gdbParams := db.GossipDbParameters{
62+
DataDir: dataDir,
63+
LiveDbCache: ctx.GlobalInt64(flags.LiveDbCacheFlag.Name),
64+
ArchiveCache: ctx.GlobalInt64(flags.ArchiveCacheFlag.Name),
65+
}
66+
6067
log.Info("Exporting events to file", "file", fn)
61-
err = chain.ExportEvents(writer, dataDir, from, to)
68+
err = chain.ExportEvents(gdbParams, writer, from, to)
6269
if err != nil {
6370
return fmt.Errorf("export error: %w", err)
6471
}

‎cmd/sonictool/app/export_genesis.go

+8-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import (
77
"github.com/Fantom-foundation/go-opera/cmd/sonictool/genesis"
88
"github.com/Fantom-foundation/go-opera/config/flags"
99
"github.com/Fantom-foundation/go-opera/integration"
10-
"github.com/Fantom-foundation/lachesis-base/utils/cachescale"
1110
"github.com/syndtr/goleveldb/leveldb/opt"
1211
"gopkg.in/urfave/cli.v1"
1312
"os"
@@ -48,7 +47,14 @@ func exportGenesis(ctx *cli.Context) error {
4847
}
4948
defer dbs.Close()
5049

51-
gdb, err := db.MakeGossipDb(dbs, dataDir, false, cachescale.Identity)
50+
gdb, err := db.MakeGossipDb(db.GossipDbParameters{
51+
Dbs: dbs,
52+
DataDir: dataDir,
53+
ValidatorMode: false,
54+
CacheRatio: cacheRatio,
55+
LiveDbCache: ctx.GlobalInt64(flags.LiveDbCacheFlag.Name),
56+
ArchiveCache: ctx.GlobalInt64(flags.ArchiveCacheFlag.Name),
57+
})
5258
if err != nil {
5359
return err
5460
}

‎cmd/sonictool/app/genesis.go

+24-3
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,14 @@ func gfileGenesisImport(ctx *cli.Context) error {
6666
return fmt.Errorf("genesis file check failed: %w", err)
6767
}
6868
}
69-
return genesis.ImportGenesisStore(genesisStore, dataDir, validatorMode, cacheRatio)
69+
return genesis.ImportGenesisStore(genesis.ImportParams{
70+
GenesisStore: genesisStore,
71+
DataDir: dataDir,
72+
ValidatorMode: validatorMode,
73+
CacheRatio: cacheRatio,
74+
LiveDbCache: ctx.GlobalInt64(flags.LiveDbCacheFlag.Name),
75+
ArchiveCache: ctx.GlobalInt64(flags.ArchiveCacheFlag.Name),
76+
})
7077
}
7178

7279
func jsonGenesisImport(ctx *cli.Context) error {
@@ -98,7 +105,14 @@ func jsonGenesisImport(ctx *cli.Context) error {
98105
return fmt.Errorf("failed to prepare JSON genesis: %w", err)
99106
}
100107
defer genesisStore.Close()
101-
return genesis.ImportGenesisStore(genesisStore, dataDir, validatorMode, cacheRatio)
108+
return genesis.ImportGenesisStore(genesis.ImportParams{
109+
GenesisStore: genesisStore,
110+
DataDir: dataDir,
111+
ValidatorMode: validatorMode,
112+
CacheRatio: cacheRatio,
113+
LiveDbCache: ctx.GlobalInt64(flags.LiveDbCacheFlag.Name),
114+
ArchiveCache: ctx.GlobalInt64(flags.ArchiveCacheFlag.Name),
115+
})
102116
}
103117

104118
func fakeGenesisImport(ctx *cli.Context) error {
@@ -127,7 +141,14 @@ func fakeGenesisImport(ctx *cli.Context) error {
127141

128142
genesisStore := makefakegenesis.FakeGenesisStore(idx.Validator(validatorsNumber), futils.ToFtm(1000000000), futils.ToFtm(5000000))
129143
defer genesisStore.Close()
130-
return genesis.ImportGenesisStore(genesisStore, dataDir, validatorMode, cacheRatio)
144+
return genesis.ImportGenesisStore(genesis.ImportParams{
145+
GenesisStore: genesisStore,
146+
DataDir: dataDir,
147+
ValidatorMode: validatorMode,
148+
CacheRatio: cacheRatio,
149+
LiveDbCache: ctx.GlobalInt64(flags.LiveDbCacheFlag.Name),
150+
ArchiveCache: ctx.GlobalInt64(flags.ArchiveCacheFlag.Name),
151+
})
131152
}
132153

133154
func isValidatorModeSet(ctx *cli.Context) (bool, error) {

‎cmd/sonictool/app/main.go

+2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ func Run() error {
1818
app.Flags = []cli.Flag{
1919
flags.DataDirFlag,
2020
flags.CacheFlag,
21+
flags.LiveDbCacheFlag,
22+
flags.ArchiveCacheFlag,
2123
}
2224
app.Commands = []cli.Command{
2325
{

‎cmd/sonictool/chain/export_events.go

+7-3
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,19 @@ var (
2424
// always print out progress. This avoids the user wondering what's going on.
2525
const statsReportLimit = 8 * time.Second
2626

27-
func ExportEvents(w io.Writer, dataDir string, from, to idx.Epoch) (err error) {
28-
chaindataDir := filepath.Join(dataDir, "chaindata")
27+
func ExportEvents(gdbParams db.GossipDbParameters, w io.Writer, from, to idx.Epoch) (err error) {
28+
chaindataDir := filepath.Join(gdbParams.DataDir, "chaindata")
2929
dbs, err := db.MakeDbProducer(chaindataDir, cachescale.Identity)
3030
if err != nil {
3131
return err
3232
}
3333
defer dbs.Close()
3434

35-
gdb, err := db.MakeGossipDb(dbs, dataDir, false, cachescale.Identity)
35+
// Fill the rest of the params
36+
gdbParams.Dbs = dbs
37+
gdbParams.CacheRatio = cachescale.Identity
38+
39+
gdb, err := db.MakeGossipDb(gdbParams)
3640
if err != nil {
3741
return err
3842
}

‎cmd/sonictool/db/dbutils.go

+22-6
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func makeDatabaseHandles() uint64 {
3131
if err != nil {
3232
panic(fmt.Errorf("failed to raise file descriptor allowance: %v", err))
3333
}
34-
return raised / 6 + 1
34+
return raised/6 + 1
3535
}
3636

3737
func AssertDatabaseNotInitialized(dataDir string) error {
@@ -61,16 +61,32 @@ func MakeDbProducer(chaindataDir string, cacheRatio cachescale.Func) (kvdb.FullD
6161
})
6262
}
6363

64-
func MakeGossipDb(dbs kvdb.FullDBProducer, dataDir string, validatorMode bool, cacheRatio cachescale.Func) (*gossip.Store, error) {
65-
gdbConfig := gossip.DefaultStoreConfig(cacheRatio)
66-
gdbConfig.EVM.StateDb.Directory = filepath.Join(dataDir, "carmen")
67-
if validatorMode {
64+
// GossipDbParameters are parameters for GossipDb factory func.
65+
type GossipDbParameters struct {
66+
Dbs kvdb.FullDBProducer
67+
DataDir string
68+
ValidatorMode bool
69+
CacheRatio cachescale.Func
70+
LiveDbCache, ArchiveCache int64 // in bytes
71+
}
72+
73+
func MakeGossipDb(params GossipDbParameters) (*gossip.Store, error) {
74+
gdbConfig := gossip.DefaultStoreConfig(params.CacheRatio)
75+
if params.ValidatorMode {
6876
gdbConfig.EVM.StateDb.Archive = carmen.NoArchive
6977
gdbConfig.EVM.DisableLogsIndexing = true
7078
gdbConfig.EVM.DisableTxHashesIndexing = true
7179
}
7280

73-
gdb, err := gossip.NewStore(dbs, gdbConfig)
81+
if params.LiveDbCache > 0 {
82+
gdbConfig.EVM.StateDb.LiveCache = params.LiveDbCache
83+
}
84+
if params.ArchiveCache > 0 {
85+
gdbConfig.EVM.StateDb.ArchiveCache = params.ArchiveCache
86+
}
87+
gdbConfig.EVM.StateDb.Directory = filepath.Join(params.DataDir, "carmen")
88+
89+
gdb, err := gossip.NewStore(params.Dbs, gdbConfig)
7490
if err != nil {
7591
return nil, fmt.Errorf("failed to create gossip store: %w", err)
7692
}

‎cmd/sonictool/genesis/import.go

+24-8
Original file line numberDiff line numberDiff line change
@@ -13,29 +13,45 @@ import (
1313
"path/filepath"
1414
)
1515

16-
func ImportGenesisStore(genesisStore *genesisstore.Store, dataDir string, validatorMode bool, cacheRatio cachescale.Func) error {
17-
if err := db.AssertDatabaseNotInitialized(dataDir); err != nil {
16+
// ImportParams are parameters for ImportGenesisStore func.
17+
type ImportParams struct {
18+
GenesisStore *genesisstore.Store
19+
DataDir string
20+
ValidatorMode bool
21+
CacheRatio cachescale.Func
22+
LiveDbCache, ArchiveCache int64 // in bytes
23+
}
24+
25+
func ImportGenesisStore(params ImportParams) error {
26+
if err := db.AssertDatabaseNotInitialized(params.DataDir); err != nil {
1827
return fmt.Errorf("database in datadir is already initialized: %w", err)
1928
}
20-
if err := db.RemoveDatabase(dataDir); err != nil {
29+
if err := db.RemoveDatabase(params.DataDir); err != nil {
2130
return fmt.Errorf("failed to remove existing data from the datadir: %w", err)
2231
}
2332

24-
chaindataDir := filepath.Join(dataDir, "chaindata")
25-
dbs, err := db.MakeDbProducer(chaindataDir, cacheRatio)
33+
chaindataDir := filepath.Join(params.DataDir, "chaindata")
34+
dbs, err := db.MakeDbProducer(chaindataDir, params.CacheRatio)
2635
if err != nil {
2736
return err
2837
}
2938
defer dbs.Close()
3039
setGenesisProcessing(chaindataDir)
3140

32-
gdb, err := db.MakeGossipDb(dbs, dataDir, validatorMode, cacheRatio)
41+
gdb, err := db.MakeGossipDb(db.GossipDbParameters{
42+
Dbs: dbs,
43+
DataDir: params.DataDir,
44+
ValidatorMode: params.ValidatorMode,
45+
CacheRatio: params.CacheRatio,
46+
LiveDbCache: params.LiveDbCache,
47+
ArchiveCache: params.ArchiveCache,
48+
})
3349
if err != nil {
3450
return err
3551
}
3652
defer gdb.Close()
3753

38-
err = gdb.ApplyGenesis(genesisStore.Genesis())
54+
err = gdb.ApplyGenesis(params.GenesisStore.Genesis())
3955
if err != nil {
4056
return fmt.Errorf("failed to write Gossip genesis state: %v", err)
4157
}
@@ -54,7 +70,7 @@ func ImportGenesisStore(genesisStore *genesisstore.Store, dataDir string, valida
5470
abftCrit := func(err error) {
5571
panic(fmt.Errorf("lachesis store error: %w", err))
5672
}
57-
cdb := abft.NewStore(cMainDb, cGetEpochDB, abftCrit, abft.DefaultStoreConfig(cacheRatio))
73+
cdb := abft.NewStore(cMainDb, cGetEpochDB, abftCrit, abft.DefaultStoreConfig(params.CacheRatio))
5874
defer cdb.Close()
5975

6076
err = cdb.ApplyGenesis(&abft.Genesis{

‎config/flags/flags.go

+17
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package flags
1818

1919
import (
20+
"fmt"
2021
"strings"
2122

2223
"github.com/Fantom-foundation/go-opera/evmcore"
@@ -348,4 +349,20 @@ var (
348349
Name: "lachesis.suppress-frame-panic",
349350
Usage: "Suppress frame missmatch error (when testing on historical imported/synced events)",
350351
}
352+
353+
// StateDb
354+
LiveDbCacheFlag = cli.Int64Flag{
355+
Name: "statedb.livecache",
356+
Usage: fmt.Sprintf("Size of live db cache in bytes. Leaving this blank (which is generally recommended),"+
357+
"or setting this to <1 will automatically allocate cache size depending on how much cache you use with %s."+
358+
"Setting this value to <=2000 will result in pre-confugired cache capacity of 2KB", CacheFlag.Name),
359+
Value: 0,
360+
}
361+
ArchiveCacheFlag = cli.IntFlag{
362+
Name: "statedb.archivecache",
363+
Usage: fmt.Sprintf("Size of archive cache in bytes. Leaving this blank (which is generally recommended),"+
364+
"or setting this to <1 will automatically allocate cache size depending on how much cache you use with %s."+
365+
"Setting this value to <=2000 will result in pre-confugired cache capacity of 2KB", CacheFlag.Name),
366+
Value: 0,
367+
}
351368
)

‎tests/integration_test_net.go

+17-3
Original file line numberDiff line numberDiff line change
@@ -169,13 +169,20 @@ func startIntegrationTestNet(directory string, args []string) (*IntegrationTestN
169169
// initialize the data directory for the single node on the test network
170170
// equivalent to running `sonictool --datadir <dataDir> genesis fake 1`
171171
originalArgs := os.Args
172-
os.Args = append([]string{"sonictool", "--datadir", result.stateDir()}, args...)
172+
os.Args = append([]string{
173+
"sonictool",
174+
"--datadir", result.stateDir(),
175+
"--statedb.livecache", "1",
176+
"--statedb.archivecache", "1",
177+
}, args...)
173178
if err := sonictool.Run(); err != nil {
174179
os.Args = originalArgs
175180
return nil, fmt.Errorf("failed to initialize the test network: %w", err)
176181
}
177182
os.Args = originalArgs
178183

184+
os.Args = originalArgs
185+
179186
if err := result.start(); err != nil {
180187
return nil, fmt.Errorf("failed to start the test network: %w", err)
181188
}
@@ -200,10 +207,17 @@ func (n *IntegrationTestNet) start() error {
200207
"sonicd",
201208
"--datadir", n.stateDir(),
202209
"--fakenet", "1/1",
203-
"--http", "--http.addr", "0.0.0.0", "--http.port", "18545",
210+
"--http",
211+
"--http.addr", "0.0.0.0",
212+
"--http.port", "18545",
204213
"--http.api", "admin,eth,web3,net,txpool,ftm,trace,debug",
205-
"--ws", "--ws.addr", "0.0.0.0", "--ws.port", "18546", "--ws.api", "admin,eth,ftm",
214+
"--ws",
215+
"--ws.addr", "0.0.0.0",
216+
"--ws.port", "18546",
217+
"--ws.api", "admin,eth,ftm",
206218
"--datadir.minfreedisk", "0",
219+
"--statedb.livecache", "1",
220+
"--statedb.archivecache", "1",
207221
}
208222
sonicd.Run()
209223
}()

0 commit comments

Comments
 (0)