Lower memory consumption for integration tests. (#334)

cabrador · web-flow · commit 4a2d82be76aa · 2024-12-04T12:08:45.000+01:00
diff --git a/cmd/sonicd/app/launcher.go b/cmd/sonicd/app/launcher.go
@@ -66,6 +66,7 @@ func initFlags() {
 		flags.CacheFlag,
 		flags.LiveDbCacheFlag,
 		flags.ArchiveCacheFlag,
+		flags.StateDbCacheCapacityFlag,
 	}
 	networkingFlags = []cli.Flag{
 		flags.BootnodesFlag,
diff --git a/cmd/sonicd/app/run_test.go b/cmd/sonicd/app/run_test.go
@@ -23,7 +23,11 @@ func tmpdir(t *testing.T) string {
 }
 
 func initFakenetDatadir(dataDir string, validatorsNum idx.Validator) {
-	genesisStore := makefakegenesis.FakeGenesisStore(validatorsNum, futils.ToFtm(1000000000), futils.ToFtm(5000000))
+	genesisStore := makefakegenesis.FakeGenesisStore(
+		validatorsNum,
+		futils.ToFtm(1000000000),
+		futils.ToFtm(5000000),
+	)
 	defer genesisStore.Close()
 
 	if err := genesis.ImportGenesisStore(genesis.ImportParams{
diff --git a/cmd/sonictool/app/genesis.go b/cmd/sonictool/app/genesis.go
@@ -100,6 +100,7 @@ func jsonGenesisImport(ctx *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to load JSON genesis: %w", err)
 	}
+
 	genesisStore, err := makefakegenesis.ApplyGenesisJson(genesisJson)
 	if err != nil {
 		return fmt.Errorf("failed to prepare JSON genesis: %w", err)
@@ -139,7 +140,11 @@ func fakeGenesisImport(ctx *cli.Context) error {
 		return err
 	}
 
-	genesisStore := makefakegenesis.FakeGenesisStore(idx.Validator(validatorsNumber), futils.ToFtm(1000000000), futils.ToFtm(5000000))
+	genesisStore := makefakegenesis.FakeGenesisStore(
+		idx.Validator(validatorsNumber),
+		futils.ToFtm(1000000000),
+		futils.ToFtm(5000000),
+	)
 	defer genesisStore.Close()
 	return genesis.ImportGenesisStore(genesis.ImportParams{
 		GenesisStore:  genesisStore,
diff --git a/cmd/sonictool/app/main.go b/cmd/sonictool/app/main.go
@@ -20,6 +20,7 @@ func Run() error {
 		flags.CacheFlag,
 		flags.LiveDbCacheFlag,
 		flags.ArchiveCacheFlag,
+		flags.StateDbCacheCapacityFlag,
 	}
 	app.Commands = []cli.Command{
 		{
diff --git a/config/config.go b/config/config.go
@@ -341,6 +341,10 @@ func MakeAllConfigsFromFile(ctx *cli.Context, configFile string) (*Config, error
 		cfg.Lachesis.SuppressFramePanic = true
 	}
 
+	if ctx.IsSet(flags.StateDbCacheCapacityFlag.Name) {
+		cfg.OperaStore.EVM.Cache.StateDbCapacity = ctx.GlobalInt(flags.StateDbCacheCapacityFlag.Name)
+	}
+
 	return &cfg, nil
 }
 
diff --git a/config/flags/flags.go b/config/flags/flags.go
@@ -365,4 +365,11 @@ var (
 			"Setting this value to <=2000 will result in pre-confugired cache capacity of 2KB", CacheFlag.Name),
 		Value: 0,
 	}
+	StateDbCacheCapacityFlag = cli.IntFlag{
+		Name: "statedb.cache",
+		Usage: "The number of cached data elements by each StateDb instance. Since this is an experimental feature " +
+			"used mainly by tests it is generally recommended leaving this blank. In tests no value lower than 1024 " +
+			"is recommended. Setting this to <1 will automatically set the cache capacity to a DB defined default value.",
+		Value: 0,
+	}
 )
diff --git a/gossip/common_test.go b/gossip/common_test.go
@@ -143,7 +143,14 @@ func newTestEnv(firstEpoch idx.Epoch, validatorsNum idx.Validator, tb testing.TB
 	rules.Blocks.MaxEmptyBlockSkipPeriod = 0
 	rules.Emitter.Interval = 0
 
-	genStore := makefakegenesis.FakeGenesisStoreWithRulesAndStart(validatorsNum, utils.ToFtm(genesisBalance), utils.ToFtm(genesisStake), rules, firstEpoch, 2)
+	genStore := makefakegenesis.FakeGenesisStoreWithRulesAndStart(
+		validatorsNum,
+		utils.ToFtm(genesisBalance),
+		utils.ToFtm(genesisStake),
+		rules,
+		firstEpoch,
+		2,
+	)
 	genesis := genStore.Genesis()
 
 	store, err := NewMemStore(tb)
diff --git a/gossip/evmstore/config.go b/gossip/evmstore/config.go
@@ -19,6 +19,8 @@ type (
 		EvmBlocksNum int
 		// Cache size for EvmBlock (size in bytes).
 		EvmBlocksSize uint
+		// Cache size for StateDb instances (0 for DB-selected default)
+		StateDbCapacity int
 	}
 	// StoreConfig is a config for store db.
 	StoreConfig struct {
@@ -38,11 +40,11 @@ type (
 func DefaultStoreConfig(scale cachescale.Func) StoreConfig {
 	return StoreConfig{
 		Cache: StoreCacheConfig{
-			ReceiptsSize:      scale.U(4 * opt.MiB),
-			ReceiptsBlocks:    scale.I(4000),
-			TxPositions:       scale.I(20000),
-			EvmBlocksNum:      scale.I(5000),
-			EvmBlocksSize:     scale.U(6 * opt.MiB),
+			ReceiptsSize:   scale.U(4 * opt.MiB),
+			ReceiptsBlocks: scale.I(4000),
+			TxPositions:    scale.I(20000),
+			EvmBlocksNum:   scale.I(5000),
+			EvmBlocksSize:  scale.U(6 * opt.MiB),
 		},
 		StateDb: carmen.Parameters{
 			Variant:      "go-file",
diff --git a/gossip/evmstore/store.go b/gossip/evmstore/store.go
@@ -78,7 +78,7 @@ func (s *Store) Open() error {
 	if err != nil {
 		return fmt.Errorf("failed to create carmen state; %s", err)
 	}
-	s.liveStateDb = carmen.CreateStateDBUsing(s.carmenState)
+	s.liveStateDb = carmen.CreateCustomStateDBUsing(s.carmenState, s.cfg.Cache.StateDbCapacity)
 	return nil
 }
 
diff --git a/gossip/handler_fuzz.go b/gossip/handler_fuzz.go
@@ -68,7 +68,11 @@ func makeFuzzedHandler() (h *handler, err error) {
 		genesisStake   = 2 * 4e6
 	)
 
-	genStore := makefakegenesis.FakeGenesisStore(genesisStakers, utils.ToFtm(genesisBalance), utils.ToFtm(genesisStake))
+	genStore := makefakegenesis.FakeGenesisStore(
+		genesisStakers,
+		utils.ToFtm(genesisBalance),
+		utils.ToFtm(genesisStake),
+	)
 	genesis := genStore.Genesis()
 
 	config := DefaultConfig(cachescale.Identity)
diff --git a/integration/makefakegenesis/genesis.go b/integration/makefakegenesis/genesis.go
@@ -50,7 +50,13 @@ func FakeGenesisStoreWithRules(num idx.Validator, balance, stake *big.Int, rules
 	return FakeGenesisStoreWithRulesAndStart(num, balance, stake, rules, 2, 1)
 }
 
-func FakeGenesisStoreWithRulesAndStart(num idx.Validator, balance, stake *big.Int, rules opera.Rules, epoch idx.Epoch, block idx.Block) *genesisstore.Store {
+func FakeGenesisStoreWithRulesAndStart(
+	num idx.Validator,
+	balance, stake *big.Int,
+	rules opera.Rules,
+	epoch idx.Epoch,
+	block idx.Block,
+) *genesisstore.Store {
 	builder := makegenesis.NewGenesisBuilder()
 
 	validators := GetFakeValidators(num)
diff --git a/integration/makegenesis/genesis.go b/integration/makegenesis/genesis.go
@@ -128,7 +128,8 @@ func NewGenesisBuilder() *GenesisBuilder {
 	if err != nil {
 		panic(fmt.Errorf("failed to create carmen state; %s", err))
 	}
-	carmenStateDb := carmen.CreateStateDBUsing(carmenState)
+	// Set cache size to lowest value possible
+	carmenStateDb := carmen.CreateCustomStateDBUsing(carmenState, 1024)
 	tmpStateDB := evmstore.CreateCarmenStateDb(carmenStateDb)
 	return &GenesisBuilder{
 		tmpStateDB:    tmpStateDB,
diff --git a/tests/integration_test_net.go b/tests/integration_test_net.go
@@ -201,15 +201,14 @@ func startIntegrationTestNet(directory string, args []string) (*IntegrationTestN
 		"--datadir", result.stateDir(),
 		"--statedb.livecache", "1",
 		"--statedb.archivecache", "1",
+		"--statedb.cache", "1024",
 	}, args...)
 	if err := sonictool.Run(); err != nil {
 		os.Args = originalArgs
 		return nil, fmt.Errorf("failed to initialize the test network: %w", err)
 	}
 	os.Args = originalArgs
 
-	os.Args = originalArgs
-
 	if err := result.start(); err != nil {
 		return nil, fmt.Errorf("failed to start the test network: %w", err)
 	}
@@ -274,6 +273,7 @@ func (n *IntegrationTestNet) start() error {
 			// database memory usage options
 			"--statedb.livecache", "1",
 			"--statedb.archivecache", "1",
+			"--statedb.cache", "1024",
 		}
 
 		err := sonicd.Run()

Original file line number	Diff line number	Diff line change
`@@ -66,6 +66,7 @@ func initFlags() {`
`66`	`66`	`flags.CacheFlag,`
`67`	`67`	`flags.LiveDbCacheFlag,`
`68`	`68`	`flags.ArchiveCacheFlag,`
	`69`	`+ flags.StateDbCacheCapacityFlag,`
`69`	`70`	`}`
`70`	`71`	`networkingFlags = []cli.Flag{`
`71`	`72`	`flags.BootnodesFlag,`
Original file line number	Diff line number	Diff line change
`@@ -20,6 +20,7 @@ func Run() error {`
`20`	`20`	`flags.CacheFlag,`
`21`	`21`	`flags.LiveDbCacheFlag,`
`22`	`22`	`flags.ArchiveCacheFlag,`
	`23`	`+ flags.StateDbCacheCapacityFlag,`
`23`	`24`	`}`
`24`	`25`	`app.Commands = []cli.Command{`
`25`	`26`	`{`
Original file line number	Diff line number	Diff line change
`@@ -341,6 +341,10 @@ func MakeAllConfigsFromFile(ctx cli.Context, configFile string) (Config, error`
`341`	`341`	`cfg.Lachesis.SuppressFramePanic = true`
`342`	`342`	`}`
`343`	`343`
	`344`	`+ if ctx.IsSet(flags.StateDbCacheCapacityFlag.Name) {`
	`345`	`+ cfg.OperaStore.EVM.Cache.StateDbCapacity = ctx.GlobalInt(flags.StateDbCacheCapacityFlag.Name)`
	`346`	`+ }`
	`347`	`+`
`344`	`348`	`return &cfg, nil`
`345`	`349`	`}`
`346`	`350`
Original file line number	Diff line number	Diff line change
`@@ -365,4 +365,11 @@ var (`
`365`	`365`	`"Setting this value to <=2000 will result in pre-confugired cache capacity of 2KB", CacheFlag.Name),`
`366`	`366`	`Value: 0,`
`367`	`367`	`}`
	`368`	`+ StateDbCacheCapacityFlag = cli.IntFlag{`
	`369`	`+ Name: "statedb.cache",`
	`370`	`+ Usage: "The number of cached data elements by each StateDb instance. Since this is an experimental feature " +`
	`371`	`+ "used mainly by tests it is generally recommended leaving this blank. In tests no value lower than 1024 " +`
	`372`	`+ "is recommended. Setting this to <1 will automatically set the cache capacity to a DB defined default value.",`
	`373`	`+ Value: 0,`
	`374`	`+ }`
`368`	`375`	`)`
Original file line number	Diff line number	Diff line change
`@@ -78,7 +78,7 @@ func (s *Store) Open() error {`
`78`	`78`	`if err != nil {`
`79`	`79`	`return fmt.Errorf("failed to create carmen state; %s", err)`
`80`	`80`	`}`
`81`		`- s.liveStateDb = carmen.CreateStateDBUsing(s.carmenState)`
	`81`	`+ s.liveStateDb = carmen.CreateCustomStateDBUsing(s.carmenState, s.cfg.Cache.StateDbCapacity)`
`82`	`82`	`return nil`
`83`	`83`	`}`
`84`	`84`