new erc20 holder provider following the holder provider interface, it…

… works but it is not perfect

api/api.go

@@ -9,7 +9,6 @@ import (
 	"github.com/vocdoni/census3/queue"
 	"github.com/vocdoni/census3/service"
 	"github.com/vocdoni/census3/service/web3"
-	"github.com/vocdoni/census3/state"
 	"go.vocdoni.io/dvote/api/censusdb"
 	storagelayer "go.vocdoni.io/dvote/data"
 	"go.vocdoni.io/dvote/data/downloader"
@@ -29,7 +28,7 @@ type Census3APIConf struct {
 	DataDir       string
 	GroupKey      string
 	Web3Providers web3.NetworkEndpoints
-	ExtProviders  map[state.TokenType]service.HolderProvider
+	ExtProviders  map[uint64]service.HolderProvider
 	AdminToken    string
 }
 
@@ -42,7 +41,7 @@ type census3API struct {
 	w3p          web3.NetworkEndpoints
 	storage      storagelayer.Storage
 	downloader   *downloader.Downloader
-	extProviders map[state.TokenType]service.HolderProvider
+	extProviders map[uint64]service.HolderProvider
 	cache        *lru.Cache[CacheKey, any]
 }
 

api/holders.go

@@ -13,7 +13,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	queries "github.com/vocdoni/census3/db/sqlc"
 	"github.com/vocdoni/census3/service/web3"
-	"github.com/vocdoni/census3/state"
 	"go.vocdoni.io/dvote/httprouter"
 	api "go.vocdoni.io/dvote/httprouter/apirest"
 	"go.vocdoni.io/dvote/log"
@@ -108,7 +107,7 @@ func (capi *census3API) listHoldersAtLastBlock(address common.Address,
 	}
 	// if the token is external, return an error
 	// TODO: implement external token holders
-	if _, isExternal := capi.extProviders[state.TokenType(tokenData.TypeID)]; isExternal {
+	if _, isExternal := capi.extProviders[tokenData.TypeID]; isExternal {
 		return nil, 0, ErrCantCreateCensus.With("not implemented for external providers")
 	}
 	// get correct web3 uri provider

api/tokens.go

@@ -15,6 +15,7 @@ import (
 	queries "github.com/vocdoni/census3/db/sqlc"
 	"github.com/vocdoni/census3/internal"
 	"github.com/vocdoni/census3/lexer"
+	"github.com/vocdoni/census3/service/web3"
 	"github.com/vocdoni/census3/state"
 	"go.vocdoni.io/dvote/httprouter"
 	api "go.vocdoni.io/dvote/httprouter/apirest"
@@ -224,61 +225,47 @@ func (capi *census3API) createToken(msg *api.APIdata, ctx *httprouter.HTTPContex
 	internalCtx, cancel := context.WithTimeout(ctx.Request.Context(), createTokenTimeout)
 	defer cancel()
 
-	var info *state.TokenData
-	tokenType := state.TokenTypeFromString(req.Type)
-	if provider, exists := capi.extProviders[tokenType]; exists {
-		// get token information from the external provider
-		address := provider.Address()
-		name, err := provider.Name([]byte(req.ExternalID))
-		if err != nil {
-			return ErrCantGetToken.WithErr(err)
-		}
-		symbol, err := provider.Symbol([]byte(req.ExternalID))
-		if err != nil {
-			return ErrCantGetToken.WithErr(err)
-		}
-		decimals, err := provider.Decimals([]byte(req.ExternalID))
-		if err != nil {
-			return ErrCantGetToken.WithErr(err)
-		}
-		totalSupply, err := provider.TotalSupply([]byte(req.ExternalID))
-		if err != nil {
-			return ErrCantGetToken.WithErr(err)
-		}
-		iconURI, err := provider.IconURI([]byte(req.ExternalID))
+	endpoint, ok := capi.w3p.EndpointByChainID(req.ChainID)
+	if !ok {
+		return ErrChainIDNotSupported.With("chain ID not supported")
+	}
+	tokenType := web3.TokenTypeFromString(req.Type)
+	provider, isExternal := capi.extProviders[tokenType]
+	if !isExternal {
+		client, err := endpoint.GetClient(web3.DefaultMaxWeb3ClientRetries)
 		if err != nil {
-			return ErrCantGetToken.WithErr(err)
+			return ErrInitializingWeb3.WithErr(err)
 		}
-		// build token information struct with the data from the external
-		// provider
-		info = &state.TokenData{
-			Type:        tokenType,
-			Address:     address,
-			Name:        name,
-			Symbol:      symbol,
-			Decimals:    decimals,
-			TotalSupply: totalSupply,
-			IconURI:     iconURI,
+		provider = &web3.ERC20HolderProvider{
+			HexAddress: req.ID,
+			ChainID:    req.ChainID,
+			Client:     client,
 		}
-	} else {
-		addr := common.HexToAddress(req.ID)
-		// init web3 client to get the token information before register in the
-		// database
-		w3 := state.Web3{}
-		// get correct web3 uri provider
-		w3URI, exists := capi.w3p.EndpointByChainID(req.ChainID)
-		if !exists {
-			return ErrChainIDNotSupported.With("chain ID not supported")
-		}
-		// init web3 client to get the token information
-		err := w3.Init(internalCtx, w3URI, addr, tokenType)
-		if err != nil {
+		if err := provider.Init(); err != nil {
 			return ErrInitializingWeb3.WithErr(err)
 		}
-		// get token information from the web3 client
-		if info, err = w3.TokenData(); err != nil {
-			return ErrCantGetToken.WithErr(err)
-		}
+	}
+	// get token information from the external provider
+	address := provider.Address()
+	name, err := provider.Name([]byte(req.ExternalID))
+	if err != nil {
+		return ErrCantGetToken.WithErr(err)
+	}
+	symbol, err := provider.Symbol([]byte(req.ExternalID))
+	if err != nil {
+		return ErrCantGetToken.WithErr(err)
+	}
+	decimals, err := provider.Decimals([]byte(req.ExternalID))
+	if err != nil {
+		return ErrCantGetToken.WithErr(err)
+	}
+	totalSupply, err := provider.TotalSupply([]byte(req.ExternalID))
+	if err != nil {
+		return ErrCantGetToken.WithErr(err)
+	}
+	iconURI, err := provider.IconURI([]byte(req.ExternalID))
+	if err != nil {
+		return ErrCantGetToken.WithErr(err)
 	}
 	// init db transaction
 	tx, err := capi.db.RW.BeginTx(internalCtx, nil)
@@ -291,29 +278,29 @@ func (capi *census3API) createToken(msg *api.APIdata, ctx *httprouter.HTTPContex
 		}
 	}()
 	// get the chain address for the token based on the chainID and tokenID
-	chainAddress, ok := capi.w3p.ChainAddress(req.ChainID, info.Address.String())
+	chainAddress, ok := capi.w3p.ChainAddress(req.ChainID, address.String())
 	if !ok {
 		return ErrChainIDNotSupported.Withf("chainID: %d, tokenID: %s", req.ChainID, req.ID)
 	}
-	totalSupply := big.NewInt(0).String()
-	if info.TotalSupply != nil {
-		totalSupply = info.TotalSupply.String()
+	sTotalSupply := big.NewInt(0).String()
+	if totalSupply != nil {
+		sTotalSupply = totalSupply.String()
 	}
 	qtx := capi.db.QueriesRW.WithTx(tx)
 	_, err = qtx.CreateToken(internalCtx, queries.CreateTokenParams{
-		ID:            info.Address.Bytes(),
-		Name:          info.Name,
-		Symbol:        info.Symbol,
-		Decimals:      info.Decimals,
-		TotalSupply:   annotations.BigInt(totalSupply),
+		ID:            address.Bytes(),
+		Name:          name,
+		Symbol:        symbol,
+		Decimals:      decimals,
+		TotalSupply:   annotations.BigInt(sTotalSupply),
 		CreationBlock: 0,
-		TypeID:        uint64(tokenType),
+		TypeID:        tokenType,
 		Synced:        false,
 		Tags:          req.Tags,
 		ChainID:       req.ChainID,
 		ChainAddress:  chainAddress,
 		ExternalID:    req.ExternalID,
-		IconUri:       info.IconURI,
+		IconUri:       iconURI,
 	})
 	if err != nil {
 		if strings.Contains(err.Error(), "UNIQUE constraint failed") {
@@ -322,12 +309,12 @@ func (capi *census3API) createToken(msg *api.APIdata, ctx *httprouter.HTTPContex
 		return ErrCantCreateToken.WithErr(err)
 	}
 	strategyID, err := capi.createDefaultTokenStrategy(internalCtx, qtx,
-		info.Address, req.ChainID, chainAddress, info.Symbol, req.ExternalID)
+		address, req.ChainID, chainAddress, symbol, req.ExternalID)
 	if err != nil {
 		return ErrCantCreateToken.WithErr(err)
 	}
 	if _, err := qtx.UpdateTokenDefaultStrategy(internalCtx, queries.UpdateTokenDefaultStrategyParams{
-		ID:              info.Address.Bytes(),
+		ID:              address.Bytes(),
 		DefaultStrategy: uint64(strategyID),
 		ChainID:         req.ChainID,
 		ExternalID:      req.ExternalID,

cmd/census3/main.go

@@ -19,7 +19,6 @@ import (
 	"github.com/vocdoni/census3/service"
 	"github.com/vocdoni/census3/service/poap"
 	"github.com/vocdoni/census3/service/web3"
-	"github.com/vocdoni/census3/state"
 	"go.vocdoni.io/dvote/log"
 )
 
@@ -126,8 +125,8 @@ func main() {
 		log.Fatal(err)
 	}
 	// start the holder scanner with the database and the external providers
-	externalProviders := map[state.TokenType]service.HolderProvider{
-		state.CONTRACT_TYPE_POAP: poapProvider,
+	externalProviders := map[uint64]service.HolderProvider{
+		web3.CONTRACT_TYPE_POAP: poapProvider,
 	}
 	hc, err := service.NewHoldersScanner(database, w3p, externalProviders, config.scannerCoolDown)
 	if err != nil {

service/holder_scanner_test.go

@@ -12,7 +12,6 @@ import (
 	qt "github.com/frankban/quicktest"
 	queries "github.com/vocdoni/census3/db/sqlc"
 	"github.com/vocdoni/census3/service/web3"
-	"github.com/vocdoni/census3/state"
 )
 
 var (
@@ -82,7 +81,7 @@ func Test_getTokensToScan(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	_, err = testdb.db.QueriesRW.CreateToken(ctx, testTokenParams("0x1", "test0",
-		"test0", 0, MonkeysDecimals, uint64(state.CONTRACT_TYPE_ERC20),
+		"test0", 0, MonkeysDecimals, uint64(web3.CONTRACT_TYPE_ERC20),
 		MonkeysTotalSupply.Int64(), false, 5, ""))
 	c.Assert(err, qt.IsNil)
 
@@ -92,7 +91,7 @@ func Test_getTokensToScan(t *testing.T) {
 	c.Assert(hs.tokens[0].Address().String(), qt.Equals, common.HexToAddress("0x1").String())
 
 	_, err = testdb.db.QueriesRW.CreateToken(ctx, testTokenParams("0x2", "test2",
-		"test3", 10, MonkeysDecimals, uint64(state.CONTRACT_TYPE_ERC20),
+		"test3", 10, MonkeysDecimals, uint64(web3.CONTRACT_TYPE_ERC20),
 		MonkeysTotalSupply.Int64(), false, 5, ""))
 	c.Assert(err, qt.IsNil)
 
@@ -111,12 +110,12 @@ func Test_saveHolders(t *testing.T) {
 	hs, err := NewHoldersScanner(testdb.db, web3Endpoints, nil, 20)
 	c.Assert(err, qt.IsNil)
 
-	th := new(state.TokenHolders).Init(MonkeysAddress, state.CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5, "")
+	th := new(TokenHolders).Init(MonkeysAddress, web3.CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5, "")
 	// no registered token
 	c.Assert(hs.saveHolders(th), qt.ErrorIs, ErrTokenNotExists)
 	_, err = testdb.db.QueriesRW.CreateToken(context.Background(), testTokenParams(
 		MonkeysAddress.String(), MonkeysName, MonkeysSymbol, MonkeysCreationBlock,
-		MonkeysDecimals, uint64(state.CONTRACT_TYPE_ERC20), MonkeysTotalSupply.Int64(), false, 5, ""))
+		MonkeysDecimals, uint64(web3.CONTRACT_TYPE_ERC20), MonkeysTotalSupply.Int64(), false, 5, ""))
 	c.Assert(err, qt.IsNil)
 	// check no new holders
 	c.Assert(hs.saveHolders(th), qt.IsNil)
@@ -177,7 +176,7 @@ func Test_scanHolders(t *testing.T) {
 
 	_, err = testdb.db.QueriesRW.CreateToken(context.Background(), testTokenParams(
 		MonkeysAddress.String(), MonkeysName, MonkeysSymbol, MonkeysCreationBlock,
-		MonkeysDecimals, uint64(state.CONTRACT_TYPE_ERC20), 10, false, 5, ""))
+		MonkeysDecimals, uint64(web3.CONTRACT_TYPE_ERC20), 10, false, 5, ""))
 	c.Assert(err, qt.IsNil)
 	// token exists and the scanner gets the holders
 	ctx2, cancel := context.WithTimeout(context.Background(), 10*time.Second)
@@ -203,13 +202,13 @@ func Test_calcTokenCreationBlock(t *testing.T) {
 	defer testdb.Close(t)
 
 	hs, err := NewHoldersScanner(testdb.db, web3Endpoints, nil, 20)
-	hs.tokens = append(hs.tokens, new(state.TokenHolders).Init(MonkeysAddress, state.CONTRACT_TYPE_ERC20, 0, 5, ""))
+	hs.tokens = append(hs.tokens, new(TokenHolders).Init(MonkeysAddress, web3.CONTRACT_TYPE_ERC20, 0, 5, ""))
 	c.Assert(err, qt.IsNil)
 	c.Assert(hs.calcTokenCreationBlock(context.Background(), 5), qt.IsNotNil)
 
 	_, err = testdb.db.QueriesRW.CreateToken(context.Background(), testTokenParams(
 		MonkeysAddress.String(), MonkeysName, MonkeysSymbol, MonkeysCreationBlock,
-		MonkeysDecimals, uint64(state.CONTRACT_TYPE_ERC20), MonkeysTotalSupply.Int64(), false, 5, ""))
+		MonkeysDecimals, uint64(web3.CONTRACT_TYPE_ERC20), MonkeysTotalSupply.Int64(), false, 5, ""))
 	c.Assert(err, qt.IsNil)
 
 	c.Assert(hs.calcTokenCreationBlock(context.Background(), 0), qt.IsNil)

service/holders_provider.go

@@ -23,7 +23,7 @@ type HolderProvider interface {
 	SetLastBalances(ctx context.Context, id []byte, balances map[common.Address]*big.Int, from uint64) error
 	// HoldersBalances returns the balances of the token holders for the given
 	// id and delta point in time, from the stored last snapshot.
-	HoldersBalances(ctx context.Context, id []byte, to uint64) (map[common.Address]*big.Int, error)
+	HoldersBalances(ctx context.Context, id []byte, to uint64) (map[common.Address]*big.Int, uint64, error)
 	// Close closes the provider and its internal structures.
 	Close() error
 	// Token realated methods