clean version of ERC20 holder provider, merged with external providers

api/api.go

@@ -7,8 +7,8 @@ import (
 	lru "github.com/hashicorp/golang-lru/v2"
 	"github.com/vocdoni/census3/db"
 	"github.com/vocdoni/census3/queue"
-	"github.com/vocdoni/census3/service"
-	"github.com/vocdoni/census3/service/web3"
+	"github.com/vocdoni/census3/service/providers"
+	"github.com/vocdoni/census3/service/providers/web3"
 	"go.vocdoni.io/dvote/api/censusdb"
 	storagelayer "go.vocdoni.io/dvote/data"
 	"go.vocdoni.io/dvote/data/downloader"
@@ -23,26 +23,26 @@ import (
 )
 
 type Census3APIConf struct {
-	Hostname      string
-	Port          int
-	DataDir       string
-	GroupKey      string
-	Web3Providers web3.NetworkEndpoints
-	ExtProviders  map[uint64]service.HolderProvider
-	AdminToken    string
+	Hostname        string
+	Port            int
+	DataDir         string
+	GroupKey        string
+	Web3Providers   web3.NetworkEndpoints
+	HolderProviders map[uint64]providers.HolderProvider
+	AdminToken      string
 }
 
 type census3API struct {
-	conf         Census3APIConf
-	db           *db.DB
-	endpoint     *api.API
-	censusDB     *censusdb.CensusDB
-	queue        *queue.BackgroundQueue
-	w3p          web3.NetworkEndpoints
-	storage      storagelayer.Storage
-	downloader   *downloader.Downloader
-	extProviders map[uint64]service.HolderProvider
-	cache        *lru.Cache[CacheKey, any]
+	conf            Census3APIConf
+	db              *db.DB
+	endpoint        *api.API
+	censusDB        *censusdb.CensusDB
+	queue           *queue.BackgroundQueue
+	w3p             web3.NetworkEndpoints
+	storage         storagelayer.Storage
+	downloader      *downloader.Downloader
+	holderProviders map[uint64]providers.HolderProvider
+	cache           *lru.Cache[CacheKey, any]
 }
 
 func Init(db *db.DB, conf Census3APIConf) (*census3API, error) {
@@ -51,12 +51,12 @@ func Init(db *db.DB, conf Census3APIConf) (*census3API, error) {
 		return nil, err
 	}
 	newAPI := &census3API{
-		conf:         conf,
-		db:           db,
-		w3p:          conf.Web3Providers,
-		queue:        queue.NewBackgroundQueue(),
-		extProviders: conf.ExtProviders,
-		cache:        cache,
+		conf:            conf,
+		db:              db,
+		w3p:             conf.Web3Providers,
+		queue:           queue.NewBackgroundQueue(),
+		holderProviders: conf.HolderProviders,
+		cache:           cache,
 	}
 	// get the current chainID
 	log.Infow("starting API", "web3Providers", conf.Web3Providers.String())

api/censuses.go

@@ -140,7 +140,7 @@ func (capi *census3API) createAndPublishCensus(req *CreateCensusRequest, qID str
 	}
 	// init some variables to get computed in the following steps
 	strategyHolders, censusWeight, totalTokensBlockNumber, err := CalculateStrategyHolders(
-		internalCtx, capi.db.QueriesRO, capi.w3p, req.StrategyID, strategy.Predicate)
+		internalCtx, capi.db.QueriesRO, capi.holderProviders, req.StrategyID, strategy.Predicate)
 	if err != nil {
 		return 0, ErrEvalStrategyPredicate.WithErr(err)
 	}

api/helpers.go

@@ -14,8 +14,8 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	queries "github.com/vocdoni/census3/db/sqlc"
 	"github.com/vocdoni/census3/lexer"
-	"github.com/vocdoni/census3/service/web3"
-	"github.com/vocdoni/census3/state"
+	"github.com/vocdoni/census3/service/providers"
+	"github.com/vocdoni/census3/service/providers/web3"
 	"github.com/vocdoni/census3/strategyoperators"
 	"go.vocdoni.io/dvote/api/censusdb"
 	"go.vocdoni.io/dvote/censustree"
@@ -208,8 +208,8 @@ func InnerCensusID(blockNumber, strategyID uint64, anonymous bool) uint64 {
 // The evaluator uses the strategy operators to evaluate the predicate which
 // uses the database queries to get the token holders and their balances, and
 // combines them.
-func CalculateStrategyHolders(ctx context.Context, qdb *queries.Queries, w3p web3.NetworkEndpoints,
-	id uint64, predicate string,
+func CalculateStrategyHolders(ctx context.Context, qdb *queries.Queries,
+	providers map[uint64]providers.HolderProvider, id uint64, predicate string,
 ) (map[common.Address]*big.Int, *big.Int, uint64, error) {
 	// TODO: write a benchmark and try to optimize this function
 
@@ -237,15 +237,20 @@ func CalculateStrategyHolders(ctx context.Context, qdb *queries.Queries, w3p web
 	// number, used to create the census id.
 	totalTokensBlockNumber := uint64(0)
 	for _, token := range strategyTokens {
-		w3endpoint, exists := w3p.EndpointByChainID(token.ChainID)
+		provider, exists := providers[token.TypeID]
 		if !exists {
-			return nil, nil, 0, fmt.Errorf("web3 endpoint not found for chain id %d", token.ChainID)
+			return nil, nil, 0, fmt.Errorf("provider not found for token type id %d", token.TypeID)
 		}
-		w3 := state.Web3{}
-		if err := w3.Init(ctx, w3endpoint, common.BytesToAddress(token.ID), state.TokenType(token.TypeID)); err != nil {
-			return nil, nil, 0, err
+		if !provider.IsExternal() {
+			if err := provider.SetRef(web3.Web3ProviderRef{
+				HexAddress: common.Bytes2Hex(token.ID),
+				ChainID:    token.ChainID,
+			}); err != nil {
+				return nil, nil, 0, err
+			}
 		}
-		currentBlockNumber, err := w3.LatestBlockNumber(ctx)
+
+		currentBlockNumber, err := provider.LatestBlockNumber(ctx, []byte(token.ExternalID))
 		if err != nil {
 			return nil, nil, 0, err
 		}

api/holders.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	queries "github.com/vocdoni/census3/db/sqlc"
-	"github.com/vocdoni/census3/service/web3"
+	"github.com/vocdoni/census3/service/providers/web3"
 	"go.vocdoni.io/dvote/httprouter"
 	api "go.vocdoni.io/dvote/httprouter/apirest"
 	"go.vocdoni.io/dvote/log"
@@ -106,25 +106,25 @@ func (capi *census3API) listHoldersAtLastBlock(address common.Address,
 		return nil, 0, ErrCantGetTokenHolders.WithErr(err)
 	}
 	// if the token is external, return an error
-	// TODO: implement external token holders
-	if _, isExternal := capi.extProviders[tokenData.TypeID]; isExternal {
-		return nil, 0, ErrCantCreateCensus.With("not implemented for external providers")
-	}
-	// get correct web3 uri provider
-	w3URI, exists := capi.w3p.EndpointByChainID(tokenData.ChainID)
+	provider, exists := capi.holderProviders[tokenData.TypeID]
 	if !exists {
-		return nil, 0, ErrChainIDNotSupported.With("chain ID not supported")
+		return nil, 0, ErrCantCreateCensus.With("token type not supported")
 	}
-	w3, err := w3URI.GetClient(web3.DefaultMaxWeb3ClientRetries)
-	if err != nil {
+	if provider.IsExternal() {
+		return nil, 0, ErrCantCreateCensus.With("not implemented for external providers")
+	}
+	if err := provider.SetRef(web3.Web3ProviderRef{
+		HexAddress: common.Bytes2Hex(tokenData.ID),
+		ChainID:    tokenData.ChainID,
+	}); err != nil {
 		return nil, 0, ErrInitializingWeb3.WithErr(err)
 	}
+
 	// get last block of the network
-	lastBlockNumber, err := w3.BlockNumber(internalCtx)
+	lastBlockNumber, err := provider.LatestBlockNumber(internalCtx, nil)
 	if err != nil {
 		return nil, 0, ErrCantGetLastBlockNumber.WithErr(err)
 	}
-	bLastBlockNumber := new(big.Int).SetUint64(lastBlockNumber)
 	// get holders balances at last block
 	balances := make(map[string]string)
 	for i, holder := range holders {
@@ -133,7 +133,7 @@ func (capi *census3API) listHoldersAtLastBlock(address common.Address,
 			"token", address.String(),
 			"progress", fmt.Sprintf("%d/%d", i+1, len(holders)))
 		holderAddress := common.BytesToAddress(holder.ID)
-		balance, err := w3.BalanceAt(internalCtx, holderAddress, bLastBlockNumber)
+		balance, err := provider.BalanceAt(internalCtx, holderAddress, nil, lastBlockNumber)
 		if err != nil {
 			return nil, lastBlockNumber, ErrCantGetTokenHolders.WithErr(err)
 		}

api/strategies.go

@@ -662,7 +662,7 @@ func (capi *census3API) estimateStrategySizeAndAccuracy(strategyID uint64, anony
 	}
 	// calculate the strategy holders
 	strategyHolders, _, _, err := CalculateStrategyHolders(internalCtx,
-		capi.db.QueriesRO, capi.w3p, strategyID, strategy.Predicate)
+		capi.db.QueriesRO, capi.holderProviders, strategyID, strategy.Predicate)
 	if err != nil {
 		return 0, 0, ErrEvalStrategyPredicate.WithErr(err)
 	}

api/tokens.go

@@ -15,8 +15,8 @@ import (
 	queries "github.com/vocdoni/census3/db/sqlc"
 	"github.com/vocdoni/census3/internal"
 	"github.com/vocdoni/census3/lexer"
-	"github.com/vocdoni/census3/service/web3"
-	"github.com/vocdoni/census3/state"
+	"github.com/vocdoni/census3/service/providers"
+	"github.com/vocdoni/census3/service/providers/web3"
 	"go.vocdoni.io/dvote/httprouter"
 	api "go.vocdoni.io/dvote/httprouter/apirest"
 	"go.vocdoni.io/dvote/log"
@@ -114,7 +114,7 @@ func (capi *census3API) getTokens(msg *api.APIdata, ctx *httprouter.HTTPContext)
 	for _, tokenData := range rows {
 		tokensResponse.Tokens = append(tokensResponse.Tokens, GetTokensItemResponse{
 			ID:              common.BytesToAddress(tokenData.ID).String(),
-			Type:            state.TokenType(int(tokenData.TypeID)).String(),
+			Type:            providers.TokenTypeStringMap[tokenData.TypeID],
 			Decimals:        tokenData.Decimals,
 			Name:            tokenData.Name,
 			StartBlock:      uint64(tokenData.CreationBlock),
@@ -224,24 +224,17 @@ func (capi *census3API) createToken(msg *api.APIdata, ctx *httprouter.HTTPContex
 	}
 	internalCtx, cancel := context.WithTimeout(ctx.Request.Context(), createTokenTimeout)
 	defer cancel()
-
-	endpoint, ok := capi.w3p.EndpointByChainID(req.ChainID)
-	if !ok {
-		return ErrChainIDNotSupported.With("chain ID not supported")
-	}
+	// get the correct holder provider for the token type
 	tokenType := web3.TokenTypeFromString(req.Type)
-	provider, isExternal := capi.extProviders[tokenType]
-	if !isExternal {
-		client, err := endpoint.GetClient(web3.DefaultMaxWeb3ClientRetries)
-		if err != nil {
-			return ErrInitializingWeb3.WithErr(err)
-		}
-		provider = &web3.ERC20HolderProvider{
+	provider, exists := capi.holderProviders[tokenType]
+	if !exists {
+		return ErrCantCreateCensus.With("token type not supported")
+	}
+	if !provider.IsExternal() {
+		if err := provider.SetRef(web3.Web3ProviderRef{
 			HexAddress: req.ID,
 			ChainID:    req.ChainID,
-			Client:     client,
-		}
-		if err := provider.Init(); err != nil {
+		}); err != nil {
 			return ErrInitializingWeb3.WithErr(err)
 		}
 	}
@@ -487,18 +480,20 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext)
 	// calculate the current scan progress
 	tokenProgress := 100
 	if !tokenData.Synced {
-		// get correct web3 uri provider
-		w3URI, exists := capi.w3p.EndpointByChainID(tokenData.ChainID)
+		provider, exists := capi.holderProviders[tokenData.TypeID]
 		if !exists {
-			return ErrChainIDNotSupported.With("chain ID not supported")
+			return ErrCantCreateCensus.With("token type not supported")
 		}
-		// get last block of the network, if something fails return progress 0
-		w3 := state.Web3{}
-		if err := w3.Init(internalCtx, w3URI, address, state.TokenType(tokenData.TypeID)); err != nil {
-			return ErrInitializingWeb3.WithErr(err)
+		if !provider.IsExternal() {
+			if err := provider.SetRef(web3.Web3ProviderRef{
+				HexAddress: common.Bytes2Hex(tokenData.ID),
+				ChainID:    tokenData.ChainID,
+			}); err != nil {
+				return ErrInitializingWeb3.WithErr(err)
+			}
 		}
 		// fetch the last block header and calculate progress
-		lastBlockNumber, err := w3.LatestBlockNumber(internalCtx)
+		lastBlockNumber, err := provider.LatestBlockNumber(internalCtx, []byte(tokenData.ExternalID))
 		if err != nil {
 			return ErrCantGetLastBlockNumber.WithErr(err)
 		}
@@ -518,7 +513,7 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext)
 	// build response
 	tokenResponse := GetTokenResponse{
 		ID:          address.String(),
-		Type:        state.TokenType(int(tokenData.TypeID)).String(),
+		Type:        providers.TokenTypeStringMap[tokenData.TypeID],
 		Decimals:    tokenData.Decimals,
 		Size:        uint64(holders),
 		Name:        tokenData.Name,
@@ -530,8 +525,7 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext)
 			Synced:   tokenData.Synced,
 			Progress: tokenProgress,
 		},
-		Tags: tokenData.Tags,
-		// TODO: Only for the MVP, consider to remove it
+		Tags:            tokenData.Tags,
 		DefaultStrategy: tokenData.DefaultStrategy,
 		ChainID:         tokenData.ChainID,
 		ChainAddress:    tokenData.ChainAddress,
@@ -593,7 +587,7 @@ func (capi *census3API) isTokenHolder(msg *api.APIdata, ctx *httprouter.HTTPCont
 // supported types of token contracts.
 func (capi *census3API) getTokenTypes(msg *api.APIdata, ctx *httprouter.HTTPContext) error {
 	supportedTypes := []string{}
-	for _, supportedType := range state.TokenTypeStringMap {
+	for _, supportedType := range providers.TokenTypeStringMap {
 		supportedTypes = append(supportedTypes, supportedType)
 	}
 	res, err := json.Marshal(TokenTypesResponse{supportedTypes})

cmd/census3/main.go

@@ -17,8 +17,9 @@ import (
 	"github.com/vocdoni/census3/db"
 	"github.com/vocdoni/census3/internal"
 	"github.com/vocdoni/census3/service"
-	"github.com/vocdoni/census3/service/poap"
-	"github.com/vocdoni/census3/service/web3"
+	"github.com/vocdoni/census3/service/providers"
+	"github.com/vocdoni/census3/service/providers/poap"
+	"github.com/vocdoni/census3/service/providers/web3"
 	"go.vocdoni.io/dvote/log"
 )
 
@@ -116,19 +117,25 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+	// init the ERC20 token providers
+	erc20Provider := new(web3.ERC20HolderProvider)
+	if err := erc20Provider.Init(web3.Web3ProviderConfig{Endpoints: w3p}); err != nil {
+		log.Fatal(err)
+	}
 	// init POAP external provider
-	poapProvider := &poap.POAPHolderProvider{
+	poapProvider := new(poap.POAPHolderProvider)
+	if err := poapProvider.Init(poap.POAPConfig{
 		URI:         config.poapAPIEndpoint,
 		AccessToken: config.poapAuthToken,
-	}
-	if err := poapProvider.Init(); err != nil {
+	}); err != nil {
 		log.Fatal(err)
 	}
-	// start the holder scanner with the database and the external providers
-	externalProviders := map[uint64]service.HolderProvider{
-		web3.CONTRACT_TYPE_POAP: poapProvider,
+	holderProviders := map[uint64]providers.HolderProvider{
+		providers.CONTRACT_TYPE_ERC20: erc20Provider,
+		providers.CONTRACT_TYPE_POAP:  poapProvider,
 	}
-	hc, err := service.NewHoldersScanner(database, w3p, externalProviders, config.scannerCoolDown)
+	// start the holder scanner with the database
+	hc, err := service.NewHoldersScanner(database, w3p, holderProviders, config.scannerCoolDown)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -143,13 +150,13 @@ func main() {
 	}
 	// Start the API
 	apiService, err := api.Init(database, api.Census3APIConf{
-		Hostname:      "0.0.0.0",
-		Port:          config.port,
-		DataDir:       config.dataDir,
-		Web3Providers: w3p,
-		GroupKey:      config.connectKey,
-		ExtProviders:  externalProviders,
-		AdminToken:    config.adminToken,
+		Hostname:        "0.0.0.0",
+		Port:            config.port,
+		DataDir:         config.dataDir,
+		Web3Providers:   w3p,
+		GroupKey:        config.connectKey,
+		HolderProviders: holderProviders,
+		AdminToken:      config.adminToken,
 	})
 	if err != nil {
 		log.Fatal(err)
@@ -175,7 +182,7 @@ func main() {
 		if err := database.Close(); err != nil {
 			log.Fatal(err)
 		}
-		for _, provider := range externalProviders {
+		for _, provider := range holderProviders {
 			if err := provider.Close(); err != nil {
 				log.Fatal(err)
 			}