From 74645c815ff4586c8c4cbfcc62b677d16d68aecd Mon Sep 17 00:00:00 2001 From: p4u Date: Wed, 19 Jun 2024 23:14:46 +0200 Subject: [PATCH] test Signed-off-by: p4u --- census.go | 33 ++++++++++++++++++++++++--------- helpers/helpers.go | 18 ++++++++++++++++++ mongo/mongo.go | 7 +++++-- mongo/user.go | 27 ++++++++++++++++++++++++++- 4 files changed, 73 insertions(+), 12 deletions(-) diff --git a/census.go b/census.go index a679a2b7..c4f78970 100644 --- a/census.go +++ b/census.go @@ -1196,6 +1196,9 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan if weightRecord == "" { weightRecord = "1" } + if weightRecord == "0" { + log.Warnf("address %s has weight %s", address, weightRecord) + } weight, ok = new(big.Int).SetString(weightRecord, 10) if !ok { log.Warnf("invalid weight for address %s: %s", address, weightRecord) @@ -1214,6 +1217,11 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan return nil, 0, ErrNoValidParticipants } + // Print addressMap (TEST) + for address, weight := range addressMap { + log.Debugf("address: %s, weight: %s", address, weight.String()) + } + // Fetch the users from the database concurrently var wg sync.WaitGroup participantsCh := make(chan *FarcasterParticipant) // Channel to collect participants @@ -1284,6 +1292,7 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan if !errors.Is(err, mongo.ErrUserUnknown) { log.Warnw("error fetching user from database", "address", addr, "error", err) } else { + log.Warnw("user not found on database", "address", addr) pendingAddressesCh <- addr } return @@ -1297,12 +1306,14 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan // the weight is the sum of the weights of all the addresses of the user weight := new(big.Int).SetUint64(0) for _, addr := range user.Addresses { - weightAddress, ok := addressMap[common.HexToAddress(addr).Hex()] + weightAddress, ok := addressMap[helpers.NormalizeAddressString(addr)] if ok { weight = weight.Add(weight, weightAddress) } } - + if weight.Cmp(big.NewInt(0)) == 0 { + log.Warnw("user address not found on addressMap ", "address", addr) + } for _, signer := range user.Signers { signerBytes, err := hex.DecodeString(strings.TrimPrefix(signer, "0x")) if err != nil { @@ -1327,7 +1338,6 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan // Fetch the remaining users from the farcaster API count := 0 - log.Debugw("fetching users from farcaster", "count", len(pendingAddresses)) for i := 0; i < len(pendingAddresses); i += neynar.MaxAddressesPerRequest { // Fetch the user data from the farcaster API ctx2, cancel := context.WithTimeout(ctx, 10*time.Second) @@ -1336,7 +1346,7 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan if to > len(pendingAddresses) { to = len(pendingAddresses) } - log.Debugw("fetching users from farcaster", "from", i, "to", to) + log.Debugw("fetching users from neynar", "from", i, "to", to, "total", len(pendingAddresses)) usersData, err := v.fcapi.UserDataByVerificationAddress(ctx2, pendingAddresses[i:to]) if err != nil { if errors.Is(err, farcasterapi.ErrNoDataFound) { @@ -1344,6 +1354,7 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan } log.Errorw(err, "error fetching users from Neynar API") } + log.Debugw("users found on neynar", "count", len(usersData)) for _, userData := range usersData { // Add or update the user on the database dbUser, err := v.db.User(userData.FID) @@ -1353,32 +1364,36 @@ func (v *vocdoniHandler) processCensusRecords(records [][]string, progress chan userData.FID, userData.Username, userData.Displayname, - userData.VerificationsAddresses, + helpers.NormalizeAddressStringSlice(userData.VerificationsAddresses), userData.Signers, - userData.CustodyAddress, + helpers.NormalizeAddressString(userData.CustodyAddress), 0, ); err != nil { return nil, 0, err } } else { log.Debugw("updating user on database", "fid", userData.FID) - dbUser.Addresses = userData.VerificationsAddresses + dbUser.Addresses = helpers.NormalizeAddressStringSlice(userData.VerificationsAddresses) dbUser.Username = userData.Username dbUser.Signers = userData.Signers - dbUser.CustodyAddress = userData.CustodyAddress + dbUser.CustodyAddress = helpers.NormalizeAddressString(userData.CustodyAddress) if err := v.db.UpdateUser(dbUser); err != nil { return nil, 0, err } } + // find the addres on the map to get the weight // the weight is the sum of the weights of all the addresses of the user weight := new(big.Int).SetUint64(0) for _, addr := range userData.VerificationsAddresses { - weightAddress, ok := addressMap[common.HexToAddress(addr).Hex()] + weightAddress, ok := addressMap[helpers.NormalizeAddressString(addr)] if ok { weight = weight.Add(weight, weightAddress) } } + if weight.Cmp(big.NewInt(0)) == 0 { + log.Warnw("user address not found on addressMap ", "address", userData.VerificationsAddresses) + } // Add the user to the participants list (with all the signers) for _, signer := range userData.Signers { diff --git a/helpers/helpers.go b/helpers/helpers.go index 482b6f84..4c797126 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -4,6 +4,7 @@ import ( "encoding/json" "math/big" + "github.com/ethereum/go-ethereum/common" "go.vocdoni.io/dvote/api" "go.vocdoni.io/dvote/log" ) @@ -127,3 +128,20 @@ func UnpackMetadata(metadata any) *api.ElectionDescription { } return desc } + +// NormalizeAddressString converts an Ethereum address to its normalized form. +func NormalizeAddressString(address string) string { + return common.HexToAddress(address).Hex() +} + +// NormalizeAddressStringSlice converts a slice of Ethereum addresses to their normalized form. +func NormalizeAddressStringSlice(addresses []string) []string { + normalizedAddresses := make([]string, 0, len(addresses)) + for _, address := range addresses { + normalizedAddress := NormalizeAddressString(address) + if normalizedAddress != "" { + normalizedAddresses = append(normalizedAddresses, normalizedAddress) + } + } + return normalizedAddresses +} diff --git a/mongo/mongo.go b/mongo/mongo.go index dec7a43b..5af614f8 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -129,11 +129,14 @@ func (ms *MongoStorage) createIndexes() error { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - // Index model for the 'addresses' field + // Create an index for the 'addresses' field on users addressesIndexModel := mongo.IndexModel{ Keys: bson.D{{Key: "addresses", Value: 1}}, // 1 for ascending order Options: nil, } + if _, err := ms.users.Indexes().CreateOne(ctx, addressesIndexModel); err != nil { + return fmt.Errorf("failed to create index on addresses for users: %w", err) + } // Index model for the 'signers' field signersIndexModel := mongo.IndexModel{ @@ -142,7 +145,7 @@ func (ms *MongoStorage) createIndexes() error { } // Create both indexes - _, err := ms.users.Indexes().CreateMany(ctx, []mongo.IndexModel{addressesIndexModel, signersIndexModel}) + _, err := ms.users.Indexes().CreateOne(ctx, signersIndexModel) if err != nil { return err } diff --git a/mongo/user.go b/mongo/user.go index b27db0be..5a32ecae 100644 --- a/mongo/user.go +++ b/mongo/user.go @@ -3,9 +3,12 @@ package mongo import ( "context" "fmt" + "regexp" "time" + "github.com/vocdoni/vote-frame/helpers" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.vocdoni.io/dvote/log" ) @@ -170,7 +173,29 @@ func (ms *MongoStorage) UserByAddress(address string) (*User, error) { defer cancel() var userByAddress User if err := ms.users.FindOne(ctx, bson.M{ - "addresses": bson.M{"$in": []string{address}}, + "addresses": bson.M{ + "$regex": "^" + regexp.QuoteMeta(address) + "$", + "$options": "i", + }, + }).Decode(&userByAddress); err != nil { + if err == mongo.ErrNoDocuments { + return nil, ErrUserUnknown + } + return nil, err + } + return &userByAddress, nil +} + +// UserByAddress returns the user that has the given address. If the user is not found, it returns an error. +func (ms *MongoStorage) UserByAddress1(address string) (*User, error) { + ms.keysLock.RLock() + defer ms.keysLock.RUnlock() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var userByAddress User + if err := ms.users.FindOne(ctx, bson.M{ + "addresses": bson.M{"$in": []string{helpers.NormalizeAddressString(address)}}, }).Decode(&userByAddress); err != nil { return nil, ErrUserUnknown }