Skip to content

Commit 3c47de6

Browse files
mr-tronmr-tron
authored
Merge pull request #594 from tonkeeper/spam_batch
get spam data in batch
2 parents d94f82b + 8c57c9b commit 3c47de6

10 files changed

+193
-33
lines changed

‎pkg/api/account_handlers.go

+24-2
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,11 @@ import (
88
"encoding/json"
99
"errors"
1010
"fmt"
11+
"go.uber.org/zap"
1112
"net/http"
1213
"sort"
1314
"strings"
15+
"sync"
1416
"time"
1517

1618
"github.com/tonkeeper/opentonapi/pkg/addressbook"
@@ -330,10 +332,22 @@ func (h *Handler) GetAccountDnsExpiring(ctx context.Context, params oas.GetAccou
330332
accounts = append(accounts, dns.DnsItem.Address)
331333
}
332334
}
335+
var wg sync.WaitGroup
336+
wg.Add(1)
337+
var nftsScamData map[ton.AccountID]core.TrustType
338+
go func() {
339+
defer wg.Done()
340+
nftsScamData, err = h.spamFilter.GetNftsScamData(ctx, accounts)
341+
if err != nil {
342+
h.logger.Warn("error getting nft scam data", zap.Error(err))
343+
}
344+
}()
333345
nfts, err := h.storage.GetNFTs(ctx, accounts)
346+
wg.Wait()
334347
if err != nil {
335348
return nil, toError(http.StatusInternalServerError, err)
336349
}
350+
337351
for _, dns := range dnsExpiring {
338352
dei := oas.DnsExpiringItemsItem{
339353
Name: dns.Name,
@@ -342,7 +356,7 @@ func (h *Handler) GetAccountDnsExpiring(ctx context.Context, params oas.GetAccou
342356
if dns.DnsItem != nil {
343357
for _, n := range nfts {
344358
if n.Address == dns.DnsItem.Address {
345-
dei.DNSItem = oas.NewOptNftItem(h.convertNFT(ctx, n, h.addressBook, h.metaCache))
359+
dei.DNSItem = oas.NewOptNftItem(h.convertNFT(ctx, n, h.addressBook, h.metaCache, nftsScamData[n.Address]))
346360
break
347361
}
348362
}
@@ -454,7 +468,15 @@ func (h *Handler) GetAccountNftHistory(ctx context.Context, params oas.GetAccoun
454468
if err != nil {
455469
return nil, toError(http.StatusInternalServerError, err)
456470
}
457-
events, lastLT, err := h.convertNftHistory(ctx, account.ID, traceIDs, params.AcceptLanguage)
471+
var eventIDs []string
472+
for _, traceID := range traceIDs {
473+
eventIDs = append(eventIDs, traceID.Hex())
474+
}
475+
isBannedTraces, err := h.spamFilter.GetEventsScamData(ctx, eventIDs)
476+
if err != nil {
477+
h.logger.Warn("error getting events spam data", zap.Error(err))
478+
}
479+
events, lastLT, err := h.convertNftHistory(ctx, account.ID, traceIDs, isBannedTraces, params.AcceptLanguage)
458480
if err != nil {
459481
return nil, toError(http.StatusInternalServerError, err)
460482
}

‎pkg/api/dns_handlers.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"errors"
66
"fmt"
7+
"go.uber.org/zap"
78
"net/http"
89
"strings"
910

@@ -149,7 +150,11 @@ func (h *Handler) GetDnsInfo(ctx context.Context, params oas.GetDnsInfoParams) (
149150
convertedDomainInfo := oas.DomainInfo{
150151
Name: params.DomainName,
151152
}
152-
convertedDomainInfo.Item.SetTo(h.convertNFT(ctx, nft, h.addressBook, h.metaCache))
153+
nftScamData, err := h.spamFilter.GetNftsScamData(ctx, []ton.AccountID{nft.Address})
154+
if err != nil {
155+
h.logger.Warn("error getting nft scam data", zap.Error(err))
156+
}
157+
convertedDomainInfo.Item.SetTo(h.convertNFT(ctx, nft, h.addressBook, h.metaCache, nftScamData[nft.Address]))
153158
if expTime != 0 {
154159
convertedDomainInfo.ExpiringAt.SetTo(expTime)
155160
}

‎pkg/api/event_converters.go

+19-5
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ import (
44
"context"
55
"encoding/hex"
66
"fmt"
7+
"go.uber.org/zap"
78
"math/big"
89
"sort"
910
"strings"
11+
"sync"
1012

1113
"github.com/tonkeeper/tongo/ton"
1214

@@ -84,12 +86,24 @@ func (h *Handler) convertRisk(ctx context.Context, risk wallet.Risk, walletAddre
8486
oasRisk.Jettons = append(oasRisk.Jettons, jettonQuantity)
8587
}
8688
if len(risk.Nfts) > 0 {
89+
var wg sync.WaitGroup
90+
wg.Add(1)
91+
var nftsScamData map[ton.AccountID]core.TrustType
92+
var err error
93+
go func() {
94+
defer wg.Done()
95+
nftsScamData, err = h.spamFilter.GetNftsScamData(ctx, risk.Nfts)
96+
if err != nil {
97+
h.logger.Warn("error getting nft scam data", zap.Error(err))
98+
}
99+
}()
87100
items, err := h.storage.GetNFTs(ctx, risk.Nfts)
101+
wg.Wait()
88102
if err != nil {
89103
return oas.Risk{}, err
90104
}
91105
for _, item := range items {
92-
nft := h.convertNFT(ctx, item, h.addressBook, h.metaCache)
106+
nft := h.convertNFT(ctx, item, h.addressBook, h.metaCache, nftsScamData[item.Address])
93107
oasRisk.Nfts = append(oasRisk.Nfts, nft)
94108
}
95109
}
@@ -562,7 +576,7 @@ func (h *Handler) convertAction(ctx context.Context, viewer *tongo.AccountID, a
562576
var name string
563577
if len(items) == 1 {
564578
// opentonapi doesn't implement GetNFTs() now
565-
nft = h.convertNFT(ctx, items[0], h.addressBook, h.metaCache)
579+
nft = h.convertNFT(ctx, items[0], h.addressBook, h.metaCache, "")
566580
if len(nft.Previews) > 0 {
567581
nftImage = nft.Previews[0].URL
568582
}
@@ -700,7 +714,7 @@ func (h *Handler) convertAction(ctx context.Context, viewer *tongo.AccountID, a
700714
if a.AuctionBid.Nft == nil {
701715
return oas.Action{}, fmt.Errorf("nft is nil")
702716
}
703-
nft.SetTo(h.convertNFT(ctx, *a.AuctionBid.Nft, h.addressBook, h.metaCache))
717+
nft.SetTo(h.convertNFT(ctx, *a.AuctionBid.Nft, h.addressBook, h.metaCache, ""))
704718
action.AuctionBid.SetTo(oas.AuctionBidAction{
705719
Amount: oas.Price{
706720
Value: fmt.Sprintf("%v", a.AuctionBid.Amount),
@@ -805,7 +819,7 @@ func (h *Handler) toEvent(ctx context.Context, trace *core.Trace, result *bath.A
805819
}
806820
event.Actions[i] = convertedAction
807821
}
808-
event.IsScam = h.spamFilter.IsScamEvent(event.EventID, event.Actions, nil, trace.Account)
822+
event.IsScam = h.spamFilter.IsScamEvent(event.Actions, nil, trace.Account, false)
809823
previews := make(map[tongo.AccountID]oas.JettonPreview)
810824
for _, flow := range result.ValueFlow.Accounts {
811825
for jettonMaster := range flow.Jettons {
@@ -886,7 +900,7 @@ func (h *Handler) toAccountEvent(ctx context.Context, account tongo.AccountID, t
886900
e.Actions = append(e.Actions, convertedAction)
887901
}
888902
if h.spamFilter != nil {
889-
e.IsScam = h.spamFilter.IsScamEvent(e.EventID, e.Actions, &account, trace.Account)
903+
e.IsScam = h.spamFilter.IsScamEvent(e.Actions, &account, trace.Account, false)
890904
}
891905
if len(e.Actions) == 0 {
892906
e.Actions = []oas.Action{

‎pkg/api/event_handlers.go

+41-8
Original file line numberDiff line numberDiff line change
@@ -7,26 +7,27 @@ import (
77
"encoding/hex"
88
"errors"
99
"fmt"
10+
"go.uber.org/zap"
11+
"golang.org/x/exp/slices"
1012
"net/http"
1113
"strings"
14+
"sync"
1215
"time"
1316

1417
"github.com/prometheus/client_golang/prometheus"
1518
"github.com/prometheus/client_golang/prometheus/promauto"
16-
"github.com/tonkeeper/opentonapi/pkg/blockchain"
17-
"github.com/tonkeeper/tongo"
18-
"github.com/tonkeeper/tongo/tlb"
19-
"github.com/tonkeeper/tongo/ton"
20-
"github.com/tonkeeper/tongo/tontest"
21-
"github.com/tonkeeper/tongo/txemulator"
22-
"golang.org/x/exp/slices"
23-
2419
"github.com/tonkeeper/opentonapi/pkg/bath"
20+
"github.com/tonkeeper/opentonapi/pkg/blockchain"
2521
"github.com/tonkeeper/opentonapi/pkg/cache"
2622
"github.com/tonkeeper/opentonapi/pkg/core"
2723
"github.com/tonkeeper/opentonapi/pkg/oas"
2824
"github.com/tonkeeper/opentonapi/pkg/sentry"
2925
"github.com/tonkeeper/opentonapi/pkg/wallet"
26+
"github.com/tonkeeper/tongo"
27+
"github.com/tonkeeper/tongo/tlb"
28+
"github.com/tonkeeper/tongo/ton"
29+
"github.com/tonkeeper/tongo/tontest"
30+
"github.com/tonkeeper/tongo/txemulator"
3031
)
3132

3233
var (
@@ -190,6 +191,11 @@ func (h *Handler) GetEvent(ctx context.Context, params oas.GetEventParams) (*oas
190191
if err != nil {
191192
return nil, toError(http.StatusInternalServerError, err)
192193
}
194+
isBannedTraces, err := h.spamFilter.GetEventsScamData(ctx, []string{traceID.Hex()})
195+
if err != nil {
196+
h.logger.Warn("error getting events spam data", zap.Error(err))
197+
}
198+
event.IsScam = event.IsScam || isBannedTraces[traceID.Hex()]
193199
if emulated {
194200
event.InProgress = true
195201
}
@@ -217,6 +223,22 @@ func (h *Handler) GetAccountEvents(ctx context.Context, params oas.GetAccountEve
217223

218224
events := make([]oas.AccountEvent, 0, len(traceIDs))
219225

226+
var wg sync.WaitGroup
227+
wg.Add(1)
228+
229+
var isBannedTraces map[string]bool
230+
go func() {
231+
defer wg.Done()
232+
var eventIDs []string
233+
for _, traceID := range traceIDs {
234+
eventIDs = append(eventIDs, traceID.Hash.Hex())
235+
}
236+
isBannedTraces, err = h.spamFilter.GetEventsScamData(ctx, eventIDs)
237+
if err != nil {
238+
h.logger.Warn("error getting events spam data", zap.Error(err))
239+
}
240+
}()
241+
220242
var lastLT uint64
221243
for _, traceID := range traceIDs {
222244
lastLT = traceID.Lt
@@ -276,6 +298,9 @@ func (h *Handler) GetAccountEvents(ctx context.Context, params oas.GetAccountEve
276298
lastLT = uint64(events[len(events)-1].Lt)
277299
}
278300
}
301+
302+
wg.Wait()
303+
279304
if len(events) < params.Limit {
280305
lastLT = 0 // dirty hack
281306

@@ -294,6 +319,9 @@ func (h *Handler) GetAccountEvents(ctx context.Context, params oas.GetAccountEve
294319
}
295320
}
296321
}
322+
for i := range events {
323+
events[i].IsScam = events[i].IsScam || isBannedTraces[events[i].EventID]
324+
}
297325
return &oas.AccountEvents{Events: events, NextFrom: int64(lastLT)}, nil
298326
}
299327

@@ -325,6 +353,11 @@ func (h *Handler) GetAccountEvent(ctx context.Context, params oas.GetAccountEven
325353
if err != nil {
326354
return nil, toError(http.StatusInternalServerError, err)
327355
}
356+
isBannedTraces, err := h.spamFilter.GetEventsScamData(ctx, []string{traceID.Hex()})
357+
if err != nil {
358+
h.logger.Warn("error getting events spam data", zap.Error(err))
359+
}
360+
event.IsScam = event.IsScam || isBannedTraces[traceID.Hex()]
328361
if emulated {
329362
event.InProgress = true
330363
}

‎pkg/api/interfaces.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -184,10 +184,12 @@ type scoreSource interface {
184184
}
185185

186186
type SpamFilter interface {
187-
IsScamEvent(eventId string, actions []oas.Action, viewer *ton.AccountID, initiator ton.AccountID) bool
187+
IsScamEvent(actions []oas.Action, viewer *ton.AccountID, initiator ton.AccountID, markedAsScam bool) bool
188+
GetEventsScamData(ctx context.Context, ids []string) (map[string]bool, error)
188189
JettonTrust(address tongo.AccountID, symbol, name, image string) core.TrustType
189190
AccountTrust(address tongo.AccountID) core.TrustType
190-
NftTrust(ctx context.Context, address tongo.AccountID, collection *ton.AccountID, description, image string) core.TrustType
191+
NftTrust(address tongo.AccountID, collection *ton.AccountID, description, image string) core.TrustType
192+
GetNftsScamData(ctx context.Context, addresses []ton.AccountID) (map[ton.AccountID]core.TrustType, error)
191193
}
192194

193195
type verifierSource interface {

‎pkg/api/jetton_converters.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func jettonMetadata(account ton.AccountID, meta NormalizedMetadata) oas.JettonMe
5252
return metadata
5353
}
5454

55-
func (h *Handler) convertJettonHistory(ctx context.Context, account ton.AccountID, master *ton.AccountID, traceIDs []ton.Bits256, acceptLanguage oas.OptString) ([]oas.AccountEvent, int64, error) {
55+
func (h *Handler) convertJettonHistory(ctx context.Context, account ton.AccountID, master *ton.AccountID, traceIDs []ton.Bits256, isBannedTraces map[string]bool, acceptLanguage oas.OptString) ([]oas.AccountEvent, int64, error) {
5656
var lastLT uint64
5757
events := make([]oas.AccountEvent, 0, len(traceIDs))
5858
for _, traceID := range traceIDs {
@@ -97,7 +97,7 @@ func (h *Handler) convertJettonHistory(ctx context.Context, account ton.AccountI
9797
}
9898
event.Actions = append(event.Actions, convertedAction)
9999
}
100-
event.IsScam = h.spamFilter.IsScamEvent(event.EventID, event.Actions, &account, trace.Account)
100+
event.IsScam = h.spamFilter.IsScamEvent(event.Actions, &account, trace.Account, isBannedTraces[event.EventID])
101101
if len(event.Actions) == 0 {
102102
continue
103103
}

‎pkg/api/jetton_handlers.go

+24-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"errors"
66
"fmt"
7+
"go.uber.org/zap"
78
"net/http"
89
"slices"
910
"strings"
@@ -95,7 +96,15 @@ func (h *Handler) GetAccountJettonsHistory(ctx context.Context, params oas.GetAc
9596
if err != nil {
9697
return nil, toError(http.StatusInternalServerError, err)
9798
}
98-
events, lastLT, err := h.convertJettonHistory(ctx, account.ID, nil, traceIDs, params.AcceptLanguage)
99+
var eventIDs []string
100+
for _, traceID := range traceIDs {
101+
eventIDs = append(eventIDs, traceID.Hex())
102+
}
103+
isBannedTraces, err := h.spamFilter.GetEventsScamData(ctx, eventIDs)
104+
if err != nil {
105+
h.logger.Warn("error getting events spam data", zap.Error(err))
106+
}
107+
events, lastLT, err := h.convertJettonHistory(ctx, account.ID, nil, traceIDs, isBannedTraces, params.AcceptLanguage)
99108
if err != nil {
100109
return nil, toError(http.StatusInternalServerError, err)
101110
}
@@ -118,7 +127,15 @@ func (h *Handler) GetAccountJettonHistoryByID(ctx context.Context, params oas.Ge
118127
if err != nil {
119128
return nil, toError(http.StatusInternalServerError, err)
120129
}
121-
events, lastLT, err := h.convertJettonHistory(ctx, account.ID, &jettonMasterAccount.ID, traceIDs, params.AcceptLanguage)
130+
var eventIDs []string
131+
for _, traceID := range traceIDs {
132+
eventIDs = append(eventIDs, traceID.Hex())
133+
}
134+
isBannedTraces, err := h.spamFilter.GetEventsScamData(ctx, eventIDs)
135+
if err != nil {
136+
h.logger.Warn("error getting events spam data", zap.Error(err))
137+
}
138+
events, lastLT, err := h.convertJettonHistory(ctx, account.ID, &jettonMasterAccount.ID, traceIDs, isBannedTraces, params.AcceptLanguage)
122139
if err != nil {
123140
return nil, toError(http.StatusInternalServerError, err)
124141
}
@@ -225,6 +242,11 @@ func (h *Handler) GetJettonsEvents(ctx context.Context, params oas.GetJettonsEve
225242
if err != nil {
226243
return nil, toError(http.StatusInternalServerError, err)
227244
}
245+
isBannedTraces, err := h.spamFilter.GetEventsScamData(ctx, []string{traceID.Hex()})
246+
if err != nil {
247+
h.logger.Warn("error getting events spam data", zap.Error(err))
248+
}
249+
response.IsScam = response.IsScam || isBannedTraces[traceID.Hex()]
228250
return &response, nil
229251
}
230252

‎pkg/api/nft_converters.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import (
1717
"github.com/tonkeeper/opentonapi/pkg/references"
1818
)
1919

20-
func (h *Handler) convertNFT(ctx context.Context, item core.NftItem, book addressBook, metaCache metadataCache) oas.NftItem {
20+
func (h *Handler) convertNFT(ctx context.Context, item core.NftItem, book addressBook, metaCache metadataCache, trustType core.TrustType) oas.NftItem {
2121
nftItem := oas.NftItem{
2222
Address: item.Address.ToRaw(),
2323
Index: item.Index.BigInt().Int64(),
@@ -77,7 +77,11 @@ func (h *Handler) convertNFT(ctx context.Context, item core.NftItem, book addres
7777
if len(nftItem.ApprovedBy) > 0 && nftItem.Verified {
7878
nftItem.Trust = oas.TrustType(core.TrustWhitelist)
7979
} else {
80-
nftItem.Trust = oas.TrustType(h.spamFilter.NftTrust(ctx, item.Address, item.CollectionAddress, description, image))
80+
nftTrust := h.spamFilter.NftTrust(item.Address, item.CollectionAddress, description, image)
81+
if nftTrust == core.TrustNone && trustType != "" {
82+
nftTrust = trustType
83+
}
84+
nftItem.Trust = oas.TrustType(nftTrust)
8185
}
8286
if image == "" {
8387
image = references.Placeholder
@@ -129,7 +133,7 @@ func convertNftCollection(collection core.NftCollection, book addressBook) oas.N
129133
return nftCollection
130134
}
131135

132-
func (h *Handler) convertNftHistory(ctx context.Context, account tongo.AccountID, traceIDs []tongo.Bits256, acceptLanguage oas.OptString) ([]oas.AccountEvent, int64, error) {
136+
func (h *Handler) convertNftHistory(ctx context.Context, account tongo.AccountID, traceIDs []tongo.Bits256, isBannedTraces map[string]bool, acceptLanguage oas.OptString) ([]oas.AccountEvent, int64, error) {
133137
var lastLT uint64
134138
events := make([]oas.AccountEvent, 0, len(traceIDs))
135139
for _, traceID := range traceIDs {
@@ -163,7 +167,7 @@ func (h *Handler) convertNftHistory(ctx context.Context, account tongo.AccountID
163167
}
164168
event.Actions = append(event.Actions, convertedAction)
165169
}
166-
event.IsScam = h.spamFilter.IsScamEvent(event.EventID, event.Actions, &account, trace.Account)
170+
event.IsScam = h.spamFilter.IsScamEvent(event.Actions, &account, trace.Account, isBannedTraces[event.EventID])
167171
if len(event.Actions) > 0 {
168172
events = append(events, event)
169173
lastLT = trace.Lt

0 commit comments

Comments
 (0)