diff --git a/pkg/scraper/sources/anubis/anubis.go b/pkg/scraper/sources/anubis/anubis.go index fc79cdf..b75bab0 100644 --- a/pkg/scraper/sources/anubis/anubis.go +++ b/pkg/scraper/sources/anubis/anubis.go @@ -33,8 +33,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result - getSubdomainsRes.Body.Close() - return } @@ -56,9 +54,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getSubdomainsRes.Body.Close() - for index := range getSubdomainsResData { - subdomain := getSubdomainsResData[index] - + for _, subdomain := range getSubdomainsResData { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/scraper/sources/bevigil/bevigil.go b/pkg/scraper/sources/bevigil/bevigil.go index bf32884..365f852 100644 --- a/pkg/scraper/sources/bevigil/bevigil.go +++ b/pkg/scraper/sources/bevigil/bevigil.go @@ -59,8 +59,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - getSubdomainsRes.Body.Close() - return } @@ -82,9 +80,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getSubdomainsRes.Body.Close() - for index := range getSubdomainsResData.Subdomains { - subdomain := getSubdomainsResData.Subdomains[index] - + for _, subdomain := range getSubdomainsResData.Subdomains { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/scraper/sources/bufferover/bufferover.go b/pkg/scraper/sources/bufferover/bufferover.go index 2f85853..2b6af33 100644 --- a/pkg/scraper/sources/bufferover/bufferover.go +++ b/pkg/scraper/sources/bufferover/bufferover.go @@ -62,8 +62,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - getTLSLogsSearchRes.Body.Close() - return } @@ -119,13 +117,10 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s entries = getTLSLogsSearchResData.Results } - for index := range entries { - entry := entries[index] + for _, entry := range entries { subdomains := regex.FindAllString(entry, -1) - for index := range subdomains { - subdomain := subdomains[index] - + for _, subdomain := range subdomains { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/scraper/sources/builtwith/builtwith.go b/pkg/scraper/sources/builtwith/builtwith.go index c75aa34..fe6673f 100644 --- a/pkg/scraper/sources/builtwith/builtwith.go +++ b/pkg/scraper/sources/builtwith/builtwith.go @@ -60,8 +60,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - getDomainInfoRes.Body.Close() - return } @@ -83,12 +81,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getDomainInfoRes.Body.Close() - for index := range getDomainInfoResData.Results { - item := getDomainInfoResData.Results[index] - - for index := range item.Result.Paths { - path := item.Result.Paths[index] - + for _, item := range getDomainInfoResData.Results { + for _, path := range item.Result.Paths { value := path.Domain if path.SubDomain != "" { diff --git a/pkg/scraper/sources/certspotter/certspotter.go b/pkg/scraper/sources/certspotter/certspotter.go index 317a706..8229973 100644 --- a/pkg/scraper/sources/certspotter/certspotter.go +++ b/pkg/scraper/sources/certspotter/certspotter.go @@ -60,8 +60,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - getCTLogsSearchRes.Body.Close() - return } @@ -87,12 +85,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s return } - for index := range getCTLogsSearchResData { - cert := getCTLogsSearchResData[index] - - for index := range cert.DNSNames { - subdomain := cert.DNSNames[index] - + for _, cert := range getCTLogsSearchResData { + for _, subdomain := range cert.DNSNames { if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) { continue } @@ -124,9 +118,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - getCTLogsSearchRes.Body.Close() - - continue + break } var getCTLogsSearchResData []getCTLogsSearchResponse @@ -142,21 +134,17 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getCTLogsSearchRes.Body.Close() - continue + break } getCTLogsSearchRes.Body.Close() if len(getCTLogsSearchResData) == 0 { - return + break } - for index := range getCTLogsSearchResData { - cert := getCTLogsSearchResData[index] - - for index := range cert.DNSNames { - subdomain := cert.DNSNames[index] - + for _, cert := range getCTLogsSearchResData { + for _, subdomain := range cert.DNSNames { if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) { continue } diff --git a/pkg/scraper/sources/chaos/chaos.go b/pkg/scraper/sources/chaos/chaos.go index 0a049b1..eb7b9cb 100644 --- a/pkg/scraper/sources/chaos/chaos.go +++ b/pkg/scraper/sources/chaos/chaos.go @@ -56,8 +56,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - getSubdomainsRes.Body.Close() - return } @@ -79,9 +77,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getSubdomainsRes.Body.Close() - for index := range getSubdomainsResData.Subdomains { - subdomain := getSubdomainsResData.Subdomains[index] - + for _, subdomain := range getSubdomainsResData.Subdomains { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/scraper/sources/commoncrawl/commoncrawl.go b/pkg/scraper/sources/commoncrawl/commoncrawl.go index 19a9b43..ed8a9e4 100644 --- a/pkg/scraper/sources/commoncrawl/commoncrawl.go +++ b/pkg/scraper/sources/commoncrawl/commoncrawl.go @@ -70,8 +70,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result - getIndexesRes.Body.Close() - return } @@ -103,12 +101,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source searchIndexes := make(map[string]string) - for index := range years { - year := years[index] - - for index := range getIndexesResData { - CCIndex := getIndexesResData[index] - + for _, year := range years { + for _, CCIndex := range getIndexesResData { if strings.Contains(CCIndex.ID, year) { if _, ok := searchIndexes[year]; !ok { searchIndexes[year] = CCIndex.API @@ -138,8 +132,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result - getPaginationRes.Body.Close() - continue } @@ -180,8 +172,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result - getURLsRes.Body.Close() - continue } @@ -219,11 +209,9 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source continue } - match := regex.FindAllString(getURLsResData.URL, -1) - - for index := range match { - subdomain := match[index] + subdomains := regex.FindAllString(getURLsResData.URL, -1) + for _, subdomain := range subdomains { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/scraper/sources/crtsh/crtsh.go b/pkg/scraper/sources/crtsh/crtsh.go index ae4f14c..9aa6efb 100644 --- a/pkg/scraper/sources/crtsh/crtsh.go +++ b/pkg/scraper/sources/crtsh/crtsh.go @@ -39,8 +39,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result - getNameValuesRes.Body.Close() - return } @@ -62,13 +60,10 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getNameValuesRes.Body.Close() - for index := range getNameValuesResData { - record := getNameValuesResData[index] + for _, record := range getNameValuesResData { subdomains := strings.Split(record.NameValue, "\n") - for index := range subdomains { - subdomain := subdomains[index] - + for _, subdomain := range subdomains { if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) { continue } diff --git a/pkg/scraper/sources/fullhunt/fullhunt.go b/pkg/scraper/sources/fullhunt/fullhunt.go index fe9368f..df4be1f 100644 --- a/pkg/scraper/sources/fullhunt/fullhunt.go +++ b/pkg/scraper/sources/fullhunt/fullhunt.go @@ -60,8 +60,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - getSubdomainsRes.Body.Close() - return } @@ -84,9 +82,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getSubdomainsRes.Body.Close() - for index := range getSubdomainsResData.Hosts { - subdomain := getSubdomainsResData.Hosts[index] - + for _, subdomain := range getSubdomainsResData.Hosts { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/scraper/sources/github/github.go b/pkg/scraper/sources/github/github.go index 5bd0193..6f5909b 100644 --- a/pkg/scraper/sources/github/github.go +++ b/pkg/scraper/sources/github/github.go @@ -145,8 +145,6 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp results <- result - getRawContentRes.Body.Close() - continue } diff --git a/pkg/scraper/sources/hackertarget/hackertarget.go b/pkg/scraper/sources/hackertarget/hackertarget.go index 0564cf7..049c710 100644 --- a/pkg/scraper/sources/hackertarget/hackertarget.go +++ b/pkg/scraper/sources/hackertarget/hackertarget.go @@ -35,8 +35,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result - hostSearchRes.Body.Close() - return } @@ -66,9 +64,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source match := regex.FindAllString(line, -1) - for index := range match { - subdomain := match[index] - + for _, subdomain := range match { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/scraper/sources/intelx/intelx.go b/pkg/scraper/sources/intelx/intelx.go index d7e6a2e..4131e70 100644 --- a/pkg/scraper/sources/intelx/intelx.go +++ b/pkg/scraper/sources/intelx/intelx.go @@ -111,8 +111,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - searchRes.Body.Close() - return } @@ -150,8 +148,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - getResultsRes.Body.Close() - return } @@ -175,9 +171,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s status = getResultsResData.Status - for index := range getResultsResData.Selectors { - record := getResultsResData.Selectors[index] - + for _, record := range getResultsResData.Selectors { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/scraper/sources/leakix/leakix.go b/pkg/scraper/sources/leakix/leakix.go index aecd246..5f1c363 100644 --- a/pkg/scraper/sources/leakix/leakix.go +++ b/pkg/scraper/sources/leakix/leakix.go @@ -63,8 +63,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - getSubdomainsRes.Body.Close() - return } @@ -86,9 +84,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getSubdomainsRes.Body.Close() - for index := range getSubdomainsResData { - record := getSubdomainsResData[index] - + for _, record := range getSubdomainsResData { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/scraper/sources/otx/otx.go b/pkg/scraper/sources/otx/otx.go index 7c2982f..3c4c254 100644 --- a/pkg/scraper/sources/otx/otx.go +++ b/pkg/scraper/sources/otx/otx.go @@ -42,8 +42,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result - getPassiveDNSRes.Body.Close() - return } @@ -77,8 +75,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source return } - for index := range getPassiveDNSResData.PassiveDNS { - subdomain := getPassiveDNSResData.PassiveDNS[index].Hostname + for _, record := range getPassiveDNSResData.PassiveDNS { + subdomain := record.Hostname if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) { continue diff --git a/pkg/scraper/sources/shodan/shodan.go b/pkg/scraper/sources/shodan/shodan.go index 61bad1e..5477429 100644 --- a/pkg/scraper/sources/shodan/shodan.go +++ b/pkg/scraper/sources/shodan/shodan.go @@ -55,8 +55,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - getDNSRes.Body.Close() - return } @@ -78,9 +76,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s getDNSRes.Body.Close() - for index := range getDNSResData.Subdomains { - subdomain := getDNSResData.Subdomains[index] - + for _, subdomain := range getDNSResData.Subdomains { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(), diff --git a/pkg/scraper/sources/urlscan/urlscan.go b/pkg/scraper/sources/urlscan/urlscan.go index 93ecb33..e6a627d 100644 --- a/pkg/scraper/sources/urlscan/urlscan.go +++ b/pkg/scraper/sources/urlscan/urlscan.go @@ -81,8 +81,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s results <- result - searchRes.Body.Close() - break } @@ -108,8 +106,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s break } - for index := range searchResData.Results { - subdomain := searchResData.Results[index].Page.Domain + for _, record := range searchResData.Results { + subdomain := record.Page.Domain if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) { continue @@ -128,6 +126,10 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s break } + if len(searchResData.Results) < 1 { + break + } + lastResult := searchResData.Results[len(searchResData.Results)-1] if lastResult.Sort != nil { diff --git a/pkg/scraper/sources/wayback/wayback.go b/pkg/scraper/sources/wayback/wayback.go index d65960f..3778dd3 100644 --- a/pkg/scraper/sources/wayback/wayback.go +++ b/pkg/scraper/sources/wayback/wayback.go @@ -40,8 +40,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source var pages uint - err = json.NewDecoder(getPagesRes.Body).Decode(&pages) - if err != nil { + if err = json.NewDecoder(getPagesRes.Body).Decode(&pages); err != nil { result := sources.Result{ Type: sources.Error, Source: source.Name(), @@ -87,8 +86,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source results <- result - getURLsRes.Body.Close() - return } @@ -117,13 +114,10 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source } // Slicing as [1:] to skip first result by default - for index := range getURLsResData[1:] { - entry := getURLsResData[1:][index] + for _, entry := range getURLsResData[1:] { match := regex.FindAllString(entry[0], -1) - for index := range match { - subdomain := match[index] - + for _, subdomain := range match { result := sources.Result{ Type: sources.Subdomain, Source: source.Name(),