refactor: Reimplement looping to avoid out of range errors

hueristiq · Nov 27, 2023 · dd1c892 · dd1c892
1 parent 57bc303
commit dd1c892
Show file tree

Hide file tree

Showing 17 changed files with 36 additions and 118 deletions.
diff --git a/pkg/scraper/sources/anubis/anubis.go b/pkg/scraper/sources/anubis/anubis.go
@@ -33,8 +33,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
-			getSubdomainsRes.Body.Close()
-
 			return
 		}
 
@@ -56,9 +54,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 		getSubdomainsRes.Body.Close()
 
-		for index := range getSubdomainsResData {
-			subdomain := getSubdomainsResData[index]
-
+		for _, subdomain := range getSubdomainsResData {
 			result := sources.Result{
 				Type:   sources.Subdomain,
 				Source: source.Name(),

diff --git a/pkg/scraper/sources/bevigil/bevigil.go b/pkg/scraper/sources/bevigil/bevigil.go
@@ -59,8 +59,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getSubdomainsRes.Body.Close()
-
 			return
 		}
 
@@ -82,9 +80,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 		getSubdomainsRes.Body.Close()
 
-		for index := range getSubdomainsResData.Subdomains {
-			subdomain := getSubdomainsResData.Subdomains[index]
-
+		for _, subdomain := range getSubdomainsResData.Subdomains {
 			result := sources.Result{
 				Type:   sources.Subdomain,
 				Source: source.Name(),

diff --git a/pkg/scraper/sources/bufferover/bufferover.go b/pkg/scraper/sources/bufferover/bufferover.go
@@ -62,8 +62,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getTLSLogsSearchRes.Body.Close()
-
 			return
 		}
 
@@ -119,13 +117,10 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 			entries = getTLSLogsSearchResData.Results
 		}
 
-		for index := range entries {
-			entry := entries[index]
+		for _, entry := range entries {
 			subdomains := regex.FindAllString(entry, -1)
 
-			for index := range subdomains {
-				subdomain := subdomains[index]
-
+			for _, subdomain := range subdomains {
 				result := sources.Result{
 					Type:   sources.Subdomain,
 					Source: source.Name(),

diff --git a/pkg/scraper/sources/builtwith/builtwith.go b/pkg/scraper/sources/builtwith/builtwith.go
@@ -60,8 +60,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getDomainInfoRes.Body.Close()
-
 			return
 		}
 
@@ -83,12 +81,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 		getDomainInfoRes.Body.Close()
 
-		for index := range getDomainInfoResData.Results {
-			item := getDomainInfoResData.Results[index]
-
-			for index := range item.Result.Paths {
-				path := item.Result.Paths[index]
-
+		for _, item := range getDomainInfoResData.Results {
+			for _, path := range item.Result.Paths {
 				value := path.Domain
 
 				if path.SubDomain != "" {

diff --git a/pkg/scraper/sources/certspotter/certspotter.go b/pkg/scraper/sources/certspotter/certspotter.go
@@ -60,8 +60,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getCTLogsSearchRes.Body.Close()
-
 			return
 		}
 
@@ -87,12 +85,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 			return
 		}
 
-		for index := range getCTLogsSearchResData {
-			cert := getCTLogsSearchResData[index]
-
-			for index := range cert.DNSNames {
-				subdomain := cert.DNSNames[index]
-
+		for _, cert := range getCTLogsSearchResData {
+			for _, subdomain := range cert.DNSNames {
 				if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
 					continue
 				}
@@ -124,9 +118,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 				results <- result
 
-				getCTLogsSearchRes.Body.Close()
-
-				continue
+				break
 			}
 
 			var getCTLogsSearchResData []getCTLogsSearchResponse
@@ -142,21 +134,17 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 				getCTLogsSearchRes.Body.Close()
 
-				continue
+				break
 			}
 
 			getCTLogsSearchRes.Body.Close()
 
 			if len(getCTLogsSearchResData) == 0 {
-				return
+				break
 			}
 
-			for index := range getCTLogsSearchResData {
-				cert := getCTLogsSearchResData[index]
-
-				for index := range cert.DNSNames {
-					subdomain := cert.DNSNames[index]
-
+			for _, cert := range getCTLogsSearchResData {
+				for _, subdomain := range cert.DNSNames {
 					if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
 						continue
 					}

diff --git a/pkg/scraper/sources/chaos/chaos.go b/pkg/scraper/sources/chaos/chaos.go
@@ -56,8 +56,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getSubdomainsRes.Body.Close()
-
 			return
 		}
 
@@ -79,9 +77,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 		getSubdomainsRes.Body.Close()
 
-		for index := range getSubdomainsResData.Subdomains {
-			subdomain := getSubdomainsResData.Subdomains[index]
-
+		for _, subdomain := range getSubdomainsResData.Subdomains {
 			result := sources.Result{
 				Type:   sources.Subdomain,
 				Source: source.Name(),

diff --git a/pkg/scraper/sources/commoncrawl/commoncrawl.go b/pkg/scraper/sources/commoncrawl/commoncrawl.go
@@ -70,8 +70,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
-			getIndexesRes.Body.Close()
-
 			return
 		}
 
@@ -103,12 +101,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 		searchIndexes := make(map[string]string)
 
-		for index := range years {
-			year := years[index]
-
-			for index := range getIndexesResData {
-				CCIndex := getIndexesResData[index]
-
+		for _, year := range years {
+			for _, CCIndex := range getIndexesResData {
 				if strings.Contains(CCIndex.ID, year) {
 					if _, ok := searchIndexes[year]; !ok {
 						searchIndexes[year] = CCIndex.API
@@ -138,8 +132,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 				results <- result
 
-				getPaginationRes.Body.Close()
-
 				continue
 			}
 
@@ -180,8 +172,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 					results <- result
 
-					getURLsRes.Body.Close()
-
 					continue
 				}
 
@@ -219,11 +209,9 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 						continue
 					}
 
-					match := regex.FindAllString(getURLsResData.URL, -1)
-
-					for index := range match {
-						subdomain := match[index]
+					subdomains := regex.FindAllString(getURLsResData.URL, -1)
 
+					for _, subdomain := range subdomains {
 						result := sources.Result{
 							Type:   sources.Subdomain,
 							Source: source.Name(),

diff --git a/pkg/scraper/sources/crtsh/crtsh.go b/pkg/scraper/sources/crtsh/crtsh.go
@@ -39,8 +39,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
-			getNameValuesRes.Body.Close()
-
 			return
 		}
 
@@ -62,13 +60,10 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 		getNameValuesRes.Body.Close()
 
-		for index := range getNameValuesResData {
-			record := getNameValuesResData[index]
+		for _, record := range getNameValuesResData {
 			subdomains := strings.Split(record.NameValue, "\n")
 
-			for index := range subdomains {
-				subdomain := subdomains[index]
-
+			for _, subdomain := range subdomains {
 				if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
 					continue
 				}

diff --git a/pkg/scraper/sources/fullhunt/fullhunt.go b/pkg/scraper/sources/fullhunt/fullhunt.go
@@ -60,8 +60,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getSubdomainsRes.Body.Close()
-
 			return
 		}
 
@@ -84,9 +82,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 		getSubdomainsRes.Body.Close()
 
-		for index := range getSubdomainsResData.Hosts {
-			subdomain := getSubdomainsResData.Hosts[index]
-
+		for _, subdomain := range getSubdomainsResData.Hosts {
 			result := sources.Result{
 				Type:   sources.Subdomain,
 				Source: source.Name(),

diff --git a/pkg/scraper/sources/github/github.go b/pkg/scraper/sources/github/github.go
@@ -145,8 +145,6 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp
 
 			results <- result
 
-			getRawContentRes.Body.Close()
-
 			continue
 		}
 

diff --git a/pkg/scraper/sources/hackertarget/hackertarget.go b/pkg/scraper/sources/hackertarget/hackertarget.go
@@ -35,8 +35,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
-			hostSearchRes.Body.Close()
-
 			return
 		}
 
@@ -66,9 +64,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			match := regex.FindAllString(line, -1)
 
-			for index := range match {
-				subdomain := match[index]
-
+			for _, subdomain := range match {
 				result := sources.Result{
 					Type:   sources.Subdomain,
 					Source: source.Name(),

diff --git a/pkg/scraper/sources/intelx/intelx.go b/pkg/scraper/sources/intelx/intelx.go
@@ -111,8 +111,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			searchRes.Body.Close()
-
 			return
 		}
 
@@ -150,8 +148,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 				results <- result
 
-				getResultsRes.Body.Close()
-
 				return
 			}
 
@@ -175,9 +171,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			status = getResultsResData.Status
 
-			for index := range getResultsResData.Selectors {
-				record := getResultsResData.Selectors[index]
-
+			for _, record := range getResultsResData.Selectors {
 				result := sources.Result{
 					Type:   sources.Subdomain,
 					Source: source.Name(),

diff --git a/pkg/scraper/sources/leakix/leakix.go b/pkg/scraper/sources/leakix/leakix.go
@@ -63,8 +63,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getSubdomainsRes.Body.Close()
-
 			return
 		}
 
@@ -86,9 +84,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 		getSubdomainsRes.Body.Close()
 
-		for index := range getSubdomainsResData {
-			record := getSubdomainsResData[index]
-
+		for _, record := range getSubdomainsResData {
 			result := sources.Result{
 				Type:   sources.Subdomain,
 				Source: source.Name(),

diff --git a/pkg/scraper/sources/otx/otx.go b/pkg/scraper/sources/otx/otx.go
@@ -42,8 +42,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
-			getPassiveDNSRes.Body.Close()
-
 			return
 		}
 
@@ -77,8 +75,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 			return
 		}
 
-		for index := range getPassiveDNSResData.PassiveDNS {
-			subdomain := getPassiveDNSResData.PassiveDNS[index].Hostname
+		for _, record := range getPassiveDNSResData.PassiveDNS {
+			subdomain := record.Hostname
 
 			if subdomain != domain && !strings.HasSuffix(subdomain, "."+domain) {
 				continue

diff --git a/pkg/scraper/sources/shodan/shodan.go b/pkg/scraper/sources/shodan/shodan.go
@@ -55,8 +55,6 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
-			getDNSRes.Body.Close()
-
 			return
 		}
 
@@ -78,9 +76,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 		getDNSRes.Body.Close()
 
-		for index := range getDNSResData.Subdomains {
-			subdomain := getDNSResData.Subdomains[index]
-
+		for _, subdomain := range getDNSResData.Subdomains {
 			result := sources.Result{
 				Type:   sources.Subdomain,
 				Source: source.Name(),
-Original file line number
+Diff line change
@@ Expand Up @@
     			results <- result
-    			getRawContentRes.Body.Close()
     			continue
     		}
@@ Expand Down @@