chore: Discard HTTP responses on demand

hueristiq · Nov 27, 2023 · 76e63c7 · 76e63c7
1 parent dd1c892
commit 76e63c7
Show file tree

Hide file tree

Showing 20 changed files with 65 additions and 10 deletions.
diff --git a/go.mod b/go.mod
@@ -16,9 +16,9 @@ require (
 require (
 	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
 	github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 // indirect
-	golang.org/x/net v0.18.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
-	golang.org/x/term v0.14.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )
diff --git a/go.sum b/go.sum
@@ -35,12 +35,12 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
-golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
-golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
-golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go
@@ -74,3 +74,14 @@ func SimpleGet(URL string) (*http.Response, error) {
 func Post(URL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) {
 	return HTTPRequest(methods.Post, URL, cookies, headers, body)
 }
+
+func DiscardResponse(response *http.Response) {
+	if response != nil {
+		_, err := io.Copy(io.Discard, response.Body)
+		if err != nil {
+			return
+		}
+
+		response.Body.Close()
+	}
+}
diff --git a/pkg/scraper/sources/anubis/anubis.go b/pkg/scraper/sources/anubis/anubis.go
@@ -33,6 +33,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
+			httpclient.DiscardResponse(getSubdomainsRes)
+
 			return
 		}
 

diff --git a/pkg/scraper/sources/bevigil/bevigil.go b/pkg/scraper/sources/bevigil/bevigil.go
@@ -59,6 +59,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
+			httpclient.DiscardResponse(getSubdomainsRes)
+
 			return
 		}
 

diff --git a/pkg/scraper/sources/bufferover/bufferover.go b/pkg/scraper/sources/bufferover/bufferover.go
@@ -62,6 +62,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
+			httpclient.DiscardResponse(getTLSLogsSearchRes)
+
 			return
 		}
 

diff --git a/pkg/scraper/sources/builtwith/builtwith.go b/pkg/scraper/sources/builtwith/builtwith.go
@@ -60,6 +60,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
+			httpclient.DiscardResponse(getDomainInfoRes)
+
 			return
 		}
 

diff --git a/pkg/scraper/sources/certspotter/certspotter.go b/pkg/scraper/sources/certspotter/certspotter.go
@@ -60,6 +60,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
+			httpclient.DiscardResponse(getCTLogsSearchRes)
+
 			return
 		}
 
@@ -118,6 +120,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 				results <- result
 
+				httpclient.DiscardResponse(getCTLogsSearchRes)
+
 				break
 			}
 

diff --git a/pkg/scraper/sources/chaos/chaos.go b/pkg/scraper/sources/chaos/chaos.go
@@ -56,6 +56,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
+			httpclient.DiscardResponse(getSubdomainsRes)
+
 			return
 		}
 

diff --git a/pkg/scraper/sources/commoncrawl/commoncrawl.go b/pkg/scraper/sources/commoncrawl/commoncrawl.go
@@ -70,6 +70,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
+			httpclient.DiscardResponse(getIndexesRes)
+
 			return
 		}
 
@@ -132,6 +134,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 				results <- result
 
+				httpclient.DiscardResponse(getPaginationRes)
+
 				continue
 			}
 
@@ -172,6 +176,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 					results <- result
 
+					httpclient.DiscardResponse(getURLsRes)
+
 					continue
 				}
 

diff --git a/pkg/scraper/sources/crtsh/crtsh.go b/pkg/scraper/sources/crtsh/crtsh.go
@@ -39,6 +39,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
+			httpclient.DiscardResponse(getNameValuesRes)
+
 			return
 		}
 

diff --git a/pkg/scraper/sources/fullhunt/fullhunt.go b/pkg/scraper/sources/fullhunt/fullhunt.go
@@ -60,6 +60,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
+			httpclient.DiscardResponse(getSubdomainsRes)
+
 			return
 		}
 

diff --git a/pkg/scraper/sources/github/github.go b/pkg/scraper/sources/github/github.go
@@ -98,7 +98,7 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp
 
 		results <- result
 
-		searchRes.Body.Close()
+		httpclient.DiscardResponse(searchRes)
 
 		return
 	}
@@ -145,6 +145,8 @@ func (source *Source) Enumerate(searchReqURL string, domainRegexp *regexp.Regexp
 
 			results <- result
 
+			httpclient.DiscardResponse(getRawContentRes)
+
 			continue
 		}
 

diff --git a/pkg/scraper/sources/hackertarget/hackertarget.go b/pkg/scraper/sources/hackertarget/hackertarget.go
@@ -35,6 +35,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
+			httpclient.DiscardResponse(hostSearchRes)
+
 			return
 		}
 

diff --git a/pkg/scraper/sources/intelx/intelx.go b/pkg/scraper/sources/intelx/intelx.go
@@ -111,6 +111,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
+			httpclient.DiscardResponse(searchRes)
+
 			return
 		}
 
@@ -148,6 +150,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 				results <- result
 
+				httpclient.DiscardResponse(getResultsRes)
+
 				return
 			}
 

diff --git a/pkg/scraper/sources/leakix/leakix.go b/pkg/scraper/sources/leakix/leakix.go
@@ -63,6 +63,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
+			httpclient.DiscardResponse(getSubdomainsRes)
+
 			return
 		}
 

diff --git a/pkg/scraper/sources/otx/otx.go b/pkg/scraper/sources/otx/otx.go
@@ -42,6 +42,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
+			httpclient.DiscardResponse(getPassiveDNSRes)
+
 			return
 		}
 

diff --git a/pkg/scraper/sources/shodan/shodan.go b/pkg/scraper/sources/shodan/shodan.go
@@ -55,6 +55,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 			results <- result
 
+			httpclient.DiscardResponse(getDNSRes)
+
 			return
 		}
 

diff --git a/pkg/scraper/sources/urlscan/urlscan.go b/pkg/scraper/sources/urlscan/urlscan.go
@@ -81,6 +81,8 @@ func (source *Source) Run(config *sources.Configuration, domain string) <-chan s
 
 				results <- result
 
+				httpclient.DiscardResponse(searchRes)
+
 				break
 			}
 

diff --git a/pkg/scraper/sources/wayback/wayback.go b/pkg/scraper/sources/wayback/wayback.go
@@ -35,6 +35,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 			results <- result
 
+			httpclient.DiscardResponse(getPagesRes)
+
 			return
 		}
 
@@ -86,6 +88,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source
 
 				results <- result
 
+				httpclient.DiscardResponse(getURLsRes)
+
 				return
 			}
-Original file line number
+Diff line change
@@ Expand Up @@
     			results <- result
+    			httpclient.DiscardResponse(getSubdomainsRes)
     			return
     		}
@@ Expand Down @@