From 2eb94b1bc8f69b79f315da7a3a4175dfd5fe5d05 Mon Sep 17 00:00:00 2001 From: Joe Obarzanek Date: Fri, 9 Feb 2024 19:51:36 -0500 Subject: [PATCH 1/4] Stop duplicate requests --- whodap/client.py | 48 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/whodap/client.py b/whodap/client.py index 4e79656..c4dc85b 100644 --- a/whodap/client.py +++ b/whodap/client.py @@ -14,7 +14,13 @@ import httpx from .codes import RDAPStatusCodes -from .errors import RateLimitError, NotFoundError, MalformedQueryError, BadStatusCode +from .errors import ( + RateLimitError, + NotFoundError, + MalformedQueryError, + BadStatusCode, + WhodapError, +) from .response import DomainResponse, IPv4Response, IPv6Response, ASNResponse @@ -97,7 +103,7 @@ def new_client_context(cls, httpx_client: Optional[httpx.Client] = None): @classmethod def new_client(cls, httpx_client: Optional[httpx.Client] = None): """ - Classmethod for instantiating an synchronous instance of Client + Classmethod for instantiating a synchronous instance of Client :httpx_client: pre-configured instance of `httpx.Client` :return: DNSClient with a sync httpx_client @@ -175,7 +181,7 @@ async def _aio_get_request(self, uri: str) -> httpx.Response: return await self.httpx_client.get(uri) def _get_authoritative_response( - self, href: str, depth: int = 0 + self, href: str, seen: List[str], depth: int = 0 ) -> Optional[httpx.Response]: """ Makes HTTP calls to RDAP servers until it finds @@ -209,13 +215,16 @@ def _get_authoritative_response( links = rdap_json.get("links") if links: next_href = self._check_next_href(href, links) - if next_href: - resp = self._get_authoritative_response(next_href, depth + 1) or resp + if next_href and next_href not in seen: + seen.append(next_href) + resp = ( + self._get_authoritative_response(next_href, seen, depth + 1) or resp + ) # return authoritative response return resp async def _aio_get_authoritative_response( - self, href: str, depth: int = 0 + self, href: str, seen: List[str], depth: int = 0 ) -> Optional[httpx.Response]: """ Makes HTTP calls to RDAP servers until it finds @@ -248,9 +257,12 @@ async def _aio_get_authoritative_response( links = rdap_json.get("links") if links: next_href = self._check_next_href(href, links) - if next_href: + if next_href and next_href not in seen: + seen.append(next_href) resp = ( - await self._aio_get_authoritative_response(next_href, depth + 1) + await self._aio_get_authoritative_response( + next_href, seen, depth + 1 + ) or resp ) return resp @@ -341,7 +353,7 @@ def lookup(self, domain: str, tld: str, auth_href: str = None) -> DomainResponse # build query href href = self._build_query_href(base_href, self._target) # get response - rdap_resp = self._get_authoritative_response(href) + rdap_resp = self._get_authoritative_response(href, [href]) # construct and return domain response domain_response = DomainResponse.from_json(rdap_resp.read()) return domain_response @@ -373,7 +385,7 @@ async def aio_lookup( # build query href href = self._build_query_href(base_href, self._target) # get response - rdap_resp = await self._aio_get_authoritative_response(href) + rdap_resp = await self._aio_get_authoritative_response(href, [href]) # construct and return domain response domain_response = DomainResponse.from_json(rdap_resp.read()) return domain_response @@ -430,7 +442,7 @@ def lookup( self._target = ipv4 server = self._get_rdap_server(self._target) href = self._build_query_href(server, str(self._target)) - rdap_resp = self._get_authoritative_response(href) + rdap_resp = self._get_authoritative_response(href, [href]) ipv4_response = IPv4Response.from_json(rdap_resp.read()) return ipv4_response @@ -453,7 +465,7 @@ async def aio_lookup( self._target = ipv4 server = self._get_rdap_server(self._target) href = self._build_query_href(server, str(self._target)) - rdap_resp = await self._aio_get_authoritative_response(href) + rdap_resp = await self._aio_get_authoritative_response(href, [href]) ipv4_response = IPv4Response.from_json(rdap_resp.read()) return ipv4_response @@ -507,8 +519,10 @@ def lookup( else: self._target = ipv6 server = self._get_rdap_server(self._target) + if server is None: + raise WhodapError(f"No RDAP server found for IPv6={ipv6}") href = self._build_query_href(server, str(self._target)) - rdap_resp = self._get_authoritative_response(href) + rdap_resp = self._get_authoritative_response(href, [href]) ipv6_response = IPv6Response.from_json(rdap_resp.read()) return ipv6_response @@ -530,8 +544,10 @@ async def aio_lookup( else: self._target = ipv6 server = self._get_rdap_server(self._target) + if server is None: + raise WhodapError(f"No RDAP server found for IPv6={ipv6}") href = self._build_query_href(server, str(self._target)) - rdap_resp = await self._aio_get_authoritative_response(href) + rdap_resp = await self._aio_get_authoritative_response(href, [href]) ipv6_response = IPv6Response.from_json(rdap_resp.read()) return ipv6_response @@ -575,7 +591,7 @@ def lookup(self, asn: int, auth_href: str = None) -> ASNResponse: self._target = asn server = self._get_rdap_server(asn) href = self._build_query_href(server, str(asn)) - rdap_resp = self._get_authoritative_response(href) + rdap_resp = self._get_authoritative_response(href, [href]) asn_response = ASNResponse.from_json(rdap_resp.read()) return asn_response @@ -590,7 +606,7 @@ async def aio_lookup(self, asn: int, auth_href: str = None) -> ASNResponse: self._target = asn server = self._get_rdap_server(asn) href = self._build_query_href(server, str(asn)) - rdap_resp = await self._aio_get_authoritative_response(href) + rdap_resp = await self._aio_get_authoritative_response(href, [href]) asn_response = ASNResponse.from_json(rdap_resp.read()) return asn_response From df37f0ce9f8920aa7eaa67addaedb50c9705bacc Mon Sep 17 00:00:00 2001 From: Joe Obarzanek Date: Fri, 9 Feb 2024 19:53:55 -0500 Subject: [PATCH 2/4] Remove unused href chain This member variable is not used and there are no plans to use it. It was just accummulating memory for no reason, so it's gone... --- whodap/client.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/whodap/client.py b/whodap/client.py index c4dc85b..8b768f0 100644 --- a/whodap/client.py +++ b/whodap/client.py @@ -39,7 +39,6 @@ def __init__(self, httpx_client: Union[httpx.Client, httpx.AsyncClient]): self.httpx_client = httpx_client self.version: str = "" self.publication: str = "" - self.rdap_hrefs: List[str] = [] self._target: Union[str, int, ipaddress.IPv4Address, ipaddress.IPv6Address] = "" def lookup(self, *args, **kwargs): @@ -202,8 +201,6 @@ def _get_authoritative_response( return None else: raise - # save href chain - self.rdap_hrefs.append(href) # check for more authoritative source try: # If for some reason the response is invalid json, then just return None. @@ -245,8 +242,6 @@ async def _aio_get_authoritative_response( return None else: raise - # save href chain - self.rdap_hrefs.append(href) try: # If for some reason the response is invalid json, then just return None. # This may happen if we request an authoritative href that is not actually From 7d11f34cadcc1e6e69b9c1627e86b8b666e3013d Mon Sep 17 00:00:00 2001 From: Joe Obarzanek Date: Fri, 9 Feb 2024 19:55:11 -0500 Subject: [PATCH 3/4] Try to handle misformatted nameservers --- whodap/response.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/whodap/response.py b/whodap/response.py index 040f5ee..608519f 100644 --- a/whodap/response.py +++ b/whodap/response.py @@ -173,7 +173,7 @@ def to_whois_dict( does not modify the original DomainResponse object. :param strict: If True, raises an RDAPConformanceException if - the given RDAP response is incorrectly formatted. Otherwise + the given RDAP response is incorrectly formatted. Otherwise, if False, the method will attempt to parse the RDAP response without raising any exception. :return: dict with WHOIS keys @@ -184,7 +184,13 @@ def to_whois_dict( if getattr(self, "nameservers", None): flat_nameservers = {"nameservers": []} for obj in self.nameservers: - flat_nameservers["nameservers"].append(obj.ldhName) + if hasattr(obj, "ldhName"): + flat_nameservers["nameservers"].append(obj.ldhName) + # if hostnames are not given, try ipv4 addresses + elif hasattr(obj, "ipAddresses"): + if hasattr(obj.ipAddresses, "v4"): + flat_nameservers["nameservers"].extend(obj.ipAddresses.v4) + flat.update(flat_nameservers) if getattr(self, "status", None): From f272d91a29e0e4a3168bd61b7b9c2b589bbec5f7 Mon Sep 17 00:00:00 2001 From: Joe Obarzanek Date: Fri, 9 Feb 2024 19:55:29 -0500 Subject: [PATCH 4/4] Update doc strings --- whodap/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/whodap/__init__.py b/whodap/__init__.py index 79cfe2a..5d4d640 100644 --- a/whodap/__init__.py +++ b/whodap/__init__.py @@ -31,7 +31,7 @@ def lookup_domain( submits an RDAP query for the given domain, and returns the result as a DomainResponse. - :param domain: the domain name to lookup + :param domain: the domain name to query :param tld: the top level domain (e.g. "com", "net", "buzz") :param httpx_client: Optional preconfigured instance of `httpx.Client` :return: an instance of DomainResponse @@ -55,7 +55,7 @@ async def aio_lookup_domain( a DNSClient, submits an RDAP query for the given domain, and returns the result as a DomainResponse. - :param domain: the domain name to lookup + :param domain: the domain name to query :param tld: the top level domain (e.g. "com", "net", "buzz") :param httpx_client: Optional preconfigured instance of `httpx.AsyncClient` :return: an instance of DomainResponse