Merge pull request #25 from rkolpakov/feature/recursive-dependency-re…

…solving

feat: add recursive dependency search for imported contracts

README.md

@@ -88,4 +88,12 @@ Start the script
 python3 main.py
 ```
 
+> Note: Brownie verification tooling might rewrite the imports in the source submission. It transforms relative paths to imported contracts into flat paths ('./folder/contract.sol' -> 'contract.sol'), which makes Diffyscan unable to find a contract for verification.
+
+For contracts whose sources were verified by brownie tooling:
+
+```bash
+python3 main.py --support-brownie
+```
+
 ℹ️ See more config examples inside the [config_samples](./config_samples/) dir.

main.py

@@ -7,7 +7,7 @@
 from utils.common import load_config, load_env
 from utils.constants import DIFFS_DIR, START_TIME, DEFAULT_CONFIG_PATH
 from utils.explorer import get_contract_from_explorer
-from utils.github import get_file_from_github, resolve_dep
+from utils.github import get_file_from_github, get_file_from_github_recursive, resolve_dep
 from utils.helpers import create_dirs
 from utils.logger import logger
 
@@ -17,7 +17,7 @@
 g_skip_user_input: bool = False
 
 
-def run_diff(config, name, address, explorer_api_token, github_api_token):
+def run_diff(config, name, address, explorer_api_token, github_api_token, recursive_parsing=False):
     logger.divider()
     logger.okay("Contract", address)
     logger.okay("Blockchain explorer Hostname", config["explorer_hostname"])
@@ -79,9 +79,11 @@ def run_diff(config, name, address, explorer_api_token, github_api_token):
 
         file_found = bool(repo)
 
-        github_file = get_file_from_github(
-            github_api_token, repo, path_to_file, dep_name
-        )
+        if recursive_parsing:
+            github_file = get_file_from_github_recursive(github_api_token, repo, path_to_file, dep_name)
+        else:
+            github_file = get_file_from_github(github_api_token, repo, path_to_file, dep_name)
+
         if not github_file:
             github_file = "<!-- No file content -->"
             file_found = False
@@ -121,7 +123,7 @@ def run_diff(config, name, address, explorer_api_token, github_api_token):
     logger.report_table(report)
 
 
-def process_config(path: str):
+def process_config(path: str, recursive_parsing: bool):
     logger.info(f"Loading config {path}...")
     config = load_config(path)
     explorer_token = None
@@ -131,13 +133,18 @@ def process_config(path: str):
     contracts = config["contracts"]
     logger.info(f"Running diff for contracts from config {contracts}...")
     for address, name in config["contracts"].items():
-        run_diff(config, name, address, explorer_token, GITHUB_API_TOKEN)
+        run_diff(config, name, address, explorer_token, GITHUB_API_TOKEN, recursive_parsing)
 
 
 def parse_arguments():
     parser = argparse.ArgumentParser()
     parser.add_argument("path", nargs="?", default=None, help="Path to config or directory with configs")
     parser.add_argument("--yes", "-y", help="If set don't ask for input before validating each contract", action="store_true")
+    parser.add_argument(
+        "--support-brownie",
+        help="Support recursive retrieving for contracts. It may be useful for contracts whose sources have been verified by the brownie tooling, which automatically replaces relative paths to contracts in imports with plain contract names.",
+        action=argparse.BooleanOptionalAction,
+    )
     return parser.parse_args()
 
 
@@ -151,14 +158,14 @@ def main():
     logger.divider()
 
     if args.path is None:
-        process_config(DEFAULT_CONFIG_PATH)
+        process_config(DEFAULT_CONFIG_PATH, args.support_brownie)
     elif os.path.isfile(args.path):
-        process_config(args.path)
+        process_config(args.path, args.support_brownie)
     elif os.path.isdir(args.path):
         for filename in os.listdir(args.path):
             config_path = os.path.join(args.path, filename)
             if os.path.isfile(config_path):
-                process_config(config_path)
+                process_config(config_path, args.support_brownie)
 
     execution_time = time.time() - START_TIME
 

utils/common.py

@@ -35,6 +35,9 @@ def fetch(url, headers={}):
     logger.log(f"fetch: {url}")
     response = requests.get(url, headers=headers)
 
+    if response.status_code == 404:
+        return None
+
     if not response.ok and response.status_code != 200:
         logger.error("Request failed", url)
         logger.error("Status", response.status_code)

utils/github.py

@@ -8,14 +8,17 @@ def get_file_from_github(github_api_token, dependency_repo, path_to_file, dep_na
 
     user_slash_repo = parse_repo_link(dependency_repo["url"])
 
-    github_api_url = f"https://api.github.com/repos/{user_slash_repo}/contents/{dependency_repo['relative_root']}/{path_to_file}"
-
-    github_api_url += "?ref=" + dependency_repo["commit"]
+    github_api_url = get_github_api_url(
+        user_slash_repo,
+        dependency_repo["relative_root"],
+        path_to_file,
+        dependency_repo["commit"],
+    )
 
     github_data = fetch(
         github_api_url, headers={"Authorization": f"token {github_api_token}"}
     )
-
+    
     file_content = github_data.get("content")
 
     if not file_content:
@@ -25,6 +28,109 @@ def get_file_from_github(github_api_token, dependency_repo, path_to_file, dep_na
     return base64.b64decode(file_content).decode()
 
 
+def get_file_from_github_recursive(
+    github_api_token, dependency_repo, path_to_file, dep_name
+):
+    path_to_file = path_to_file_without_dependency(path_to_file, dep_name)
+    user_slash_repo = parse_repo_link(dependency_repo["url"])
+
+    direct_file_content = _get_direct_file(
+        github_api_token,
+        user_slash_repo,
+        dependency_repo["relative_root"],
+        path_to_file,
+        dependency_repo["commit"],
+    )
+    if direct_file_content:
+        return direct_file_content
+
+    return _recursive_search(
+        github_api_token,
+        user_slash_repo,
+        dependency_repo["relative_root"],
+        path_to_file,
+        dependency_repo["commit"],
+    )
+
+
+def _get_direct_file(
+    github_api_token, user_slash_repo, relative_root, path_to_file, commit
+):
+    github_api_url = get_github_api_url(
+        user_slash_repo, relative_root, path_to_file, commit
+    )
+    response = fetch(
+        github_api_url, headers={"Authorization": f"token {github_api_token}"}
+    )
+
+    if response is None:
+        return None
+    github_data = response
+
+    if isinstance(github_data, dict) and github_data.get("type") == "file":
+        file_content = github_data.get("content")
+        if not file_content:
+            logger.error(f"No file content in {path_to_file}")
+            return None
+        return base64.b64decode(file_content).decode()
+
+    return None
+
+
+def _recursive_search(
+    github_api_token,
+    user_slash_repo,
+    relative_path,
+    filename,
+    commit,
+    checked_dirs=None,
+):
+    if checked_dirs is None:
+        checked_dirs = []
+
+    github_api_url = get_github_api_url(
+        user_slash_repo, relative_path, filename, commit
+    )
+    github_data = fetch(
+        github_api_url, headers={"Authorization": f"token {github_api_token}"}
+    )
+
+    if github_data and isinstance(github_data, dict) and "content" in github_data:
+        return base64.b64decode(github_data["content"]).decode()
+
+    github_api_url = get_github_api_url(user_slash_repo, relative_path, None, commit)
+    github_data = fetch(
+        github_api_url, headers={"Authorization": f"token {github_api_token}"}
+    )
+
+    if github_data and isinstance(github_data, list):
+        directories = [item["path"] for item in github_data if item["type"] == "dir"]
+
+        for dir_path in directories:
+            if dir_path in checked_dirs:
+                continue
+
+            checked_dirs.append(dir_path)
+            found_content = _recursive_search(
+                github_api_token,
+                user_slash_repo,
+                dir_path,
+                filename,
+                commit,
+                checked_dirs,
+            )
+            if found_content:
+                return found_content
+
+        if relative_path and relative_path not in checked_dirs:
+            checked_dirs.append(relative_path)
+            return _recursive_search(
+                github_api_token, user_slash_repo, "", filename, commit, checked_dirs
+            )
+
+    return None
+
+
 def path_to_file_without_dependency(path_to_file, dep_name):
     # exclude dependency prefix from path to file
     # "@aragon/something/lib/my.sol" => "lib/my.sol"
@@ -47,3 +153,14 @@ def resolve_dep(path_to_file, config):
             return (config["dependencies"][dep_name], dep_name)
 
     return (None, None)
+
+
+def get_github_api_url(user_slash_repo, relative_root, path_to_file, commit):
+    url = f"https://api.github.com/repos/{user_slash_repo}/contents"
+    if relative_root:
+        url += f"/{relative_root}"
+    if path_to_file:
+        url += f"/{path_to_file}"
+    if commit:
+        url += f"?ref={commit}"
+    return url