Avoid too long filenames

whiletrue-industries · Feb 20, 2025 · e8ba604 · e8ba604
1 parent 2a24c18
commit e8ba604
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/odds/backend/scanner/website/website_scanner.py b/odds/backend/scanner/website/website_scanner.py
@@ -82,8 +82,13 @@ async def scrape(self, url: str) -> list[str]:
         title = None
         final_url = None
         key = url.split('://')[1].replace('/', '_').replace(':', '_').replace('.', '_').replace('?', '_').replace('&', '_')
-        cache_file = self.CACHE / f'{key}.json'
-        cache_file_clean = self.CACHE / f'{key}.clean.html'
+        if len(key) > 240:
+            hash = sha256(key.encode()).hexdigest()[:12]
+            fkey = f'{key[:240]}_{hash}'
+        else:
+            fkey = key
+        cache_file = self.CACHE / f'{fkey}.json'
+        cache_file_clean = self.CACHE / f'{fkey}.clean.html'
         if cache_file.exists():
             with open(cache_file) as file:
                 data = json.load(file)