cleaning the cache

biobricks-ai · Jul 9, 2024 · 5b4f501 · 5b4f501
1 parent 31c0a47
commit 5b4f501
Show file tree

Hide file tree

Showing 208 changed files with 13 additions and 6,302 deletions.
diff --git a/stages/01_download.py b/stages/01_download.py
@@ -32,14 +32,14 @@
 
 # Download links to cache/{source_name}/annotations.json
 safe_sources = [re.sub(r'[\\/*?:"<>|]', '_', name) for name in source_names]
-for link, name in tqdm.tqdm(zip(links, safe_sources), total=len(links), desc="Downloading annotations"):
-  annotation_path = os.path.join('cache/01_download', name, 'annotations.json')
-  os.makedirs(os.path.dirname(annotation_path), exist_ok=True)
-  response = requests.get(link, stream=True)
-  time.sleep(3)  # Be nice to the server
-  with open(annotation_path, 'wb') as f:
-    for chunk in response.iter_content(chunk_size=8192):
-      _ = f.write(chunk)
+# for link, name in tqdm.tqdm(zip(links, safe_sources), total=len(links), desc="Downloading annotations"):
+#   annotation_path = os.path.join('cache/01_download', name, 'annotations.json')
+#  os.makedirs(os.path.dirname(annotation_path), exist_ok=True)
+#  response = requests.get(link, stream=True)
+#  time.sleep(3)  # Be nice to the server
+#  with open(annotation_path, 'wb') as f:
+#    for chunk in response.iter_content(chunk_size=8192):
+#      _ = f.write(chunk)
 
 # Make a 'headings' table by reading all the annotations.json files
 headings = []
@@ -75,6 +75,7 @@
     'response_type': 'save',
     'response_basename': f'PubChemAnnotations_{row["source"]}_heading={safe_heading}'
   }
+
   # Initial URL and download path setup
   download_path = os.path.join('cache/01_download', row['safe_source'], f'{safe_heading}.json')
   os.makedirs(os.path.dirname(download_path), exist_ok=True)
@@ -93,7 +94,6 @@
       if response.status_code == 200:
         data = response.json()
         annotations = data.get('Annotations', {})
-        print ("Annotations: %s" % annotations)
         all_data.extend(annotations.get('Annotation', []))
         # Check if more pages exist
         if current_page >= annotations.get('TotalPages', 1):

diff --git a/stages/02_process.py b/stages/02_process.py
@@ -16,7 +16,7 @@
 
 # Create a dictionary for quick lookup
 json_file_dict = {os.path.splitext(os.path.basename(file))[0]: file for file in all_json_files}
-
+print (json_file_dict)
 # Debugging: Log the total number of JSON files found
 print(f"Total JSON files found: {len(all_json_files)}")
 
@@ -25,10 +25,12 @@
     source = row['source']
     heading = row['heading']
     data_type = row['type']
+
     # Construct the expected file name without path
     file_name = f"{heading}.json"
+
     # Check if the file is in the dictionary
-    if heading in json_file_dict:
+    if heading in json_file_dict['annotations']:
         file_path = json_file_dict[heading]
         # Debugging: Log the found file path
         print(f"Processing file: {file_path}")

diff --git a/...01_download/Agency for Toxic Substances and Disease Registry (ATSDR)/Hazards Summary.json b/...01_download/Agency for Toxic Substances and Disease Registry (ATSDR)/Hazards Summary.json
diff --git a/...download/Agency for Toxic Substances and Disease Registry (ATSDR)/Record Description.json b/...download/Agency for Toxic Substances and Disease Registry (ATSDR)/Record Description.json
diff --git a/...e/01_download/Agency for Toxic Substances and Disease Registry (ATSDR)/Target Organs.json b/...e/01_download/Agency for Toxic Substances and Disease Registry (ATSDR)/Target Organs.json
diff --git a/...d/Agency for Toxic Substances and Disease Registry (ATSDR)/Toxicological Information.json b/...d/Agency for Toxic Substances and Disease Registry (ATSDR)/Toxicological Information.json
diff --git a/...download/Agency for Toxic Substances and Disease Registry (ATSDR)/Use Classification.json b/...download/Agency for Toxic Substances and Disease Registry (ATSDR)/Use Classification.json
diff --git a/...che/01_download/Agency for Toxic Substances and Disease Registry (ATSDR)/annotations.json b/...che/01_download/Agency for Toxic Substances and Disease Registry (ATSDR)/annotations.json
diff --git a/stages/cache/01_download/Alliance of Genome Resources/annotations.json b/stages/cache/01_download/Alliance of Genome Resources/annotations.json
diff --git a/stages/cache/01_download/Athena Minerals/annotations.json b/stages/cache/01_download/Athena Minerals/annotations.json
diff --git a/...tomic Mass Data Center (AMDC), International Atomic Energy Agency (IAEA)/annotations.json b/...tomic Mass Data Center (AMDC), International Atomic Energy Agency (IAEA)/annotations.json
diff --git a/.../01_download/Australian Industrial Chemicals Introduction Scheme (AICIS)/annotations.json b/.../01_download/Australian Industrial Chemicals Introduction Scheme (AICIS)/annotations.json
diff --git a/stages/cache/01_download/BRENDA_ Enzyme Functional Data/annotations.json b/stages/cache/01_download/BRENDA_ Enzyme Functional Data/annotations.json
diff --git a/...b, Chemistry Department, The University of North Carolina at Chapel Hill/annotations.json b/...b, Chemistry Department, The University of North Carolina at Chapel Hill/annotations.json
diff --git a/stages/cache/01_download/BindingDB/annotations.json b/stages/cache/01_download/BindingDB/annotations.json
diff --git a/stages/cache/01_download/BioAssay Research Database (BARD)/annotations.json b/stages/cache/01_download/BioAssay Research Database (BARD)/annotations.json
diff --git a/stages/cache/01_download/BioGRID/annotations.json b/stages/cache/01_download/BioGRID/annotations.json
diff --git a/stages/cache/01_download/Burnham Center for Chemical Genomics/annotations.json b/stages/cache/01_download/Burnham Center for Chemical Genomics/annotations.json
diff --git a/stages/cache/01_download/CAMEO Chemicals/annotations.json b/stages/cache/01_download/CAMEO Chemicals/annotations.json
diff --git a/stages/cache/01_download/CAS Common Chemistry/annotations.json b/stages/cache/01_download/CAS Common Chemistry/annotations.json
diff --git a/stages/cache/01_download/CCSbase/annotations.json b/stages/cache/01_download/CCSbase/annotations.json
diff --git a/stages/cache/01_download/COVID-19 Disease Map/annotations.json b/stages/cache/01_download/COVID-19 Disease Map/annotations.json