Improve scrape_all.py

odds/backend/backend.py

@@ -61,13 +61,16 @@ def scan_all(self) -> None:
     def scan_new(self) -> None:
         asyncio.run(self.scan(CatalogFilter(), DatasetFilterNew()))
 
-    def scan_specific(self, catalogId: str = None, datasetId: str = None) -> None:
+    def scan_specific(self, catalogId: str = None, datasetId: str = None, force=True) -> None:
         if catalogId:
             catalogFilter = CatalogFilterById(catalogId)
         else:
             catalogFilter = CatalogFilter()
         if datasetId:
             datasetFilter = DatasetFilterById(datasetId)
         else:
-            datasetFilter = DatasetFilterForce()
+            if force:
+                datasetFilter = DatasetFilterForce()
+            else:
+                datasetFilter = DatasetFilterIncomplete()
         asyncio.run(self.scan(catalogFilter, datasetFilter))

utils/scrape_all.py

@@ -1,11 +1,12 @@
 from odds.backend import backend
 import sys
+import argparse
 
-b = backend.ODDSBackend()
+parser = argparse.ArgumentParser(description='Scrape data from all catalogs')
+parser.add_argument('--catalog-id', type=str, help='ID of the catalog to scrape')
+parser.add_argument('--force', type=bool, default=False, help='Scrape all and not just missing datasets')
+args = parser.parse_args()
 
-if len(sys.argv) > 1:
-    selected_catalog = sys.argv[1]
-    b.scan_specific(catalogId=selected_catalog)
-else:
-    b.scan_required()
+b = backend.ODDSBackend()
+b.scan_specific(catalogId=args.catalog_id, force=args.force)
 del b