More robust dataset store

whiletrue-industries · Feb 7, 2025 · d68df31 · d68df31
1 parent fbc6c00
commit d68df31
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/odds/common/metadata_store/es/es_metadata_store.py b/odds/common/metadata_store/es/es_metadata_store.py
@@ -101,7 +101,12 @@ async def storeDataset(self, dataset: Dataset, ctx: str) -> None:
                         if k not in ['name', 'data_type']:
                             props[k] = field.pop(k)
                     field['props'] = json.dumps(props, ensure_ascii=False)
-            ret = await client.update(index='datasets', id=id, doc=body, doc_as_upsert=True)
+            try:
+                ret = await client.update(index='datasets', id=id, doc=body, doc_as_upsert=True)
+            except Exception as e:
+                rts.set(ctx, f'ERROR STORING DATASET {dataset.title} -> {id}: {e!r}', 'error')
+                json.dump(body, open(f'/srv/.caches/error_{id}.json', 'w'))
+                raise
 
     async def getDataset(self, datasetId: str) -> Dataset:
         async with ESClient() as client: