Merge pull request #140 from ecmwf-projects/copds-2032-multiple-repos

COPDS-2032 multiple repos

alembic/versions/36caa987229e_multiple_datasets_repo.py

@@ -0,0 +1,33 @@
+"""multiple datasets repo.
+
+Revision ID: 36caa987229e
+Revises: 6ac5a2abf5f5
+Create Date: 2024-12-27 11:12:14.551220
+
+"""
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql as dialect_postgresql
+
+import alembic
+
+# revision identifiers, used by Alembic.
+revision = "36caa987229e"
+down_revision = "6ac5a2abf5f5"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    alembic.op.drop_column("catalogue_updates", "metadata_repo_commit")
+    alembic.op.add_column(
+        "catalogue_updates",
+        sa.Column("metadata_repo_commit", dialect_postgresql.JSONB, default={}),
+    )
+
+
+def downgrade() -> None:
+    alembic.op.drop_column("catalogue_updates", "metadata_repo_commit")
+    alembic.op.add_column(
+        "catalogue_updates", sa.Column("metadata_repo_commit", sa.String)
+    )

cads_catalogue/contents.py

@@ -225,7 +225,13 @@ def yaml2context(yaml_path: str | pathlib.Path | None) -> dict[str, Any]:
         logger.warning(f"{yaml_path} not found. No variable substitution in templates.")
         return dict()
     with open(yaml_path) as fp:
-        data = yaml.load(fp.read(), Loader=yaml.loader.BaseLoader)
+        try:
+            data = yaml.load(fp.read(), Loader=yaml.loader.BaseLoader)
+        except Exception:
+            logger.exception(
+                f"{yaml_path} not parsable. No variable substitution in templates."
+            )
+            return dict()
     return data
 
 

cads_catalogue/database.py

@@ -61,7 +61,7 @@ class CatalogueUpdate(BaseModel):
         sa.types.DateTime(timezone=True), default=datetime.datetime.utcnow
     )
     catalogue_repo_commit = sa.Column(sa.String)
-    metadata_repo_commit = sa.Column(sa.String)
+    metadata_repo_commit = sa.Column(dialect_postgresql.JSONB, default={})
     licence_repo_commit = sa.Column(sa.String)
     message_repo_commit = sa.Column(sa.String)
     cim_repo_commit = sa.Column(sa.String)