Update verily/aou_synthetic, vumc/sdd underlay config files. (#325)

* Set maxHierarchyDepth for verily/cms_synpuf. * Update and re-index verily aou. Update sdd. * Restore dataflowTempLocation. * Restore sdd index dataset id.
DataBiosphere · Jan 19, 2023 · 4a6a65f · 4a6a65f
1 parent 0057850
commit 4a6a65f
Show file tree

Hide file tree

Showing 35 changed files with 1,091 additions and 62 deletions.
diff --git a/...thetic/expanded/verily_aou_synthetic.json → ...aou_synthetic/expanded/aou_synthetic.json b/...thetic/expanded/verily_aou_synthetic.json → ...aou_synthetic/expanded/aou_synthetic.json
@@ -1,10 +1,10 @@
 {
-  "name" : "verily_aou_synthetic",
+  "name" : "aou_synthetic",
   "dataPointers" : [ {
     "type" : "BQ_DATASET",
     "name" : "index_dataset",
     "projectId" : "verily-tanagra-dev",
-    "datasetId" : "aou_synthetic_indexes",
+    "datasetId" : "aousynthetic_index_011823",
     "queryProjectId" : "verily-tanagra-dev",
     "dataflowServiceAccountEmail" : "workflow-runner@verily-tanagra-dev.iam.gserviceaccount.com"
   }, {

diff --git a/service/src/main/resources/config/verily/aou_synthetic/expanded/entity/condition.json b/service/src/main/resources/config/verily/aou_synthetic/expanded/entity/condition.json
@@ -201,7 +201,8 @@
               "column" : "concept_id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 20
       }
     }
   },
@@ -291,7 +292,8 @@
               "column" : "id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 20
       }
     }
   }

diff --git a/service/src/main/resources/config/verily/aou_synthetic/expanded/entity/ingredient.json b/service/src/main/resources/config/verily/aou_synthetic/expanded/entity/ingredient.json
@@ -189,7 +189,8 @@
               "column" : "concept_id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 8
       }
     }
   },
@@ -279,7 +280,8 @@
               "column" : "id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 8
       }
     }
   }

diff --git a/service/src/main/resources/config/verily/aou_synthetic/expanded/entity/measurement.json b/service/src/main/resources/config/verily/aou_synthetic/expanded/entity/measurement.json
@@ -269,7 +269,8 @@
               "column" : "concept_id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 11
       }
     }
   },
@@ -359,7 +360,8 @@
               "column" : "id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 11
       }
     }
   }

diff --git a/service/src/main/resources/config/verily/aou_synthetic/expanded/entity/procedure.json b/service/src/main/resources/config/verily/aou_synthetic/expanded/entity/procedure.json
@@ -217,7 +217,8 @@
               "column" : "concept_id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 19
       }
     }
   },
@@ -307,7 +308,8 @@
               "column" : "id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 19
       }
     }
   }

diff --git a/service/src/main/resources/config/verily/aou_synthetic/original/aou_synthetic.json b/service/src/main/resources/config/verily/aou_synthetic/original/aou_synthetic.json
@@ -1,5 +1,5 @@
 {
-  "name": "verily_aou_synthetic",
+  "name": "aou_synthetic",
   "dataPointers": [
     { "type": "BQ_DATASET",
       "name": "omop_dataset",
@@ -9,7 +9,7 @@
     { "type": "BQ_DATASET",
       "name": "index_dataset",
       "projectId": "verily-tanagra-dev",
-      "datasetId": "aou_synthetic_indexes",
+      "datasetId": "aousynthetic_index_011823",
       "dataflowServiceAccountEmail": "workflow-runner@verily-tanagra-dev.iam.gserviceaccount.com" } ],
   "entities": [
     "brand.json",

diff --git a/service/src/main/resources/config/verily/aou_synthetic/original/entity/condition.json b/service/src/main/resources/config/verily/aou_synthetic/original/entity/condition.json
@@ -45,7 +45,8 @@
           "fieldPointers": {
             "id": { "column": "concept_id" }
           }
-        }
+        },
+        "maxHierarchyDepth" : 20
       }
     }
   },

diff --git a/service/src/main/resources/config/verily/aou_synthetic/original/entity/ingredient.json b/service/src/main/resources/config/verily/aou_synthetic/original/entity/ingredient.json
@@ -52,7 +52,8 @@
           "fieldPointers": {
             "id": { "column": "concept_id" }
           }
-        }
+        },
+        "maxHierarchyDepth" : 8
       }
     }
   },

diff --git a/service/src/main/resources/config/verily/aou_synthetic/original/entity/measurement.json b/service/src/main/resources/config/verily/aou_synthetic/original/entity/measurement.json
@@ -103,7 +103,8 @@
           "fieldPointers": {
             "id": { "column": "concept_id" }
           }
-        }
+        },
+        "maxHierarchyDepth": 11
       }
     }
   },

diff --git a/service/src/main/resources/config/verily/aou_synthetic/original/entity/procedure.json b/service/src/main/resources/config/verily/aou_synthetic/original/entity/procedure.json
@@ -45,7 +45,8 @@
           "fieldPointers": {
             "id": { "column": "concept_id" }
           }
-        }
+        },
+        "maxHierarchyDepth": 19
       }
     }
   },

diff --git a/service/src/main/resources/config/verily/cms_synpuf/expanded/entity/condition.json b/service/src/main/resources/config/verily/cms_synpuf/expanded/entity/condition.json
@@ -166,7 +166,7 @@
             }
           }
         },
-        "maxHierarchyDepth" : 64
+        "maxHierarchyDepth" : 20
       }
     }
   },
@@ -257,7 +257,7 @@
             }
           }
         },
-        "maxHierarchyDepth" : 64
+        "maxHierarchyDepth" : 20
       }
     }
   }

diff --git a/service/src/main/resources/config/verily/cms_synpuf/expanded/entity/ingredient.json b/service/src/main/resources/config/verily/cms_synpuf/expanded/entity/ingredient.json
@@ -178,7 +178,7 @@
             }
           }
         },
-        "maxHierarchyDepth" : 64
+        "maxHierarchyDepth" : 8
       }
     }
   },
@@ -269,7 +269,7 @@
             }
           }
         },
-        "maxHierarchyDepth" : 64
+        "maxHierarchyDepth" : 8
       }
     }
   }

diff --git a/service/src/main/resources/config/verily/cms_synpuf/expanded/entity/procedure.json b/service/src/main/resources/config/verily/cms_synpuf/expanded/entity/procedure.json
@@ -190,7 +190,7 @@
             }
           }
         },
-        "maxHierarchyDepth" : 64
+        "maxHierarchyDepth" : 19
       }
     }
   },
@@ -281,7 +281,7 @@
             }
           }
         },
-        "maxHierarchyDepth" : 64
+        "maxHierarchyDepth" : 19
       }
     }
   }

diff --git a/service/src/main/resources/config/verily/cms_synpuf/original/entity/condition.json b/service/src/main/resources/config/verily/cms_synpuf/original/entity/condition.json
@@ -45,7 +45,8 @@
           "fieldPointers": {
             "id": { "column": "concept_id" }
           }
-        }
+        },
+        "maxHierarchyDepth": 20
       }
     }
   },

diff --git a/service/src/main/resources/config/verily/cms_synpuf/original/entity/ingredient.json b/service/src/main/resources/config/verily/cms_synpuf/original/entity/ingredient.json
@@ -52,7 +52,8 @@
           "fieldPointers": {
             "id": { "column": "concept_id" }
           }
-        }
+        },
+        "maxHierarchyDepth": 8
       }
     }
   },

diff --git a/service/src/main/resources/config/verily/cms_synpuf/original/entity/procedure.json b/service/src/main/resources/config/verily/cms_synpuf/original/entity/procedure.json
@@ -45,7 +45,8 @@
           "fieldPointers": {
             "id": { "column": "concept_id" }
           }
-        }
+        },
+        "maxHierarchyDepth": 19
       }
     }
   },

diff --git a/service/src/main/resources/config/vumc/sdd/expanded/entity/brand.json b/service/src/main/resources/config/vumc/sdd/expanded/entity/brand.json
@@ -7,24 +7,24 @@
     "dataType" : "STRING",
     "displayHint" : {
       "type" : "ENUM",
-      "enumVals" : [ ]
+      "enumVals" : [ {
+        "enumVal" : {
+          "value" : {
+            "stringVal" : ""
+          },
+          "display" : "Unknown"
+        },
+        "count" : 108832
+      } ]
     }
   }, {
     "type" : "SIMPLE",
     "name" : "name",
-    "dataType" : "STRING",
-    "displayHint" : {
-      "type" : "ENUM",
-      "enumVals" : [ ]
-    }
+    "dataType" : "STRING"
   }, {
     "type" : "SIMPLE",
     "name" : "concept_code",
-    "dataType" : "STRING",
-    "displayHint" : {
-      "type" : "ENUM",
-      "enumVals" : [ ]
-    }
+    "dataType" : "STRING"
   }, {
     "type" : "SIMPLE",
     "name" : "id",
@@ -55,13 +55,6 @@
           "value" : {
             "stringVal" : "Brand Name"
           }
-        }, {
-          "type" : "BINARY",
-          "field" : {
-            "column" : "invalid_reason"
-          },
-          "operator" : "IS",
-          "value" : { }
         }, {
           "type" : "BOOLEAN_AND_OR",
           "operator" : "OR",
@@ -116,7 +109,7 @@
     "textSearchMapping" : {
       "searchStringTable" : {
         "tablePointer" : {
-          "rawSql" : "SELECT textsearch.id, textsearch.text FROM (      SELECT       c.concept_id AS id, c.concept_name AS text     FROM  `victr-tanagra-test.sd_static.concept` c      UNION ALL      SELECT       c.concept_id AS id, CAST(c.concept_id AS STRING) AS text     FROM  `victr-tanagra-test.sd_static.concept` c      UNION ALL      SELECT       c.concept_id AS id, c.concept_code AS text     FROM  `victr-tanagra-test.sd_static.concept` c ) AS textsearch  JOIN `victr-tanagra-test.sd_static.concept` c ON c.concept_id = textsearch.id  WHERE c.domain_id = 'Drug' AND c.concept_class_id = 'Brand Name' AND c.invalid_reason IS NULL AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension')"
+          "rawSql" : "SELECT textsearch.id, textsearch.text FROM (      SELECT       c.concept_id AS id, c.concept_name AS text     FROM  `victr-tanagra-test.sd_static.concept` c      UNION ALL      SELECT       c.concept_id AS id, CAST(c.concept_id AS STRING) AS text     FROM  `victr-tanagra-test.sd_static.concept` c      UNION ALL      SELECT       c.concept_id AS id, c.concept_code AS text     FROM  `victr-tanagra-test.sd_static.concept` c ) AS textsearch  JOIN `victr-tanagra-test.sd_static.concept` c ON c.concept_id = textsearch.id  WHERE c.domain_id = 'Drug' AND c.concept_class_id = 'Brand Name' AND c.vocabulary_id IN ('RxNorm', 'RxNorm Extension')"
         },
         "fieldPointers" : {
           "id" : {

diff --git a/service/src/main/resources/config/vumc/sdd/expanded/entity/condition.json b/service/src/main/resources/config/vumc/sdd/expanded/entity/condition.json
@@ -333,7 +333,8 @@
               "column" : "concept_id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 20
       }
     }
   },
@@ -420,7 +421,8 @@
               "column" : "id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 20
       }
     }
   }

diff --git a/service/src/main/resources/config/vumc/sdd/expanded/entity/ingredient.json b/service/src/main/resources/config/vumc/sdd/expanded/entity/ingredient.json
@@ -185,7 +185,8 @@
               "column" : "concept_id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 8
       }
     }
   },
@@ -272,7 +273,8 @@
               "column" : "id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 8
       }
     }
   }

diff --git a/service/src/main/resources/config/vumc/sdd/expanded/entity/measurement.json b/service/src/main/resources/config/vumc/sdd/expanded/entity/measurement.json
@@ -273,7 +273,8 @@
               "column" : "concept_id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 11
       }
     }
   },
@@ -360,7 +361,8 @@
               "column" : "id"
             }
           }
-        }
+        },
+        "maxHierarchyDepth" : 11
       }
     }
   }
-Original file line number
+Diff line change
@@ Expand Up / @@ -201,7 +201,8 @@ @@
                   "column" : "concept_id"
                 }
               }
-            }
+            },
+            "maxHierarchyDepth" : 20
           }
         }
       },
@@ Expand Down Expand Up / @@ -291,7 +292,8 @@ @@
                   "column" : "id"
                 }
               }
-            }
+            },
+            "maxHierarchyDepth" : 20
           }
         }
       }
@@ Expand Down @@