Centralized jinja template entrypoint (#177)

* Centralized jinja template entrypoint

* SQL linting fix

* sqlfluff bug

* sqlfluff bug

cumulus_library/.sqlfluff

@@ -17,7 +17,7 @@ capitalisation_policy = upper
 
 
 [sqlfluff:templater:jinja]
-load_macros_from_path = cumulus_library/studies/core/core_templates
+load_macros_from_path = cumulus_library/template_sql/shared_macros,cumulus_library/studies/core/core_templates
 
 [sqlfluff:templater:jinja:context]
 code_systems = ["http://snomed.info/sct", "http://hl7.org/fhir/sid/icd-10-cm"]

cumulus_library/statistics/counts.py

@@ -5,8 +5,8 @@
 
 from cumulus_library.base_table_builder import BaseTableBuilder
 from cumulus_library.errors import CountsBuilderError
+from cumulus_library.statistics.statistics_templates import counts_templates
 from cumulus_library.study_parser import StudyManifestParser
-from cumulus_library.template_sql.statistics import counts_templates
 
 
 class CountsBuilder(BaseTableBuilder):
@@ -71,7 +71,7 @@ def get_count_query(
         :keyword min_subject: An integer setting the minimum bin size for inclusion
             (default: 10)
         :keyword fhir_resource: The type of FHIR resource to count (see
-            template_sql/templates.statistics.CountableFhirResource)
+            statistics/statistics_templates/count_templates.CountableFhirResource)
         """
         if not table_name or not source_table or not table_cols:
             raise CountsBuilderError(

cumulus_library/statistics/psm.py

@@ -17,8 +17,8 @@
 from psmpy.functions import cohenD
 
 from cumulus_library import base_table_builder, databases
+from cumulus_library.statistics.statistics_templates import psm_templates
 from cumulus_library.template_sql import base_templates
-from cumulus_library.template_sql.statistics import psm_templates
 
 
 @dataclass

cumulus_library/template_sql/statistics/count.sql.jinja → cumulus_library/statistics/statistics_templates/count.sql.jinja

@@ -1,9 +1,4 @@
-{%- macro comma_delineate(loop) -%}
-{%- if not loop.last -%}
-, 
-{%- endif -%}
-{%- endmacro -%}
-
+{%- import 'syntax.sql.jinja' as syntax -%}
 {#- TODO: move secondary/tertiary resource declaration upstream to
 statistics/counts.py level -#}
 {%- macro secondary_resource(fhir_resource) -%}
@@ -30,7 +25,7 @@ logic at the statistics/counts.py level -#}
 {%- macro cols_delineated_list(table_cols, fhir_resource)-%}
             {% for col in table_cols %}
             "{{ column_or_alias(col) }}"
-            {{- comma_delineate(loop) }}
+            {{- syntax.comma_delineate(loop) }}
             {%- endfor %}
             {%- if fhir_resource in ('documentreference', 'observation' ) -%},
             enc_class_display
@@ -42,7 +37,7 @@ which will drop nulls, resulting in potentially non-unique keys -#}
 {%- macro cols_coalesced_list(table_cols, fhir_resource)-%}
                 {% for col in table_cols %}
                 COALESCE("{{ column_or_alias(col) }}",'')
-                {{- comma_delineate(loop) }}
+                {{- syntax.comma_delineate(loop) }}
                 {%- endfor %}
                 {%- if fhir_resource in ('documentreference', 'observation' ) -%},
                 COALESCE(enc_class_display,'')
@@ -90,7 +85,7 @@ CREATE TABLE {{ table_name }} AS (
             {%- else %}
             s."{{ col.name }}" AS {{ col.alias }}
             {%- endif -%}
-            {{- comma_delineate(loop) -}}
+            {{- syntax.comma_delineate(loop) -}}
             {% endfor %}
             --noqa: enable=RF03, AL02
         FROM {{ source_table }} AS s
@@ -137,7 +132,7 @@ CREATE TABLE {{ table_name }} AS (
                 cast({{ column_or_alias(col) }} AS varchar),
                 '{{ missing_null }}'
             ) AS {{ column_or_alias(col) }}
-            {{- comma_delineate(loop) }}
+            {{- syntax.comma_delineate(loop) }}
             {%- endfor -%}
         {%- if filter_resource %}
         FROM filtered_table
@@ -205,7 +200,7 @@ CREATE TABLE {{ table_name }} AS (
         {%- endif %}
         {%- for col in table_cols %}
         p."{{ column_or_alias(col) }}"
-        {{- comma_delineate(loop) }}
+        {{- syntax.comma_delineate(loop) }}
         {%- endfor %}
         {%- if fhir_resource in ('documentreference', 'observation' ) -%},
         p.enc_class_display

cumulus_library/template_sql/statistics/counts_templates.py → cumulus_library/statistics/statistics_templates/counts_templates.py

@@ -2,9 +2,8 @@
 from enum import Enum
 from pathlib import Path
 
-from jinja2 import Template
-
 from cumulus_library.errors import CountsBuilderError
+from cumulus_library.template_sql import base_templates
 
 
 class CountableFhirResource(Enum):
@@ -62,15 +61,16 @@ def get_count_query(
             )
     table_cols = table_col_classed
 
-    with open(f"{path}/count.sql.jinja") as count_query:
-        query = Template(count_query.read()).render(
-            table_name=table_name,
-            source_table=source_table,
-            table_cols=table_cols,
-            min_subject=min_subject,
-            where_clauses=where_clauses,
-            fhir_resource=fhir_resource,
-            filter_resource=filter_resource,
-        )
-        # workaround for conflicting sqlfluff enforcement
-        return query.replace("-- noqa: disable=LT02\n", "")
+    query = base_templates.get_base_template(
+        "count",
+        path,
+        table_name=table_name,
+        source_table=source_table,
+        table_cols=table_cols,
+        min_subject=min_subject,
+        where_clauses=where_clauses,
+        fhir_resource=fhir_resource,
+        filter_resource=filter_resource,
+    )
+    # workaround for conflicting sqlfluff enforcement
+    return query.replace("-- noqa: disable=LT02\n", "")

cumulus_library/template_sql/statistics/psm_templates.py → cumulus_library/statistics/statistics_templates/psm_templates.py

@@ -2,9 +2,8 @@
 
 from pathlib import Path
 
-from jinja2 import Template
-
 from cumulus_library.errors import CumulusLibraryError
+from cumulus_library.template_sql import base_templates
 
 
 def get_distinct_ids(
@@ -33,14 +32,14 @@ def get_distinct_ids(
             "to be defined if supplied",
         )
 
-    path = Path(__file__).parent
-    with open(f"{path}/psm_distinct_ids.sql.jinja") as distinct_ids:
-        return Template(distinct_ids.read()).render(
-            columns=columns,
-            source_table=source_table,
-            join_id=join_id,
-            filter_table=filter_table,
-        )
+    return base_templates.get_base_template(
+        "psm_distinct_ids",
+        Path(__file__).parent,
+        columns=columns,
+        source_table=source_table,
+        join_id=join_id,
+        filter_table=filter_table,
+    )
 
 
 def get_create_covariate_table(
@@ -74,16 +73,16 @@ def get_create_covariate_table(
             "to be defined if supplied",
         )
 
-    path = Path(__file__).parent
-    with open(f"{path}/psm_create_covariate_table.sql.jinja") as create_covariate_table:
-        return Template(create_covariate_table.read()).render(
-            target_table=target_table,
-            pos_source_table=pos_source_table,
-            neg_source_table=neg_source_table,
-            table_suffix=table_suffix,
-            primary_ref=primary_ref,
-            dependent_variable=dependent_variable,
-            count_ref=count_ref,
-            count_table=count_table,
-            join_cols_by_table=join_cols_by_table,
-        )
+    return base_templates.get_base_template(
+        "psm_create_covariate_table",
+        Path(__file__).parent,
+        target_table=target_table,
+        pos_source_table=pos_source_table,
+        neg_source_table=neg_source_table,
+        table_suffix=table_suffix,
+        primary_ref=primary_ref,
+        dependent_variable=dependent_variable,
+        count_ref=count_ref,
+        count_table=count_table,
+        join_cols_by_table=join_cols_by_table,
+    )

cumulus_library/studies/core/core_templates/core_templates.py

@@ -1,7 +1,5 @@
 import pathlib
 
-import jinja2
-
 from cumulus_library.template_sql import base_templates
 
 PATH = pathlib.Path(__file__).parent
@@ -13,11 +11,9 @@ def get_core_template(
     config: dict | None = None,
 ) -> str:
     """Extracts code system details as a standalone table"""
-    with open(f"{PATH}/{target_table}.sql.jinja") as file:
-        core_template = file.read()
-        loader = jinja2.FileSystemLoader(PATH)
-        env = jinja2.Environment(loader=loader).from_string(core_template)
-        return env.render(schema=schema, config=config)
+    return base_templates.get_base_template(
+        target_table, path=pathlib.Path(__file__).parent, schema=schema, config=config
+    )
 
 
 def validate_schema(cursor: object, schema: str, expected_table_cols, parser):

cumulus_library/studies/core/core_templates/core_utils.jinja

@@ -1,3 +1,4 @@
+{% import 'syntax.sql.jinja' as syntax %}
 {#- A collection of macros for constructing common safe column selects
 
     For layout, is is generally assumed that these are being invoked from within
@@ -9,7 +10,7 @@
 {% macro basic_cols(table, alias, targets) %}
         {%- for col in targets %}
         {{ alias }}.{{ col }}
-        {{- comma_delineate(loop) }}
+        {{- syntax.comma_delineate(loop) }}
         {%- endfor %}
 {%- endmacro %}
 
@@ -43,7 +44,7 @@ targets is an array expecting a data type of the following:
         {%- else %}
         NULL AS {{ col }}
         {%- endif %}        
-        {{- comma_delineate(loop) }}
+        {{- syntax.comma_delineate(loop) }}
         {%- endfor %}
 {%- endmacro %}
 
@@ -75,7 +76,7 @@ targets is an array expecting a data type of the following:
         {%- else %}
         cast(NULL AS date) AS {{ col.lower() }}
         {%- endif %}    
-        {{- comma_delineate(loop) }}
+        {{- syntax.comma_delineate(loop) }}
         {%- endfor %}
 {%- endmacro %}
 
@@ -110,16 +111,7 @@ targets is assumed to be a list of tuples of one of the following format:
         {%- else %}
         cast(NULL AS date) AS {{ col[0] }}_{{ col[1] }}
         {%- endif %}    
-        {{- comma_delineate(loop) }}
+        {{- syntax.comma_delineate(loop) }}
         {%- endfor %}
 {%- endmacro %}
 
-{#- convenience function for looping over items
-
-TODO: this should move to some kind of global jinja utils location -#}
-{%- macro comma_delineate(loop) -%}
-{%- if not loop.last -%}
-, 
-{%- endif -%}
-{%- endmacro -%}
-