2332 transition illuminate dbt sources to pull from dlt tables

src/dbt/kipptaf/dbt_project.yml

@@ -144,11 +144,15 @@ models:
             group: google_appsheet
     illuminate:
       +schema: illuminate
-      +materialized: table
       staging:
+        archive:
+          +enabled: false
         repositories:
+          +materialized: table
           archive:
             +enabled: false
+      intermediate:
+        +materialized: table
     instagram:
       +enabled: false
       business:

src/dbt/kipptaf/macros/illuminate.sql

@@ -1,28 +1,38 @@
-{% macro illuminate_repository_unpivot(model_name) %}
+{%- macro illuminate_repository_unpivot(model_name) -%}
+    {%- set repository_id = model_name | replace(
+        "stg_illuminate__dna_repositories__repository_", ""
+    ) -%}
+    {%- set relation = source(
+        "illuminate_dna_repositories", "repository_" + repository_id
+    ) -%}
+
+    {%- set cols = adapter.get_columns_in_relation(relation) -%}
+
     with
         dbt_unpivot as (
-            {{
-                dbt_utils.unpivot(
-                    relation=source(
-                        "illuminate", model_name | replace("stg_illuminate__", "")
-                    ),
-                    cast_to="string",
-                    exclude=[
-                        "_fivetran_deleted",
-                        "_fivetran_synced",
-                        "repository_row_id",
-                        "student_id",
-                    ],
-                )
-            }}
+            {% if cols %}
+                {{
+                    dbt_utils.unpivot(
+                        relation=relation,
+                        cast_to="string",
+                        exclude=["repository_row_id", "student_id"],
+                    )
+                }}
+            {% else %}
+                select
+                    cast(null as int) as repository_row_id,
+                    cast(null as int) as student_id,
+                    cast(null as string) as field_name,
+                    cast(null as string) as `value`,
+            {% endif %}
         )
 
     select
-        * except (_fivetran_deleted, _fivetran_synced),
+        repository_row_id,
+        student_id,
+        field_name,
+        `value`,
 
-        safe_cast(
-            regexp_extract('{{ model_name }}', r'repository_(\d+)') as int
-        ) as repository_id,
+        {{ repository_id }} as repository_id,
     from dbt_unpivot
-    where not _fivetran_deleted
-{% endmacro %}
+{%- endmacro -%}

src/dbt/kipptaf/models/assessments/intermediate/int_assessments__scaffold.sql

@@ -75,7 +75,7 @@ with
             a.region,
         from assessment_region_scaffold as a
         inner join
-            {{ ref("stg_illuminate__assessment_grade_levels") }} as agl
+            {{ ref("stg_illuminate__dna_assessments__assessment_grade_levels") }} as agl
             on a.assessment_id = agl.assessment_id
         where a.academic_year < 2025
     ),
@@ -125,7 +125,8 @@ with
             and a.grade_level_id = ssa.grade_level_id
             and ssa.rn_student_session_desc = 1
         inner join
-            {{ ref("stg_illuminate__students") }} as s on ssa.student_id = s.student_id
+            {{ ref("stg_illuminate__public__students") }} as s
+            on ssa.student_id = s.student_id
         inner join
             {{ ref("int_assessments__course_enrollments") }} as ce
             on a.subject_area = ce.illuminate_subject_area
@@ -164,7 +165,7 @@ with
             and a.region = ce.region
             and a.administered_at between ce.cc_dateenrolled and ce.cc_dateleft
         inner join
-            {{ ref("stg_illuminate__students") }} as s
+            {{ ref("stg_illuminate__public__students") }} as s
             on ce.powerschool_student_number = s.local_student_id
             and ce.illuminate_grade_level_id >= 10
     )
@@ -193,7 +194,7 @@ select
     false as is_replacement,
 from internal_assessments as ia
 left join
-    {{ ref("stg_illuminate__students_assessments") }} as sa
+    {{ ref("stg_illuminate__dna_assessments__students_assessments") }} as sa
     on ia.illuminate_student_id = sa.student_id
     and ia.assessment_id = sa.assessment_id
 
@@ -228,9 +229,10 @@ select
     true as is_replacement,
 from {{ ref("int_assessments__assessments") }} as a
 inner join
-    {{ ref("stg_illuminate__students_assessments") }} as sa
+    {{ ref("stg_illuminate__dna_assessments__students_assessments") }} as sa
     on a.assessment_id = sa.assessment_id
-inner join {{ ref("stg_illuminate__students") }} as s on sa.student_id = s.student_id
+inner join
+    {{ ref("stg_illuminate__public__students") }} as s on sa.student_id = s.student_id
 inner join
     {{ ref("int_illuminate__student_session_aff") }} as ssa
     on sa.student_id = ssa.student_id
@@ -273,9 +275,10 @@ select
     false as is_replacement,
 from {{ ref("int_assessments__assessments") }} as a
 inner join
-    {{ ref("stg_illuminate__students_assessments") }} as sa
+    {{ ref("stg_illuminate__dna_assessments__students_assessments") }} as sa
     on a.assessment_id = sa.assessment_id
-inner join {{ ref("stg_illuminate__students") }} as s on sa.student_id = s.student_id
+inner join
+    {{ ref("stg_illuminate__public__students") }} as s on sa.student_id = s.student_id
 inner join
     {{ ref("int_illuminate__student_session_aff") }} as ssa
     on a.academic_year = ssa.academic_year

src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__ddi_audit.sql

@@ -1,7 +1,7 @@
 with
     grade_level as (
         select assessment_id, min(grade_level) as grade_level,
-        from {{ ref("stg_illuminate__assessment_grade_levels") }}
+        from {{ ref("stg_illuminate__dna_assessments__assessment_grade_levels") }}
         group by assessment_id
     )
 
@@ -29,9 +29,11 @@ select
 from {{ ref("int_assessments__assessments") }} as a
 left join grade_level as g on a.assessment_id = g.assessment_id
 left join
-    {{ ref("stg_illuminate__assessment_standards") }} as st
+    {{ ref("stg_illuminate__dna_assessments__assessment_standards") }} as st
     on a.assessment_id = st.assessment_id
-left join {{ ref("stg_illuminate__standards") }} as s on st.standard_id = s.standard_id
+left join
+    {{ ref("stg_illuminate__standards__standards") }} as s
+    on st.standard_id = s.standard_id
 left join
     {{ ref("stg_reporting__terms") }} as rt
     on a.administered_at between rt.start_date and rt.end_date
@@ -80,9 +82,11 @@ select
 from {{ ref("int_assessments__assessments") }} as a
 left join grade_level as g on a.assessment_id = g.assessment_id
 left join
-    {{ ref("stg_illuminate__assessment_standards") }} as st
+    {{ ref("stg_illuminate__dna_assessments__assessment_standards") }} as st
     on a.assessment_id = st.assessment_id
-left join {{ ref("stg_illuminate__standards") }} as s on st.standard_id = s.standard_id
+left join
+    {{ ref("stg_illuminate__standards__standards") }} as s
+    on st.standard_id = s.standard_id
 left join
     {{ ref("stg_reporting__terms") }} as rt
     on a.administered_at between rt.start_date and rt.end_date
@@ -94,6 +98,6 @@ left join
     and rt.name = ps.term_name
     and a.subject_area = ps.illuminate_subject_area
 where
-    a.scope in ('HS Unit Quiz', 'Midterm or Final', 'Power Standard Pre Quiz')
-    and a.academic_year_clean = {{ var("current_academic_year") }}
+    a.academic_year_clean = {{ var("current_academic_year") }}
+    and a.scope in ('HS Unit Quiz', 'Midterm or Final', 'Power Standard Pre Quiz')
     and g.grade_level > 8

src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__sight_words_dashboard.sql

@@ -1,12 +1,38 @@
-select
-    r.repository_id,
-    r.title,
-    r.date_administered,
-
-    rt.name as term_name,
-    rt.academic_year,
+with
+    swq as (
+        select
+            r.repository_id,
+            r.title,
+            r.date_administered,
+
+            rt.name as term_name,
+            rt.academic_year,
+
+            f.label,
+
+            g.grade_level,
+        from {{ ref("int_illuminate__repositories") }} as r
+        inner join
+            {{ ref("stg_reporting__terms") }} as rt
+            on r.date_administered between rt.start_date and rt.end_date
+            and rt.type = 'RT'
+            and rt.school_id = 0
+        inner join
+            {{ ref("stg_illuminate__dna_repositories__repository_fields") }} as f
+            on r.repository_id = f.repository_id
+        inner join
+            {{ ref("stg_illuminate__dna_repositories__repository_grade_levels") }} as g
+            on r.repository_id = g.repository_id
+        where r.scope = 'Sight Words Quiz'
+    )
 
-    f.label as sight_word,
+select
+    swq.repository_id,
+    swq.title,
+    swq.date_administered,
+    swq.term_name,
+    swq.academic_year,
+    swq.label as sight_word,
 
     co.student_number,
     co.student_name,
@@ -21,58 +47,33 @@ select
     co.gifted_and_talented,
     co.gender,
     co.ethnicity,
+    co.nj_student_tier,
+    co.hos,
 
     sw.value,
 
-    nj.nj_student_tier,
-
-    hos.head_of_school_preferred_name_lastfirst as hos,
-
     false as is_replacement,
-from {{ ref("int_illuminate__repositories") }} as r
-inner join
-    {{ ref("stg_reporting__terms") }} as rt
-    on r.date_administered between rt.start_date and rt.end_date
-    and rt.type = 'RT'
-    and rt.school_id = 0
+from swq
 inner join
-    {{ ref("stg_illuminate__repository_fields") }} as f
-    on r.repository_id = f.repository_id
-inner join
-    {{ ref("stg_illuminate__repository_grade_levels") }} as g
-    on r.repository_id = g.repository_id
-inner join
-    {{ ref("int_extracts__student_enrollments") }} as co
-    on g.grade_level = co.grade_level
-    and rt.academic_year = co.academic_year
-    and co.is_enrolled_recent
+    {{ ref("int_extracts__student_enrollments_subjects") }} as co
+    on swq.academic_year = co.academic_year
+    and swq.grade_level = co.grade_level
+    and co.iready_subject = 'Reading'
 left join
     {{ ref("int_illuminate__repository_data") }} as sw
-    on co.student_number = sw.local_student_id
-    and r.repository_id = sw.repository_id
-    and f.label = sw.field_label
-left join
-    {{ ref("int_extracts__student_enrollments_subjects") }} as nj
-    on co.academic_year = nj.academic_year
-    and co.student_number = nj.student_number
-    and {{ union_dataset_join_clause(left_alias="co", right_alias="nj") }}
-    and nj.iready_subject = 'Reading'
-left join
-    {{ ref("int_people__leadership_crosswalk") }} as hos
-    on co.schoolid = hos.home_work_location_powerschool_school_id
-where r.scope = 'Sight Words Quiz'
+    on swq.repository_id = sw.repository_id
+    and swq.label = sw.field_label
+    and co.student_number = sw.local_student_id
 
 union all
 
 select
-    r.repository_id,
-    r.title,
-    r.date_administered,
-
-    rt.name as term_name,
-    rt.academic_year,
-
-    f.label as sight_word,
+    swq.repository_id,
+    swq.title,
+    swq.date_administered,
+    swq.term_name,
+    swq.academic_year,
+    swq.label as sight_word,
 
     co.student_number,
     co.student_name,
@@ -87,43 +88,21 @@ select
     co.gifted_and_talented,
     co.gender,
     co.ethnicity,
+    co.nj_student_tier,
+    co.hos,
 
     sw.value,
 
-    nj.nj_student_tier,
-
-    hos.head_of_school_preferred_name_lastfirst as hos,
-
     true as is_replacement,
-from {{ ref("int_illuminate__repositories") }} as r
-inner join
-    {{ ref("stg_reporting__terms") }} as rt
-    on r.date_administered between rt.start_date and rt.end_date
-    and rt.type = 'RT'
-    and rt.school_id = 0
-inner join
-    {{ ref("stg_illuminate__repository_fields") }} as f
-    on r.repository_id = f.repository_id
+from swq
 inner join
-    {{ ref("stg_illuminate__repository_grade_levels") }} as g
-    on r.repository_id = g.repository_id
+    {{ ref("int_illuminate__repository_data") }} as sw
+    on swq.repository_id = sw.repository_id
+    and swq.label = sw.field_label
 inner join
-    {{ ref("int_extracts__student_enrollments") }} as co
-    on g.grade_level != co.grade_level
-    and rt.academic_year = co.academic_year
+    {{ ref("int_extracts__student_enrollments_subjects") }} as co
+    on swq.academic_year = co.academic_year
+    and swq.grade_level != co.grade_level
+    and sw.local_student_id = co.student_number
     and co.is_enrolled_recent
-inner join
-    {{ ref("int_illuminate__repository_data") }} as sw
-    on co.student_number = sw.local_student_id
-    and r.repository_id = sw.repository_id
-    and f.label = sw.field_label
-left join
-    {{ ref("int_extracts__student_enrollments_subjects") }} as nj
-    on co.academic_year = nj.academic_year
-    and co.student_number = nj.student_number
-    and {{ union_dataset_join_clause(left_alias="co", right_alias="nj") }}
-    and nj.iready_subject = 'Reading'
-left join
-    {{ ref("int_people__leadership_crosswalk") }} as hos
-    on co.schoolid = hos.home_work_location_powerschool_school_id
-where r.scope = 'Sight Words Quiz'
+    and co.iready_subject = 'Reading'

src/dbt/kipptaf/models/illuminate/intermediate/int_illuminate__agg_student_responses.sql

@@ -3,7 +3,9 @@ with
         {{
             dbt_utils.union_relations(
                 relations=[
-                    ref("stg_illuminate__agg_student_responses_overall"),
+                    ref(
+                        "stg_illuminate__dna_assessments__agg_student_responses_overall"
+                    ),
                     ref("int_illuminate__agg_student_responses_standard"),
                     ref("int_illuminate__agg_student_responses_group"),
                 ]
@@ -29,10 +31,12 @@ select
     ur.performance_band_set_id,
     ur.custom_code as response_type_code,
     ur.root_standard_description as response_type_root_description,
+
     regexp_extract(ur._dbt_source_relation, r'_([a-z]+)`$') as response_type,
+
     coalesce(ur.standard_id, ur.reporting_group_id) as response_type_id,
     coalesce(ur.standard_description, ur.label) as response_type_description,
     coalesce(ur.points_possible, ur.raw_score_possible) as points_possible,
     coalesce(ur.points, ur.raw_score) as points,
-from {{ ref("stg_illuminate__students_assessments") }} as sa
+from {{ ref("stg_illuminate__dna_assessments__students_assessments") }} as sa
 inner join union_relations as ur on sa.student_assessment_id = ur.student_assessment_id