TDOE#174: Refactor Student Disability Models

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Unreleased
 ## New features
+
 ## Under the hood
+ - Breaking change of grain in `stg_ef3__stu_ed_org__disabilities` and `stg_ef3__stu_spec_ed__disabilities` where double-flattening `designations` was multiplying rows per disability entry. Designations are now passed forward as `v_designations` (nested variant) instead.
+ - Add missing columns to disability sub-tables: `school_year` on `stg_ef3__stu_ed_org__disabilities`; `api_year`, `school_year`, `k_student_program`, and `ed_org_id` on `stg_ef3__stu_spec_ed__disabilities`
 ## Fixes
 
 # edu_edfi_source v0.6.5

models/staging/edfi_3/stage/stg_ef3__stu_ed_org__disabilities.sql

@@ -5,6 +5,7 @@ flattened as (
     select
         tenant_code,
         api_year,
+        school_year,
         k_student,
         k_student_xyear,
         ed_org_id,
@@ -14,12 +15,8 @@ flattened as (
         {{ extract_descriptor('disab.value:disabilityDeterminationSourceTypeDescriptor::string') }} as disability_source_type,
         disab.value:disabilityDiagnosis::string as disability_diagnosis,
         disab.value:orderOfDisability::int as order_of_disability,
-        -- todo: perhaps these would better serve as wide booleans
-        -- in which case we would not want to double-flatten, but leave nested
-        -- for a downstream step
-        {{ extract_descriptor('desig.value:disabilityDesignationDescriptor::string') }} as disability_designation
+        disab.value:designations as v_designations
     from stg_stu_ed_org
         {{ json_flatten('v_disabilities', 'disab') }}
-        {{ json_flatten('disab.value:designations', 'desig') }}
 )
 select * from flattened

models/staging/edfi_3/stage/stg_ef3__stu_spec_ed__disabilities.sql

@@ -5,14 +5,17 @@ flattened as (
     select
         k_student_program,
         tenant_code,
+        api_year,
+        school_year,
+        k_student_program,
         k_student,
         k_student_xyear,
-        k_program,
+        ed_org_id,
         k_lea,
         k_school,
+        k_program,
         program_enroll_begin_date,
         program_enroll_end_date,
-        school_year,
         {{ extract_descriptor('disab.value:disabilityDescriptor::string') }} as disability_type,
         {{ extract_descriptor('disab.value:disabilityDeterminationSourceTypeDescriptor::string') }} as disability_source_type,
         disab.value:disabilityDiagnosis::string as disability_diagnosis,