From 446b3b109615068057a020ad9da170a2ba65f215 Mon Sep 17 00:00:00 2001 From: rebuilt Date: Tue, 15 Apr 2025 15:03:32 -0700 Subject: [PATCH] WIP: Update the parent table to hold a reference to a language Update the data uploader script to read the language from the csv and update the language information for any parent items that already exist (or create database entries if none already exist) --- app/models/housing.rb | 19 ++++++++ app/models/language.rb | 30 ++++++++++++ app/models/parent.rb | 2 + app/models/survey_item_response.rb | 4 ++ .../analyze/graph/column/parent/language.rb | 46 +++++++++++++++++++ .../analyze/graph/parents_by_language.rb | 42 +++++++++++++++++ app/services/survey_responses_data_loader.rb | 1 + .../20250415211114_add_language_to_parents.rb | 5 ++ db/schema.rb | 8 +++- spec/fixtures/demographic_glossary.csv | 45 +++++++++--------- spec/services/cleaner_spec.rb | 3 +- 11 files changed, 182 insertions(+), 23 deletions(-) create mode 100644 app/models/housing.rb create mode 100644 app/models/language.rb create mode 100644 app/presenters/analyze/graph/column/parent/language.rb create mode 100644 app/presenters/analyze/graph/parents_by_language.rb create mode 100644 db/migrate/20250415211114_add_language_to_parents.rb diff --git a/app/models/housing.rb b/app/models/housing.rb new file mode 100644 index 00000000..63411772 --- /dev/null +++ b/app/models/housing.rb @@ -0,0 +1,19 @@ +class Housing < ApplicationRecord + has_many :parents, dependent: :nullify + + def self.to_designation(housing) + return "Unknown" if housing.blank? + + housing = housing + case housing + in /^1$/i + "Own" + in /^2$/i + "Rent" + in /^99$|^100$/i + "Unknown" + else + "Unknown" + end + end +end diff --git a/app/models/language.rb b/app/models/language.rb new file mode 100644 index 00000000..a7c3cbe9 --- /dev/null +++ b/app/models/language.rb @@ -0,0 +1,30 @@ +class Language < ApplicationRecord + scope :by_designation, -> { all.map { |language| [language.designation, language] }.to_h } + has_many :parents, dependent: :nullify + + include FriendlyId + + friendly_id :designation, use: [:slugged] + def self.to_designation(language) + return "Unknown" if language.blank? + + case language + in /^1$|^1[^0]/i + "English" + in /^2$/i + "Portuguese" + in /^3$/i + "Spanish" + in /^99$|^100$/i + "Unknown" + else + puts "************************************" + puts "******** ERROR **********" + puts "" + puts "Error parsing Language column. '#{language}' is not a known value. Halting execution" + puts "" + puts "************************************" + exit + end + end +end diff --git a/app/models/parent.rb b/app/models/parent.rb index 2d556d15..7486bf7f 100644 --- a/app/models/parent.rb +++ b/app/models/parent.rb @@ -1,2 +1,4 @@ class Parent < ApplicationRecord + belongs_to :language, optional: true + belongs_to :housing, optional: true end diff --git a/app/models/survey_item_response.rb b/app/models/survey_item_response.rb index ed559983..d89eee5b 100644 --- a/app/models/survey_item_response.rb +++ b/app/models/survey_item_response.rb @@ -51,6 +51,10 @@ class SurveyItemResponse < ActiveRecord::Base ).where("student_races.race_id": race.id).group(:survey_item).having("count(*) >= 10").average(:likert_score) } + scope :averages_for_language, lambda { |survey_items, school, academic_year, language| + SurveyItemResponse.where(survey_item: survey_items, school:, + academic_year:, language:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score) + } def self.grouped_responses(school:, academic_year:) @grouped_responses ||= Hash.new do |memo, (school, academic_year)| memo[[school, academic_year]] = diff --git a/app/presenters/analyze/graph/column/parent/language.rb b/app/presenters/analyze/graph/column/parent/language.rb new file mode 100644 index 00000000..1680deb6 --- /dev/null +++ b/app/presenters/analyze/graph/column/parent/language.rb @@ -0,0 +1,46 @@ +module Analyze + module Graph + module Column + module Parent + class Language < ColumnBase + attr_reader :parent + + def initialize(parent:) + @parent = parent + end + + def label + ["#{parent.designation}"] + end + + def basis + "parent" + end + + def show_irrelevancy_message?(measure:) + false + end + + def show_insufficient_data_message?(measure:, school:, academic_years:) + false + end + + def type + :parent + end + + def n_size(measure:, school:, academic_year:) + SurveyItemResponse.where( survey_item: measure.parent_survey_items, school:, academic_year:), + academic_year:).select(:response_id).distinct.count + end + + def score(measure:, school:, academic_year:) + Score.new(average: 3, + meets_teacher_threshold: false, + meets_student_threshold:, + meets_admin_data_threshold: false) + end + end + end + end +end diff --git a/app/presenters/analyze/graph/parents_by_language.rb b/app/presenters/analyze/graph/parents_by_language.rb new file mode 100644 index 00000000..28ca7aa6 --- /dev/null +++ b/app/presenters/analyze/graph/parents_by_language.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Analyze + module Graph + class ParentsByLanguage + attr_reader :speds + + def initialize(speds:) + @speds = speds + end + + def to_s + "Parents by Language" + end + + def slug + "parents-by-language" + end + + def columns + [].tap do |array| + speds.each do |sped| + array << Analyze::Graph::Column::Sped.new(sped:) + end + array << Analyze::Graph::Column::AllStudent.new + end + end + + def source + Analyze::Source::SurveyData.new(slices: nil, graph: self) + end + + def slice + Analyze::Slice::StudentsByGroup.new(graph: self) + end + + def group + Analyze::Group::Base.new(name: "Special Education", slug: "sped", graph: self) + end + end + end +end diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index 5b0859d3..340e8bf9 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -108,6 +108,7 @@ class SurveyResponsesDataLoader if row.respondent_type == :parent parent = Parent.find_or_create_by(response_id: row.response_id) parent.number_of_children = row.number_of_children + parent.language = row.language parent.save end diff --git a/db/migrate/20250415211114_add_language_to_parents.rb b/db/migrate/20250415211114_add_language_to_parents.rb new file mode 100644 index 00000000..60519128 --- /dev/null +++ b/db/migrate/20250415211114_add_language_to_parents.rb @@ -0,0 +1,5 @@ +class AddLanguageToParents < ActiveRecord::Migration[8.0] + def change + add_reference :parents, :language, foreign_key: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 42071cfc..4c533464 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2025_01_15_011457) do +ActiveRecord::Schema[8.0].define(version: 2025_04_15_211114) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" enable_extension "pg_stat_statements" @@ -322,6 +322,10 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_15_011457) do t.integer "number_of_children" t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.bigint "housing_id", null: false + t.bigint "language_id" + t.index ["housing_id"], name: "index_parents_on_housing_id" + t.index ["language_id"], name: "index_parents_on_language_id" end create_table "races", force: :cascade do |t| @@ -510,6 +514,8 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_15_011457) do add_foreign_key "legacy_school_categories", "legacy_categories", column: "category_id" add_foreign_key "legacy_school_categories", "legacy_schools", column: "school_id" add_foreign_key "measures", "subcategories" + add_foreign_key "parents", "housings" + add_foreign_key "parents", "languages" add_foreign_key "respondents", "academic_years" add_foreign_key "respondents", "schools" add_foreign_key "response_rates", "academic_years" diff --git a/spec/fixtures/demographic_glossary.csv b/spec/fixtures/demographic_glossary.csv index 4ff277d8..beb32a2c 100644 --- a/spec/fixtures/demographic_glossary.csv +++ b/spec/fixtures/demographic_glossary.csv @@ -1,21 +1,24 @@ -ELL Value,ELL Type,ELL Headers,Sped Value,Sped Type,Sped Headers,Income Value,Income Type,Income Headers -EL student not 1st year,ELL,EL Student First Year,A,Special Education,Special Ed Status,Free Lunch,Economically Disadvantaged – Y,Low Income -"EL student, not 1st year",ELL,Raw ELL,active,Special Education,Raw SPED,Reduced Lunch,Economically Disadvantaged – Y,Raw Income -EL student 1st year,ELL,ELL- SIS,1,Special Education,SPED- SIS,LowIncome,Economically Disadvantaged – Y,SES- SIS -"EL student, 1st year",ELL,DirectCert,Special Education,Special Education,SPED,Low Income,Economically Disadvantaged – Y,EconDisadvantaged -EL - Early Child. or PK,ELL,ELL,Referred,Not Special Education,,Reduced price lunch,Economically Disadvantaged – Y,Income SIS -1,ELL,English Learner,Ineligible,Not Special Education,,TRUE,Economically Disadvantaged – Y,SES -lep student 1st year,ELL,,I,Not Special Education,,1,Economically Disadvantaged – Y,DirectCert -lep student not 1st year,ELL,,exited,Not Special Education,,Not Eligible,Economically Disadvantaged – N, -LEP Not1stYr,ELL,,0,Not Special Education,,FALSE,Economically Disadvantaged – N, -LEP1stYr US Sch,ELL,,Not Special Education,Not Special Education,,0,Economically Disadvantaged – N, -Does not apply,Not ELL,,Does not apply,Not Special Education,,[blanks],Economically Disadvantaged – N, -0,Not ELL,,[blanks],Not Special Education,,#NA,Unknown, -2,Not ELL,,#NA,Unknown,,NA,Unknown, -3,Not ELL,,NA,Unknown,,N/A,Unknown, -[blanks],Not ELL,,N/A,Unknown,,#N/A,Unknown, -#NA,Unknown,,#N/A,Unknown,,Income,Unknown, -NA,Unknown,,SPED,Unknown,,Yes,Economically Disadvantaged – Y, -N/A,Unknown,,No special needs,Not Special Education,,No,Economically Disadvantaged – N, -#N/A,Unknown,,,,,,, -ELL,ELL,,,,,,, +ELL Value,ELL Type,ELL Headers,Sped Value,Sped Type,Sped Headers,Income Value,Income Type,Income Headers,Language Value,Language Type +EL student not 1st year,ELL,EL Student First Year,A,Special Education,Special Ed Status,Free Lunch,Economically Disadvantaged – Y,Low Income,1,English +"EL student, not 1st year",ELL,Raw ELL,active,Special Education,Raw SPED,Reduced Lunch,Economically Disadvantaged – Y,Raw Income,10,English +EL student 1st year,ELL,ELL- SIS,1,Special Education,SPED- SIS,LowIncome,Economically Disadvantaged – Y,SES- SIS,2,Portuguese +"EL student, 1st year",ELL,DirectCert,Special Education,Special Education,SPED,Low Income,Economically Disadvantaged – Y,EconDisadvantaged,3,Spanish +EL - Early Child. or PK,ELL,ELL,Referred,Not Special Education,,Reduced price lunch,Economically Disadvantaged – Y,Income SIS,99,Unknown +1,ELL,English Learner,Ineligible,Not Special Education,,TRUE,Economically Disadvantaged – Y,SES ,100,Unknown +lep student 1st year,ELL,ELL Status- SIS,I,Not Special Education,,1,Economically Disadvantaged – Y,DirectCert,, +lep student not 1st year,ELL,,exited,Not Special Education,,Not Eligible,Economically Disadvantaged – N,,, +LEP Not1stYr,ELL,,0,Not Special Education,,FALSE,Economically Disadvantaged – N,,, +LEP1stYr US Sch,ELL,,Not Special Education,Not Special Education,,0,Economically Disadvantaged – N,,, +Does not apply,Not ELL,,Does not apply,Not Special Education,,[blanks],Economically Disadvantaged – N,,, +0,Not ELL,,[blanks],Not Special Education,,#NA,Unknown,,, +2,Not ELL,,#NA,Not Special Education,,NA,Unknown,,, +3,Not ELL,,NA,Not Special Education,,N/A,Unknown,,, +[blanks],Not ELL,,N/A,Not Special Education,,#N/A,Unknown,,, +#NA,Unknown,,#N/A,Not Special Education,,Income,Unknown,,, +NA,Unknown,,SPED,Unknown,,Yes,Economically Disadvantaged – Y,,, +N/A,Unknown,,No special needs,Not Special Education,,No,Economically Disadvantaged – N,,, +#N/A,Unknown,,Not SPED,Not Special Education,,,,,, +ELL,ELL,,,,,,,,, +LEP Not 1st Year,ELL,,,,,,,,, +Yes,ELL,,,,,,,,, +No,Not ELL,,,,,,,,, diff --git a/spec/services/cleaner_spec.rb b/spec/services/cleaner_spec.rb index cad0aab9..a0a5bf00 100644 --- a/spec/services/cleaner_spec.rb +++ b/spec/services/cleaner_spec.rb @@ -301,6 +301,7 @@ end def reads_headers_from_raw_csv(processed_data) processed_data in [headers, clean_csv, log_csv, data] + expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", "Finished", "RecordedDate", "ResponseId", "District", "School", "LASID", "Gender", "Race", "What grade are you in?", "s-tint-q1", @@ -314,7 +315,7 @@ def reads_headers_from_raw_csv(processed_data) "s-grit-q1", "s-grit-q2", "s-grit-q3", "s-grit-q4", "s-expa-q1", "s-poaf-q1", "s-poaf-q2", "s-poaf-q3", "s-poaf-q4", "s-tint-q1-1", "s-tint-q2-1", "s-tint-q3-1", "s-tint-q4-1", "s-tint-q5-1", "s-acpr-q1-1", "s-acpr-q2-1", "s-acpr-q3-1", "s-acpr-q4-1", "s-peff-q1-1", "s-peff-q2-1", "s-peff-q3-1", "s-peff-q4-1", - "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count"].to_set.sort + "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count", "Housing Status", "Raw Housing Status", "Home Language"].to_set.sort end def invalid_rows_are_rejected_for_the_correct_reasons(data)