From b0df611f4dbf7bcc98846b72c35952b788a8ac87 Mon Sep 17 00:00:00 2001 From: rebuilt Date: Fri, 11 Apr 2025 14:37:18 -0700 Subject: [PATCH] feat: create a parents by language graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update demographics table with lanugage options Create a lanugage table to hold the new languages Update the demographic loader to input languages into the database Update the cleaner to read the language column Update the parent table to hold a reference to a language Update the data uploader script to read the language from the csv and update the language information for any parent items that already exist (or create database entries if none already exist) update the analyze interface to add controls for selecting ‘parents by group’ and a dropdown for ‘parent by language’ Update the analyze controller to read the parent-by-group parameter Create a graph for the parent-by-group view Bubble up averages for language calculations. Make sure n-size only counts responses for a given measure. --- app/models/housing.rb | 1 + app/models/language.rb | 5 +- .../analyze/graph/column/language.rb | 2 + .../analyze/graph/column/parent/language.rb | 46 ------------------- .../analyze/graph/parents_by_language.rb | 5 +- .../analyze/slice/parents_by_group.rb | 9 ++++ app/services/cleaner.rb | 2 +- app/services/survey_item_values.rb | 7 ++- app/services/survey_responses_data_loader.rb | 11 +++-- db/migrate/20250408000620_create_housings.rb | 10 ++++ .../20250408212201_add_housing_to_parent.rb | 2 - db/schema.rb | 7 +++ spec/fixtures/demographic_glossary.csv | 8 ++-- spec/services/cleaner_spec.rb | 2 +- 14 files changed, 53 insertions(+), 64 deletions(-) delete mode 100644 app/presenters/analyze/graph/column/parent/language.rb create mode 100644 app/presenters/analyze/slice/parents_by_group.rb create mode 100644 db/migrate/20250408000620_create_housings.rb diff --git a/app/models/housing.rb b/app/models/housing.rb index 63411772..b930a547 100644 --- a/app/models/housing.rb +++ b/app/models/housing.rb @@ -1,5 +1,6 @@ class Housing < ApplicationRecord has_many :parents, dependent: :nullify + scope :by_designation, -> { all.map { |housing| [housing.designation, housing] }.to_h } def self.to_designation(housing) return "Unknown" if housing.blank? diff --git a/app/models/language.rb b/app/models/language.rb index 07113348..71f5e992 100644 --- a/app/models/language.rb +++ b/app/models/language.rb @@ -1,12 +1,13 @@ class Language < ApplicationRecord scope :by_designation, -> { all.map { |language| [language.designation, language] }.to_h } - has_many :parents, dependent: :nullify + has_many :parent_languages, dependent: :destroy + has_many :parents, through: :parent_languages, dependent: :nullify include FriendlyId friendly_id :designation, use: [:slugged] def self.to_designation(language) - return "Unknown" if language.blank? + return "Prefer not to disclose" if language.blank? case language in /^1$/i diff --git a/app/presenters/analyze/graph/column/language.rb b/app/presenters/analyze/graph/column/language.rb index 2f4526bb..68d6915d 100644 --- a/app/presenters/analyze/graph/column/language.rb +++ b/app/presenters/analyze/graph/column/language.rb @@ -32,6 +32,8 @@ module Analyze end def score(measure:, school:, academic_year:) + return Score::NIL_SCORE if n_size(measure:, school:, academic_year:) < 10 + averages = SurveyItemResponse.averages_for_language(measure.parent_survey_items, school, academic_year, designations) average = bubble_up_averages(measure:, averages:).round(2) diff --git a/app/presenters/analyze/graph/column/parent/language.rb b/app/presenters/analyze/graph/column/parent/language.rb deleted file mode 100644 index 1680deb6..00000000 --- a/app/presenters/analyze/graph/column/parent/language.rb +++ /dev/null @@ -1,46 +0,0 @@ -module Analyze - module Graph - module Column - module Parent - class Language < ColumnBase - attr_reader :parent - - def initialize(parent:) - @parent = parent - end - - def label - ["#{parent.designation}"] - end - - def basis - "parent" - end - - def show_irrelevancy_message?(measure:) - false - end - - def show_insufficient_data_message?(measure:, school:, academic_years:) - false - end - - def type - :parent - end - - def n_size(measure:, school:, academic_year:) - SurveyItemResponse.where( survey_item: measure.parent_survey_items, school:, academic_year:), - academic_year:).select(:response_id).distinct.count - end - - def score(measure:, school:, academic_year:) - Score.new(average: 3, - meets_teacher_threshold: false, - meets_student_threshold:, - meets_admin_data_threshold: false) - end - end - end - end -end diff --git a/app/presenters/analyze/graph/parents_by_language.rb b/app/presenters/analyze/graph/parents_by_language.rb index 7dde24fd..06079db8 100644 --- a/app/presenters/analyze/graph/parents_by_language.rb +++ b/app/presenters/analyze/graph/parents_by_language.rb @@ -23,6 +23,7 @@ module Analyze array << Analyze::Graph::Column::Language.new(languages: ENGLISH_LANGUAGES, label: ["English", "Speaking"]) array << Analyze::Graph::Column::Language.new(languages: NON_ENGLISH_LANGUAGES, label: ["Non English", "Speaking"]) array << Analyze::Graph::Column::Language.new(languages: UNKNOWN_LANGUAGES, label: ["Unknown"]) + array << Analyze::Graph::Column::Language.new(languages: ALL_LANGUAGES, label: ["All", "Parents"]) end end @@ -31,11 +32,11 @@ module Analyze end def slice - Analyze::Slice::StudentsByGroup.new(graph: self) + Analyze::Slice::ParentsByGroup.new(graph: self) end def group - Analyze::Group::Base.new(name: "Special Education", slug: "sped", graph: self) + Analyze::Group::Base.new(name: "Language", slug: "language", graph: self) end end end diff --git a/app/presenters/analyze/slice/parents_by_group.rb b/app/presenters/analyze/slice/parents_by_group.rb new file mode 100644 index 00000000..f97d93f7 --- /dev/null +++ b/app/presenters/analyze/slice/parents_by_group.rb @@ -0,0 +1,9 @@ +module Analyze + module Slice + class ParentsByGroup < Base + def initialize(graph:, label: "Parents by Group", slug: "parents-by-group") + super(label:, slug:, graph:) + end + end + end +end diff --git a/app/services/cleaner.rb b/app/services/cleaner.rb index 6133da74..08877c90 100644 --- a/app/services/cleaner.rb +++ b/app/services/cleaner.rb @@ -79,7 +79,7 @@ class Cleaner headers = headers.to_set headers = headers.merge(Set.new(["Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count", - "Race", "Gender", "Raw Housing Status", "Housing Status", "Home Language"])).to_a + "Race", "Gender", "Raw Housing Status", "Housing Status", "Home Language", "Home Languages"])).to_a filtered_headers = include_all_headers(headers:) filtered_headers = remove_unwanted_headers(headers: filtered_headers) log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?", diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index dab65992..0e739883 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -171,7 +171,7 @@ class SurveyItemValues end def lasid - @lasid ||= value_from(pattern: /LASID/i) + @lasid ||= value_from(pattern: /LASID/i) || "" end def raw_income @@ -207,7 +207,7 @@ class SurveyItemValues end def raw_language - @raw_language ||= value_from(pattern: /^Language$/i) + @raw_language ||= value_from(pattern: /^Language$/i) || "" end def languages @@ -234,6 +234,9 @@ class SurveyItemValues output ||= row[match]&.strip end + output = output.delete("\u0000") if output.present? + output = output.delete("\x00") if output.present? + output.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') if output.present? output end diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index 03f1e250..a0d5c36a 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -99,24 +99,27 @@ class SurveyResponsesDataLoader @languages ||= Language.by_designation end + def housings + @housings ||= Housing.by_designation + end + def process_survey_items(row:) student = nil parent = nil if row.respondent_type == :student student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid) student.races.delete_all - tmp_races = row.races.map { |race| races[race] } + tmp_races = row.races.map { |race| races[race] }.reject(&:nil?) student.races += tmp_races end if row.respondent_type == :parent parent = Parent.find_or_create_by(response_id: row.response_id) parent.number_of_children = row.number_of_children - tmp_languages = row.languages.map { |language| languages[language] } - parent.housing_id = Housing.find_by(designation: row.housing).id + tmp_languages = row.languages.map { |language| languages[language] }.reject(&:nil?) parent.languages.delete_all parent.languages.concat(tmp_languages) - parent.housing = Housing.find_by(designation: row.housing) + parent.housing = housings[row.housing] if row.housing.present? parent.save end diff --git a/db/migrate/20250408000620_create_housings.rb b/db/migrate/20250408000620_create_housings.rb new file mode 100644 index 00000000..10eef4ad --- /dev/null +++ b/db/migrate/20250408000620_create_housings.rb @@ -0,0 +1,10 @@ +class CreateHousings < ActiveRecord::Migration[8.0] + def change + create_table :housings do |t| + t.string :designation + t.string :slug + + t.timestamps + end + end +end diff --git a/db/migrate/20250408212201_add_housing_to_parent.rb b/db/migrate/20250408212201_add_housing_to_parent.rb index 0f4261f1..ed297cf3 100644 --- a/db/migrate/20250408212201_add_housing_to_parent.rb +++ b/db/migrate/20250408212201_add_housing_to_parent.rb @@ -1,7 +1,5 @@ class AddHousingToParent < ActiveRecord::Migration[8.0] def change add_reference :parents, :housing, foreign_key: true - Parent.update_all(housing_id: Housing.find_by(designation: 'Unknown').id) - change_column_null :parents, :housing_id, false end end diff --git a/db/schema.rb b/db/schema.rb index 84afd42a..4c7504e4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -89,6 +89,13 @@ ActiveRecord::Schema[8.0].define(version: 2025_04_18_185655) do t.index ["slug"], name: "index_genders_on_slug", unique: true end + create_table "housings", force: :cascade do |t| + t.string "designation" + t.string "slug" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "incomes", force: :cascade do |t| t.string "designation" t.datetime "created_at", null: false diff --git a/spec/fixtures/demographic_glossary.csv b/spec/fixtures/demographic_glossary.csv index 9e112f8b..22036964 100644 --- a/spec/fixtures/demographic_glossary.csv +++ b/spec/fixtures/demographic_glossary.csv @@ -11,10 +11,10 @@ LEP Not1stYr,ELL,,0,Not Special Education,,FALSE,Economically Disadvantaged – LEP1stYr US Sch,ELL,,Not Special Education,Not Special Education,,0,Economically Disadvantaged – N,,, Does not apply,Not ELL,,Does not apply,Not Special Education,,[blanks],Economically Disadvantaged – N,,, 0,Not ELL,,[blanks],Not Special Education,,#NA,Unknown,,, -2,Not ELL,,#NA,Not Special Education,,NA,Unknown,,, -3,Not ELL,,NA,Not Special Education,,N/A,Unknown,,, -[blanks],Not ELL,,N/A,Not Special Education,,#N/A,Unknown,,, -#NA,Unknown,,#N/A,Not Special Education,,Income,Unknown,,, +2,Not ELL,,#NA,Unknown,,NA,Unknown,,, +3,Not ELL,,NA,Unknown,,N/A,Unknown,,, +[blanks],Not ELL,,N/A,Unknown,,#N/A,Unknown,,, +#NA,Unknown,,#N/A,Unknown,,Income,Unknown,,, NA,Unknown,,SPED,Unknown,,Yes,Economically Disadvantaged – Y,,, N/A,Unknown,,No special needs,Not Special Education,,No,Economically Disadvantaged – N,,, #N/A,Unknown,,Not SPED,Not Special Education,,,,,, diff --git a/spec/services/cleaner_spec.rb b/spec/services/cleaner_spec.rb index a0a5bf00..6b7a137c 100644 --- a/spec/services/cleaner_spec.rb +++ b/spec/services/cleaner_spec.rb @@ -315,7 +315,7 @@ def reads_headers_from_raw_csv(processed_data) "s-grit-q1", "s-grit-q2", "s-grit-q3", "s-grit-q4", "s-expa-q1", "s-poaf-q1", "s-poaf-q2", "s-poaf-q3", "s-poaf-q4", "s-tint-q1-1", "s-tint-q2-1", "s-tint-q3-1", "s-tint-q4-1", "s-tint-q5-1", "s-acpr-q1-1", "s-acpr-q2-1", "s-acpr-q3-1", "s-acpr-q4-1", "s-peff-q1-1", "s-peff-q2-1", "s-peff-q3-1", "s-peff-q4-1", - "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count", "Housing Status", "Raw Housing Status", "Home Language"].to_set.sort + "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count", "Housing Status", "Raw Housing Status", "Home Language", "Home Languages"].to_set.sort end def invalid_rows_are_rejected_for_the_correct_reasons(data)