diff --git a/app/models/housing.rb b/app/models/housing.rb index 63411772..b930a547 100644 --- a/app/models/housing.rb +++ b/app/models/housing.rb @@ -1,5 +1,6 @@ class Housing < ApplicationRecord has_many :parents, dependent: :nullify + scope :by_designation, -> { all.map { |housing| [housing.designation, housing] }.to_h } def self.to_designation(housing) return "Unknown" if housing.blank? diff --git a/app/models/language.rb b/app/models/language.rb index 07113348..71f5e992 100644 --- a/app/models/language.rb +++ b/app/models/language.rb @@ -1,12 +1,13 @@ class Language < ApplicationRecord scope :by_designation, -> { all.map { |language| [language.designation, language] }.to_h } - has_many :parents, dependent: :nullify + has_many :parent_languages, dependent: :destroy + has_many :parents, through: :parent_languages, dependent: :nullify include FriendlyId friendly_id :designation, use: [:slugged] def self.to_designation(language) - return "Unknown" if language.blank? + return "Prefer not to disclose" if language.blank? case language in /^1$/i diff --git a/app/presenters/analyze/graph/column/language.rb b/app/presenters/analyze/graph/column/language.rb index 2f4526bb..68d6915d 100644 --- a/app/presenters/analyze/graph/column/language.rb +++ b/app/presenters/analyze/graph/column/language.rb @@ -32,6 +32,8 @@ module Analyze end def score(measure:, school:, academic_year:) + return Score::NIL_SCORE if n_size(measure:, school:, academic_year:) < 10 + averages = SurveyItemResponse.averages_for_language(measure.parent_survey_items, school, academic_year, designations) average = bubble_up_averages(measure:, averages:).round(2) diff --git a/app/presenters/analyze/graph/column/parent/language.rb b/app/presenters/analyze/graph/column/parent/language.rb deleted file mode 100644 index 1680deb6..00000000 --- a/app/presenters/analyze/graph/column/parent/language.rb +++ /dev/null @@ -1,46 +0,0 @@ -module Analyze - module Graph - module Column - module Parent - class Language < ColumnBase - attr_reader :parent - - def initialize(parent:) - @parent = parent - end - - def label - ["#{parent.designation}"] - end - - def basis - "parent" - end - - def show_irrelevancy_message?(measure:) - false - end - - def show_insufficient_data_message?(measure:, school:, academic_years:) - false - end - - def type - :parent - end - - def n_size(measure:, school:, academic_year:) - SurveyItemResponse.where( survey_item: measure.parent_survey_items, school:, academic_year:), - academic_year:).select(:response_id).distinct.count - end - - def score(measure:, school:, academic_year:) - Score.new(average: 3, - meets_teacher_threshold: false, - meets_student_threshold:, - meets_admin_data_threshold: false) - end - end - end - end -end diff --git a/app/presenters/analyze/graph/parents_by_language.rb b/app/presenters/analyze/graph/parents_by_language.rb index 7dde24fd..06079db8 100644 --- a/app/presenters/analyze/graph/parents_by_language.rb +++ b/app/presenters/analyze/graph/parents_by_language.rb @@ -23,6 +23,7 @@ module Analyze array << Analyze::Graph::Column::Language.new(languages: ENGLISH_LANGUAGES, label: ["English", "Speaking"]) array << Analyze::Graph::Column::Language.new(languages: NON_ENGLISH_LANGUAGES, label: ["Non English", "Speaking"]) array << Analyze::Graph::Column::Language.new(languages: UNKNOWN_LANGUAGES, label: ["Unknown"]) + array << Analyze::Graph::Column::Language.new(languages: ALL_LANGUAGES, label: ["All", "Parents"]) end end @@ -31,11 +32,11 @@ module Analyze end def slice - Analyze::Slice::StudentsByGroup.new(graph: self) + Analyze::Slice::ParentsByGroup.new(graph: self) end def group - Analyze::Group::Base.new(name: "Special Education", slug: "sped", graph: self) + Analyze::Group::Base.new(name: "Language", slug: "language", graph: self) end end end diff --git a/app/presenters/analyze/slice/parents_by_group.rb b/app/presenters/analyze/slice/parents_by_group.rb new file mode 100644 index 00000000..f97d93f7 --- /dev/null +++ b/app/presenters/analyze/slice/parents_by_group.rb @@ -0,0 +1,9 @@ +module Analyze + module Slice + class ParentsByGroup < Base + def initialize(graph:, label: "Parents by Group", slug: "parents-by-group") + super(label:, slug:, graph:) + end + end + end +end diff --git a/app/services/cleaner.rb b/app/services/cleaner.rb index 6133da74..08877c90 100644 --- a/app/services/cleaner.rb +++ b/app/services/cleaner.rb @@ -79,7 +79,7 @@ class Cleaner headers = headers.to_set headers = headers.merge(Set.new(["Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count", - "Race", "Gender", "Raw Housing Status", "Housing Status", "Home Language"])).to_a + "Race", "Gender", "Raw Housing Status", "Housing Status", "Home Language", "Home Languages"])).to_a filtered_headers = include_all_headers(headers:) filtered_headers = remove_unwanted_headers(headers: filtered_headers) log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?", diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index dab65992..0e739883 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -171,7 +171,7 @@ class SurveyItemValues end def lasid - @lasid ||= value_from(pattern: /LASID/i) + @lasid ||= value_from(pattern: /LASID/i) || "" end def raw_income @@ -207,7 +207,7 @@ class SurveyItemValues end def raw_language - @raw_language ||= value_from(pattern: /^Language$/i) + @raw_language ||= value_from(pattern: /^Language$/i) || "" end def languages @@ -234,6 +234,9 @@ class SurveyItemValues output ||= row[match]&.strip end + output = output.delete("\u0000") if output.present? + output = output.delete("\x00") if output.present? + output.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') if output.present? output end diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index 03f1e250..a0d5c36a 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -99,24 +99,27 @@ class SurveyResponsesDataLoader @languages ||= Language.by_designation end + def housings + @housings ||= Housing.by_designation + end + def process_survey_items(row:) student = nil parent = nil if row.respondent_type == :student student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid) student.races.delete_all - tmp_races = row.races.map { |race| races[race] } + tmp_races = row.races.map { |race| races[race] }.reject(&:nil?) student.races += tmp_races end if row.respondent_type == :parent parent = Parent.find_or_create_by(response_id: row.response_id) parent.number_of_children = row.number_of_children - tmp_languages = row.languages.map { |language| languages[language] } - parent.housing_id = Housing.find_by(designation: row.housing).id + tmp_languages = row.languages.map { |language| languages[language] }.reject(&:nil?) parent.languages.delete_all parent.languages.concat(tmp_languages) - parent.housing = Housing.find_by(designation: row.housing) + parent.housing = housings[row.housing] if row.housing.present? parent.save end diff --git a/db/migrate/20250408000620_create_housings.rb b/db/migrate/20250408000620_create_housings.rb new file mode 100644 index 00000000..10eef4ad --- /dev/null +++ b/db/migrate/20250408000620_create_housings.rb @@ -0,0 +1,10 @@ +class CreateHousings < ActiveRecord::Migration[8.0] + def change + create_table :housings do |t| + t.string :designation + t.string :slug + + t.timestamps + end + end +end diff --git a/db/migrate/20250408212201_add_housing_to_parent.rb b/db/migrate/20250408212201_add_housing_to_parent.rb index 0f4261f1..ed297cf3 100644 --- a/db/migrate/20250408212201_add_housing_to_parent.rb +++ b/db/migrate/20250408212201_add_housing_to_parent.rb @@ -1,7 +1,5 @@ class AddHousingToParent < ActiveRecord::Migration[8.0] def change add_reference :parents, :housing, foreign_key: true - Parent.update_all(housing_id: Housing.find_by(designation: 'Unknown').id) - change_column_null :parents, :housing_id, false end end diff --git a/db/schema.rb b/db/schema.rb index 84afd42a..4c7504e4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -89,6 +89,13 @@ ActiveRecord::Schema[8.0].define(version: 2025_04_18_185655) do t.index ["slug"], name: "index_genders_on_slug", unique: true end + create_table "housings", force: :cascade do |t| + t.string "designation" + t.string "slug" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "incomes", force: :cascade do |t| t.string "designation" t.datetime "created_at", null: false diff --git a/spec/fixtures/demographic_glossary.csv b/spec/fixtures/demographic_glossary.csv index 9e112f8b..22036964 100644 --- a/spec/fixtures/demographic_glossary.csv +++ b/spec/fixtures/demographic_glossary.csv @@ -11,10 +11,10 @@ LEP Not1stYr,ELL,,0,Not Special Education,,FALSE,Economically Disadvantaged – LEP1stYr US Sch,ELL,,Not Special Education,Not Special Education,,0,Economically Disadvantaged – N,,, Does not apply,Not ELL,,Does not apply,Not Special Education,,[blanks],Economically Disadvantaged – N,,, 0,Not ELL,,[blanks],Not Special Education,,#NA,Unknown,,, -2,Not ELL,,#NA,Not Special Education,,NA,Unknown,,, -3,Not ELL,,NA,Not Special Education,,N/A,Unknown,,, -[blanks],Not ELL,,N/A,Not Special Education,,#N/A,Unknown,,, -#NA,Unknown,,#N/A,Not Special Education,,Income,Unknown,,, +2,Not ELL,,#NA,Unknown,,NA,Unknown,,, +3,Not ELL,,NA,Unknown,,N/A,Unknown,,, +[blanks],Not ELL,,N/A,Unknown,,#N/A,Unknown,,, +#NA,Unknown,,#N/A,Unknown,,Income,Unknown,,, NA,Unknown,,SPED,Unknown,,Yes,Economically Disadvantaged – Y,,, N/A,Unknown,,No special needs,Not Special Education,,No,Economically Disadvantaged – N,,, #N/A,Unknown,,Not SPED,Not Special Education,,,,,, diff --git a/spec/services/cleaner_spec.rb b/spec/services/cleaner_spec.rb index a0a5bf00..6b7a137c 100644 --- a/spec/services/cleaner_spec.rb +++ b/spec/services/cleaner_spec.rb @@ -315,7 +315,7 @@ def reads_headers_from_raw_csv(processed_data) "s-grit-q1", "s-grit-q2", "s-grit-q3", "s-grit-q4", "s-expa-q1", "s-poaf-q1", "s-poaf-q2", "s-poaf-q3", "s-poaf-q4", "s-tint-q1-1", "s-tint-q2-1", "s-tint-q3-1", "s-tint-q4-1", "s-tint-q5-1", "s-acpr-q1-1", "s-acpr-q2-1", "s-acpr-q3-1", "s-acpr-q4-1", "s-peff-q1-1", "s-peff-q2-1", "s-peff-q3-1", "s-peff-q4-1", - "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count", "Housing Status", "Raw Housing Status", "Home Language"].to_set.sort + "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count", "Housing Status", "Raw Housing Status", "Home Language", "Home Languages"].to_set.sort end def invalid_rows_are_rejected_for_the_correct_reasons(data)