feat: create a parents by language graph

Update demographics table with lanugage options

Create a lanugage table to hold the new languages

Update the demographic loader to input languages into the database

Update the cleaner to read the language column

Update the parent table to hold a reference to a language

Update the data uploader script to read the language from the csv and update the language information for any parent items that already exist (or create database entries if none already exist)

update the analyze interface to add controls for selecting ‘parents by group’ and a dropdown for ‘parent by language’

Update the analyze controller to read the parent-by-group parameter

Create a graph for the parent-by-group view

Bubble up averages for language calculations.

Make sure n-size only counts responses for a given measure.

app/models/housing.rb

@@ -1,5 +1,6 @@
 class Housing < ApplicationRecord
   has_many :parents, dependent: :nullify
+  scope :by_designation, -> { all.map { |housing| [housing.designation, housing] }.to_h }
 
   def self.to_designation(housing)
     return "Unknown" if housing.blank?

app/models/language.rb

@@ -1,12 +1,13 @@
 class Language < ApplicationRecord
   scope :by_designation, -> { all.map { |language| [language.designation, language] }.to_h }
-  has_many :parents, dependent: :nullify
+  has_many :parent_languages, dependent: :destroy
+  has_many :parents, through: :parent_languages, dependent: :nullify
 
   include FriendlyId
 
   friendly_id :designation, use: [:slugged]
   def self.to_designation(language)
-    return "Unknown" if language.blank?
+    return "Prefer not to disclose" if language.blank?
 
     case language
     in /^1$/i

app/presenters/analyze/graph/column/language.rb

@@ -32,6 +32,8 @@ module Analyze
         end
 
         def score(measure:, school:, academic_year:)
+          return Score::NIL_SCORE if n_size(measure:, school:, academic_year:) < 10
+
           averages = SurveyItemResponse.averages_for_language(measure.parent_survey_items, school, academic_year,
                                                          designations)
           average = bubble_up_averages(measure:, averages:).round(2)

app/presenters/analyze/graph/column/parent/language.rb

@@ -1,46 +0,0 @@
-module Analyze
-  module Graph
-    module Column
-      module Parent
-      class Language < ColumnBase
-        attr_reader :parent
-
-        def initialize(parent:)
-          @parent = parent
-        end
-
-        def label
-          ["#{parent.designation}"]
-        end
-
-        def basis
-          "parent"
-        end
-
-        def show_irrelevancy_message?(measure:)
-          false
-        end
-
-        def show_insufficient_data_message?(measure:, school:, academic_years:)
-          false
-        end
-
-        def type
-          :parent
-        end
-
-        def n_size(measure:, school:, academic_year:)
-          SurveyItemResponse.where( survey_item: measure.parent_survey_items, school:,  academic_year:),
-                                   academic_year:).select(:response_id).distinct.count
-        end
-
-        def score(measure:, school:, academic_year:)
-          Score.new(average: 3,
-                    meets_teacher_threshold: false,
-                    meets_student_threshold:,
-                    meets_admin_data_threshold: false)
-        end
-      end
-    end
-  end
-end

app/presenters/analyze/graph/parents_by_language.rb

@@ -23,6 +23,7 @@ module Analyze
           array << Analyze::Graph::Column::Language.new(languages: ENGLISH_LANGUAGES, label: ["English", "Speaking"])
           array << Analyze::Graph::Column::Language.new(languages: NON_ENGLISH_LANGUAGES, label: ["Non English", "Speaking"])
           array << Analyze::Graph::Column::Language.new(languages: UNKNOWN_LANGUAGES, label: ["Unknown"])
+          array << Analyze::Graph::Column::Language.new(languages: ALL_LANGUAGES, label: ["All", "Parents"])
         end
       end
 
@@ -31,11 +32,11 @@ module Analyze
       end
 
       def slice
-        Analyze::Slice::StudentsByGroup.new(graph: self)
+        Analyze::Slice::ParentsByGroup.new(graph: self)
       end
 
       def group
-        Analyze::Group::Base.new(name: "Special Education", slug: "sped", graph: self)
+        Analyze::Group::Base.new(name: "Language", slug: "language", graph: self)
       end
     end
   end

app/presenters/analyze/slice/parents_by_group.rb

@@ -0,0 +1,9 @@
+module Analyze
+  module Slice
+    class ParentsByGroup < Base
+      def initialize(graph:, label: "Parents by Group", slug: "parents-by-group")
+        super(label:, slug:, graph:)
+      end
+    end
+  end
+end

app/services/cleaner.rb

@@ -79,7 +79,7 @@ class Cleaner
 
     headers = headers.to_set
     headers = headers.merge(Set.new(["Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count",
-                                     "Race", "Gender", "Raw Housing Status", "Housing Status", "Home Language"])).to_a
+                                     "Race", "Gender", "Raw Housing Status", "Housing Status", "Home Language", "Home Languages"])).to_a
     filtered_headers = include_all_headers(headers:)
     filtered_headers = remove_unwanted_headers(headers: filtered_headers)
     log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?",

app/services/survey_item_values.rb

@@ -171,7 +171,7 @@ class SurveyItemValues
   end
 
   def lasid
-    @lasid ||= value_from(pattern: /LASID/i)
+    @lasid ||= value_from(pattern: /LASID/i) || ""
   end
 
   def raw_income
@@ -207,7 +207,7 @@ class SurveyItemValues
   end
 
   def raw_language
-    @raw_language ||= value_from(pattern: /^Language$/i)
+    @raw_language ||= value_from(pattern: /^Language$/i) || ""
   end
 
   def languages
@@ -234,6 +234,9 @@ class SurveyItemValues
       output ||= row[match]&.strip
     end
 
+    output = output.delete("\u0000") if output.present?
+    output = output.delete("\x00") if output.present?
+    output.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') if output.present?
     output
   end
 

app/services/survey_responses_data_loader.rb

@@ -99,24 +99,27 @@ class SurveyResponsesDataLoader
     @languages ||= Language.by_designation
   end
 
+  def housings
+    @housings ||= Housing.by_designation
+  end
+
   def process_survey_items(row:)
     student = nil
     parent = nil
     if row.respondent_type == :student
       student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid)
       student.races.delete_all
-      tmp_races = row.races.map { |race| races[race] }
+      tmp_races = row.races.map { |race| races[race] }.reject(&:nil?)
       student.races += tmp_races
     end
 
     if row.respondent_type == :parent
       parent = Parent.find_or_create_by(response_id: row.response_id)
       parent.number_of_children = row.number_of_children
-      tmp_languages = row.languages.map { |language| languages[language] }
+      tmp_languages = row.languages.map { |language| languages[language] }.reject(&:nil?)
-      parent.housing_id = Housing.find_by(designation: row.housing).id
       parent.languages.delete_all
       parent.languages.concat(tmp_languages)
-      parent.housing = Housing.find_by(designation: row.housing)
+      parent.housing = housings[row.housing] if row.housing.present?
       parent.save
     end
 

db/migrate/20250408000620_create_housings.rb

@@ -0,0 +1,10 @@
+class CreateHousings < ActiveRecord::Migration[8.0]
+  def change
+    create_table :housings do |t|
+      t.string :designation
+      t.string :slug
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20250408212201_add_housing_to_parent.rb

@@ -1,7 +1,5 @@
 class AddHousingToParent < ActiveRecord::Migration[8.0]
   def change
     add_reference :parents, :housing, foreign_key: true
-    Parent.update_all(housing_id: Housing.find_by(designation: 'Unknown').id)
-    change_column_null :parents, :housing_id, false
   end
 end

db/schema.rb

@@ -89,6 +89,13 @@ ActiveRecord::Schema[8.0].define(version: 2025_04_18_185655) do
     t.index ["slug"], name: "index_genders_on_slug", unique: true
   end
 
+  create_table "housings", force: :cascade do |t|
+    t.string "designation"
+    t.string "slug"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "incomes", force: :cascade do |t|
     t.string "designation"
     t.datetime "created_at", null: false

spec/fixtures/demographic_glossary.csv

@@ -11,10 +11,10 @@ LEP Not1stYr,ELL,,0,Not Special Education,,FALSE,Economically Disadvantaged –
 LEP1stYr US Sch,ELL,,Not Special Education,Not Special Education,,0,Economically Disadvantaged – N,,,
 Does not apply,Not ELL,,Does not apply,Not Special Education,,[blanks],Economically Disadvantaged – N,,,
 0,Not ELL,,[blanks],Not Special Education,,#NA,Unknown,,,
-2,Not ELL,,#NA,Not Special Education,,NA,Unknown,,,
+2,Not ELL,,#NA,Unknown,,NA,Unknown,,,
-3,Not ELL,,NA,Not Special Education,,N/A,Unknown,,,
+3,Not ELL,,NA,Unknown,,N/A,Unknown,,,
-[blanks],Not ELL,,N/A,Not Special Education,,#N/A,Unknown,,,
+[blanks],Not ELL,,N/A,Unknown,,#N/A,Unknown,,,
-#NA,Unknown,,#N/A,Not Special Education,,Income,Unknown,,,
+#NA,Unknown,,#N/A,Unknown,,Income,Unknown,,,
 NA,Unknown,,SPED,Unknown,,Yes,Economically Disadvantaged – Y,,,
 N/A,Unknown,,No special needs,Not Special Education,,No,Economically Disadvantaged – N,,,
 #N/A,Unknown,,Not SPED,Not Special Education,,,,,,

spec/services/cleaner_spec.rb

@@ -315,7 +315,7 @@ def reads_headers_from_raw_csv(processed_data)
                                      "s-grit-q1", "s-grit-q2", "s-grit-q3", "s-grit-q4", "s-expa-q1", "s-poaf-q1", "s-poaf-q2", "s-poaf-q3",
                                      "s-poaf-q4", "s-tint-q1-1", "s-tint-q2-1", "s-tint-q3-1", "s-tint-q4-1", "s-tint-q5-1", "s-acpr-q1-1",
                                      "s-acpr-q2-1", "s-acpr-q3-1", "s-acpr-q4-1", "s-peff-q1-1", "s-peff-q2-1", "s-peff-q3-1", "s-peff-q4-1",
-                                     "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count", "Housing Status", "Raw Housing Status", "Home Language"].to_set.sort
+                                     "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count", "Housing Status", "Raw Housing Status", "Home Language", "Home Languages"].to_set.sort
 end
 
 def invalid_rows_are_rejected_for_the_correct_reasons(data)