feat: create a parents by language graph

Update demographics table with lanugage options

Create a lanugage table to hold the new languages

Update the demographic loader to input languages into the database

Update the cleaner to read the language column

Update the parent table to hold a reference to a language

Update the data uploader script to read the language from the csv and update the language information for any parent items that already exist (or create database entries if none already exist)

update the analyze interface to add controls for selecting ‘parents by group’ and a dropdown for ‘parent by language’

Update the analyze controller to read the parent-by-group parameter

Create a graph for the parent-by-group view

Bubble up averages for language calculations.

Make sure n-size only counts responses for a given measure.

app/services/cleaner.rb

@@ -79,7 +79,7 @@ class Cleaner
 
     headers = headers.to_set
     headers = headers.merge(Set.new(["Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd", "Progress Count",
-                                     "Race", "Gender", "Raw Housing Status", "Housing Status"])).to_a
+                                     "Race", "Gender", "Raw Housing Status", "Housing Status", "Home Language", "Home Languages"])).to_a
     filtered_headers = include_all_headers(headers:)
     filtered_headers = remove_unwanted_headers(headers: filtered_headers)
     log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?",

app/services/demographic_loader.rb

@@ -9,6 +9,7 @@ class DemographicLoader
       create_from_column(column: "ELL", row:, model: Ell)
       create_from_column(column: "Special Ed Status", row:, model: Sped)
       create_from_column(column: "Housing", row:, model: Housing)
+      create_from_column(column: "Language", row:, model: Language)
     end
   end
 

app/services/survey_item_values.rb

@@ -22,6 +22,7 @@ class SurveyItemValues
     row["Gender"] ||= gender
     row["Raw Housing Status"] = raw_housing
     row["Housing Status"] = housing
+    row["Home Languages"] = languages.join(",")
 
     copy_data_to_main_column(main: /Race/i, secondary: /Race Secondary|Race-1/i)
     copy_data_to_main_column(main: /Gender/i, secondary: /Gender Secondary|Gender-1/i)
@@ -161,7 +162,7 @@ class SurveyItemValues
       # Only check the secondary hispanic column if we don't have self reported data and are relying on SIS data
       if self_report.nil? && sis.present?
         hispanic = value_from(pattern: /Hispanic\s*Latino/i)&.downcase
-        race_codes = race_codes.reject { |code| code == 5 } if hispanic == "true" && race_codes.count == 1
+        race_codes = race_codes.reject { |code| code == 5 } if ["true", "1"].include?(hispanic) || race_codes.count == 1
         race_codes = race_codes.push(4) if %w[true 1].include?(hispanic)
       end
 
@@ -170,7 +171,7 @@ class SurveyItemValues
   end
 
   def lasid
-    @lasid ||= value_from(pattern: /LASID/i)
+    @lasid ||= value_from(pattern: /LASID/i) || ""
   end
 
   def raw_income
@@ -205,6 +206,20 @@ class SurveyItemValues
     @housing ||= Housing.to_designation(raw_housing)
   end
 
+  def raw_language
+    @raw_language ||= value_from(pattern: /^Language$/i) || ""
+  end
+
+  def languages
+    @languages ||= [].tap do |languages|
+      if raw_language.present?
+        raw_language.split(",").each do |item|
+          languages << Language.to_designation(item)
+        end
+      end
+    end
+  end
+
   def number_of_children
     @number_of_children ||= value_from(pattern: /Number\s*Of\s*Children/i).to_i
   end
@@ -219,6 +234,9 @@ class SurveyItemValues
       output ||= row[match]&.strip
     end
 
+    output = output.delete("\u0000") if output.present?
+    output = output.delete("\x00") if output.present?
+    output.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') if output.present?
     output
   end
 

app/services/survey_responses_data_loader.rb

@@ -86,19 +86,31 @@ class SurveyResponsesDataLoader
     process_survey_items(row:)
   end
 
+  def languages
+    @languages ||= Language.by_designation
+  end
+
+  def housings
+    @housings ||= Housing.by_designation
+  end
+
   def process_survey_items(row:)
     student = nil
     parent = nil
     if row.respondent_type == :student
       student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid)
       student.races.delete_all
-      tmp_races = row.races.map { |race| races[race] }
+      tmp_races = row.races.map { |race| races[race] }.reject(&:nil?)
       student.races += tmp_races
     end
 
     if row.respondent_type == :parent
       parent = Parent.find_or_create_by(response_id: row.response_id)
       parent.number_of_children = row.number_of_children
+      tmp_languages = row.languages.map { |language| languages[language] }.reject(&:nil?)
+      parent.languages.delete_all
+      parent.languages.concat(tmp_languages)
+      parent.housing = housings[row.housing] if row.housing.present?
       parent.save
     end