Add disaggregation by ELL

app/javascript/controllers/analyze_controller.js

@@ -28,7 +28,9 @@ export default class extends Controller {
       "&incomes=" +
       this.selected_items("income").join(",") +
       "&grades=" +
-      this.selected_items("grade").join(",");
+      this.selected_items("grade").join(",") +
+      "&ells=" +
+      this.selected_items("ell").join(",");
 
     this.go_to(url);
   }
@@ -126,7 +128,8 @@ export default class extends Controller {
       ['gender', 'students-by-gender'],
       ['grade', 'students-by-grade'],
       ['income', 'students-by-income'],
-      ['race', 'students-by-race']
+      ['race', 'students-by-race'],
+      ['ell', 'students-by-ell'],
     ])
 
     if (target.name === 'slice' || target.name === 'group') {

app/models/ell.rb

@@ -0,0 +1,7 @@
+class Ell < ApplicationRecord
+  scope :by_designation, -> { all.map { |ell| [ell.designation, ell] }.to_h }
+
+  include FriendlyId
+
+  friendly_id :designation, use: [:slugged]
+end

app/models/gender.rb

@@ -1,5 +1,5 @@
 class Gender < ApplicationRecord
-  scope :gender_hash, lambda {
+  scope :by_qualtrics_code, lambda {
     all.map { |gender| [gender.qualtrics_code, gender] }.to_h
   }
 end

app/models/income.rb

@@ -1,5 +1,6 @@
 class Income < ApplicationRecord
   scope :by_designation, -> { all.map { |income| [income.designation, income] }.to_h }
+  scope :by_slug, -> { all.map { |income| [income.slug, income] }.to_h }
 
   include FriendlyId
 

app/models/survey_item_response.rb

@@ -10,6 +10,7 @@ class SurveyItemResponse < ActiveRecord::Base
   belongs_to :student, foreign_key: :student_id, optional: true
   belongs_to :gender
   belongs_to :income
+  belongs_to :ell
 
   has_one :measure, through: :survey_item
 
@@ -32,6 +33,11 @@ class SurveyItemResponse < ActiveRecord::Base
                              academic_year:, income:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score)
   }
 
+  scope :averages_for_ell, lambda { |survey_items, school, academic_year, ell|
+    SurveyItemResponse.where(survey_item: survey_items, school:,
+                             academic_year:, ell:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score)
+  }
+
   scope :averages_for_race, lambda { |school, academic_year, race|
     SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where(
       school:, academic_year:, grade: school.grades(academic_year:)

app/presenters/analyze/graph/column/ell_column/ell.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module EllColumn
+        class Ell < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::EllColumn::ScoreForEll
+          include Analyze::Graph::Column::EllColumn::EllCount
+          def label
+            %w[ELL]
+          end
+
+          def basis
+            "student"
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def ell
+            ::Ell.find_by_slug "ell"
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/ell_column/ell_count.rb

@@ -0,0 +1,18 @@
+module Analyze
+  module Graph
+    module Column
+      module EllColumn
+        module EllCount
+          def type
+            :student
+          end
+
+          def n_size(year_index)
+            SurveyItemResponse.where(ell:, survey_item: measure.student_survey_items, school:, grade: grades(year_index),
+                                     academic_year: academic_years[year_index]).select(:response_id).distinct.count
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/ell_column/not_ell.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module EllColumn
+        class NotEll < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::EllColumn::ScoreForEll
+          include Analyze::Graph::Column::EllColumn::EllCount
+          def label
+            %w[Not-ELL]
+          end
+
+          def basis
+            "student"
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def ell
+            ::Ell.find_by_slug "not-ell"
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/ell_column/score_for_ell.rb

@@ -0,0 +1,42 @@
+module Analyze
+  module Graph
+    module Column
+      module EllColumn
+        module ScoreForEll
+          def score(year_index)
+            academic_year = academic_years[year_index]
+            meets_student_threshold = sufficient_student_responses?(academic_year:)
+            return Score::NIL_SCORE unless meets_student_threshold
+
+            averages = SurveyItemResponse.averages_for_ell(measure.student_survey_items, school, academic_year,
+                                                           ell)
+            average = bubble_up_averages(averages:).round(2)
+
+            Score.new(average:,
+                      meets_teacher_threshold: false,
+                      meets_student_threshold:,
+                      meets_admin_data_threshold: false)
+          end
+
+          def bubble_up_averages(averages:)
+            measure.student_scales.map do |scale|
+              scale.survey_items.map do |survey_item|
+                averages[survey_item]
+              end.remove_blanks.average
+            end.remove_blanks.average
+          end
+
+          def sufficient_student_responses?(academic_year:)
+            return false unless measure.subcategory.response_rate(school:, academic_year:).meets_student_threshold?
+
+            yearly_counts = SurveyItemResponse.where(school:, academic_year:,
+                                                     ell:, survey_item: measure.student_survey_items).group(:ell).select(:response_id).distinct(:response_id).count
+            yearly_counts.any? do |count|
+              count[1] >= 10
+            end
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/ell_column/unknown.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module EllColumn
+        class Unknown < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::EllColumn::ScoreForEll
+          include Analyze::Graph::Column::EllColumn::EllCount
+          def label
+            %w[Unknown]
+          end
+
+          def basis
+            "student"
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def ell
+            ::Ell.find_by_slug "unknown"
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/students_by_ell.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+#
+module Analyze
+  module Graph
+    class StudentsByEll
+      include Analyze::Graph::Column::GenderColumn
+      attr_reader :ells
+
+      def initialize(ells:)
+        @ells = ells
+      end
+
+      def to_s
+        "Students by Ell"
+      end
+
+      def slug
+        "students-by-ell"
+      end
+
+      def columns
+        [].tap do |array|
+          ells.each do |ell|
+            array << column_for_ell_code(code: ell.slug)
+          end
+          array.sort_by!(&:to_s)
+          array << Analyze::Graph::Column::AllStudent
+        end
+      end
+
+      private
+
+      def column_for_ell_code(code:)
+        CFR[code]
+      end
+
+      CFR = {
+        "ell" => Analyze::Graph::Column::EllColumn::Ell,
+        "not-ell" => Analyze::Graph::Column::EllColumn::NotEll,
+        "unknown" => Analyze::Graph::Column::EllColumn::Unknown
+      }.freeze
+    end
+  end
+end

app/presenters/analyze/graph/students_by_gender.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Analyze
   module Graph
     class StudentsByGender

app/presenters/analyze/graph/students_by_grade.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Analyze
   module Graph
     class StudentsByGrade
@@ -9,11 +11,11 @@ module Analyze
       end
 
       def to_s
-        'Students by Grade'
+        "Students by Grade"
       end
 
       def slug
-        'students-by-grade'
+        "students-by-grade"
       end
 
       def columns

app/presenters/analyze/graph/students_by_income.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Analyze
   module Graph
     class StudentsByIncome

app/presenters/analyze/graph/students_by_race.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Analyze
   module Graph
     class StudentsByRace
@@ -8,11 +10,11 @@ module Analyze
       end
 
       def to_s
-        'Students by Race'
+        "Students by Race"
       end
 
       def slug
-        'students-by-race'
+        "students-by-race"
       end
 
       def columns
@@ -31,14 +33,14 @@ module Analyze
       end
 
       CFR = {
-        '1' => Analyze::Graph::Column::RaceColumn::AmericanIndian,
-        '2' => Analyze::Graph::Column::RaceColumn::Asian,
-        '3' => Analyze::Graph::Column::RaceColumn::Black,
-        '4' => Analyze::Graph::Column::RaceColumn::Hispanic,
-        '5' => Analyze::Graph::Column::RaceColumn::White,
-        '8' => Analyze::Graph::Column::RaceColumn::MiddleEastern,
-        '99' => Analyze::Graph::Column::RaceColumn::Unknown,
-        '100' => Analyze::Graph::Column::RaceColumn::Multiracial
+        "1" => Analyze::Graph::Column::RaceColumn::AmericanIndian,
+        "2" => Analyze::Graph::Column::RaceColumn::Asian,
+        "3" => Analyze::Graph::Column::RaceColumn::Black,
+        "4" => Analyze::Graph::Column::RaceColumn::Hispanic,
+        "5" => Analyze::Graph::Column::RaceColumn::White,
+        "8" => Analyze::Graph::Column::RaceColumn::MiddleEastern,
+        "99" => Analyze::Graph::Column::RaceColumn::Unknown,
+        "100" => Analyze::Graph::Column::RaceColumn::Multiracial
       }.freeze
     end
   end

app/presenters/analyze/group/ell.rb

@@ -0,0 +1,13 @@
+module Analyze
+  module Group
+    class Ell
+      def name
+        "ELL"
+      end
+
+      def slug
+        "ell"
+      end
+    end
+  end
+end

app/presenters/analyze/presenter.rb

@@ -54,9 +54,27 @@ module Analyze
       end
     end
 
+    def ells
+      @ells ||= Ell.all.order(slug: :ASC)
+    end
+
+    def selected_ells
+      @selected_ells ||= begin
+        ell_params = params[:ells]
+        return ells unless ell_params
+
+        ell_params.split(",").map { |ell| Ell.find_by_slug ell }.compact
+      end
+    end
+
     def graphs
-      @graphs ||= [Analyze::Graph::AllData.new, Analyze::Graph::StudentsAndTeachers.new, Analyze::Graph::StudentsByRace.new(races: selected_races),
-                   Analyze::Graph::StudentsByGrade.new(grades: selected_grades), Analyze::Graph::StudentsByGender.new(genders: selected_genders), Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes)]
+      @graphs ||= [Analyze::Graph::AllData.new,
+                   Analyze::Graph::StudentsAndTeachers.new,
+                   Analyze::Graph::StudentsByRace.new(races: selected_races),
+                   Analyze::Graph::StudentsByGrade.new(grades: selected_grades),
+                   Analyze::Graph::StudentsByGender.new(genders: selected_genders),
+                   Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes),
+                   Analyze::Graph::StudentsByEll.new(ells: selected_ells)]
     end
 
     def graph
@@ -88,7 +106,7 @@ module Analyze
     end
 
     def groups
-      @groups = [Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new,
+      @groups = [Analyze::Group::Ell.new, Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new,
                  Analyze::Group::Race.new]
     end
 

app/services/cleaner.rb

@@ -1,12 +1,11 @@
 require "fileutils"
 class Cleaner
-  attr_reader :input_filepath, :output_filepath, :log_filepath, :disaggregation_filepath
+  attr_reader :input_filepath, :output_filepath, :log_filepath
 
-  def initialize(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:)
+  def initialize(input_filepath:, output_filepath:, log_filepath:)
     @input_filepath = input_filepath
     @output_filepath = output_filepath
     @log_filepath = log_filepath
-    @disaggregation_filepath = disaggregation_filepath
     initialize_directories
   end
 
@@ -14,7 +13,7 @@ class Cleaner
     Dir.glob(Rails.root.join(input_filepath, "*.csv")).each do |filepath|
       puts filepath
       File.open(filepath) do |file|
-        processed_data = process_raw_file(file:, disaggregation_data:)
+        processed_data = process_raw_file(file:)
         processed_data in [headers, clean_csv, log_csv, data]
         return if data.empty?
 
@@ -25,10 +24,6 @@ class Cleaner
     end
   end
 
-  def disaggregation_data
-    @disaggregation_data ||= DisaggregationLoader.new(path: disaggregation_filepath).load
-  end
-
   def filename(headers:, data:)
     survey_item_ids = headers.filter(&:present?).filter do |header|
                         header.start_with?("s-", "t-")
@@ -43,17 +38,16 @@ class Cleaner
     districts.join(".").to_s + "." + survey_type.to_s + "." + range + ".csv"
   end
 
-  def process_raw_file(file:, disaggregation_data:)
+  def process_raw_file(file:)
     clean_csv = []
     log_csv = []
     data = []
 
-    headers = (CSV.parse(file.first).first << "Raw Income") << "Income"
+    headers = CSV.parse(file.first).first.push("Raw Income").push("Income").push("Raw ELL").push("ELL")
     filtered_headers = include_all_headers(headers:)
     filtered_headers = remove_unwanted_headers(headers: filtered_headers)
     log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?",
                                        "Valid Standard Deviation?"]).flatten
-
     clean_csv << filtered_headers
     log_csv << log_headers
 
@@ -62,7 +56,7 @@ class Cleaner
     file.lazy.each_slice(1000) do |lines|
       CSV.parse(lines.join, headers:).map do |row|
         values = SurveyItemValues.new(row:, headers:, genders:,
-                                      survey_items: all_survey_items, schools:, disaggregation_data:)
+                                      survey_items: all_survey_items, schools:)
         next unless values.valid_school?
 
         data << values
@@ -109,7 +103,7 @@ class Cleaner
   end
 
   def genders
-    @genders ||= Gender.gender_hash
+    @genders ||= Gender.by_qualtrics_code
   end
 
   def survey_items(headers:)

app/services/demographic_loader.rb

@@ -6,12 +6,13 @@ class DemographicLoader
       process_race(row:)
       process_gender(row:)
       process_income(row:)
+      process_ell(row:)
     end
   end
 
   def self.process_race(row:)
-    qualtrics_code = row['Race Qualtrics Code'].to_i
-    designation = row['Race/Ethnicity']
+    qualtrics_code = row["Race Qualtrics Code"].to_i
+    designation = row["Race/Ethnicity"]
     return unless qualtrics_code && designation
 
     if qualtrics_code.between?(6, 7)
@@ -22,8 +23,8 @@ class DemographicLoader
   end
 
   def self.process_gender(row:)
-    qualtrics_code = row['Gender Qualtrics Code'].to_i
-    designation = row['Sex/Gender']
+    qualtrics_code = row["Gender Qualtrics Code"].to_i
+    designation = row["Sex/Gender"]
     return unless qualtrics_code && designation
 
     gender = ::Gender.find_or_create_by!(qualtrics_code:, designation:)
@@ -31,11 +32,18 @@ class DemographicLoader
   end
 
   def self.process_income(row:)
-    designation = row['Income']
+    designation = row["Income"]
     return unless designation
 
     Income.find_or_create_by!(designation:)
   end
+
+  def self.process_ell(row:)
+    designation = row["ELL"]
+    return unless designation
+
+    Ell.find_or_create_by!(designation:)
+  end
 end
 
 class KnownRace
@@ -50,7 +58,7 @@ end
 class UnknownRace
   def initialize(qualtrics_code:, designation:)
     unknown = Race.find_or_create_by!(qualtrics_code: 99)
-    unknown.designation = 'Race/Ethnicity Not Listed'
+    unknown.designation = "Race/Ethnicity Not Listed"
     unknown.slug = designation.parameterize
     unknown.save
   end

app/services/disaggregation_row.rb

@@ -14,7 +14,7 @@ class DisaggregationRow
     @academic_year ||= value_from(pattern: /Academic\s*Year/i)
   end
 
-  def income
+  def raw_income
     @income ||= value_from(pattern: /Low\s*Income/i)
   end
 
@@ -22,6 +22,25 @@ class DisaggregationRow
     @lasid ||= value_from(pattern: /LASID/i)
   end
 
+  def raw_ell
+    @raw_ell ||= value_from(pattern: /EL Student First Year/i)
+  end
+
+  def ell
+    @ell ||= begin
+      value = value_from(pattern: /EL Student First Year/i).downcase
+
+      case value
+      when /lep student 1st year|LEP student not 1st year/i
+        "ELL"
+      when /Does not apply/i
+        "Not ELL"
+      else
+        "Unknown"
+      end
+    end
+  end
+
   def value_from(pattern:)
     output = nil
     matches = headers.select do |header|

app/services/survey_item_values.rb

@@ -1,7 +1,7 @@
 class SurveyItemValues
-  attr_reader :row, :headers, :genders, :survey_items, :schools, :disaggregation_data
+  attr_reader :row, :headers, :genders, :survey_items, :schools
 
-  def initialize(row:, headers:, genders:, survey_items:, schools:, disaggregation_data: nil)
+  def initialize(row:, headers:, genders:, survey_items:, schools:)
     @row = row
     # Remove any newlines in headers
     headers = headers.map { |item| item.delete("\n") if item.present? }
@@ -9,11 +9,12 @@ class SurveyItemValues
     @genders = genders
     @survey_items = survey_items
     @schools = schools
-    @disaggregation_data = disaggregation_data
 
     copy_likert_scores_from_variant_survey_items
     row["Income"] = income
     row["Raw Income"] = raw_income
+    row["Raw ELL"] = raw_ell
+    row["ELL"] = ell
 
     copy_data_to_main_column(main: /Race/i, secondary: /Race Secondary|Race-1/i)
     copy_data_to_main_column(main: /Gender/i, secondary: /Gender Secondary|Gender-1/i)
@@ -134,20 +135,9 @@ class SurveyItemValues
 
   def raw_income
     @raw_income ||= value_from(pattern: /Low\s*Income|Raw\s*Income/i)
-    return @raw_income if @raw_income.present?
-
-    return "Unknown" unless disaggregation_data.present?
-
-    disaggregation = disaggregation_data[[lasid, district.name, academic_year.range]]
-    return "Unknown" unless disaggregation.present?
-
-    @raw_income ||= disaggregation.income
   end
 
   def income
-    @income ||= value_from(pattern: /^Income$/i)
-    return @income if @income.present?
-
     @income ||= case raw_income
                 in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income/i
                   "Economically Disadvantaged - Y"
@@ -158,6 +148,21 @@ class SurveyItemValues
                 end
   end
 
+  def raw_ell
+    @raw_ell ||= value_from(pattern: /EL Student First Year|Raw\s*ELL/i)
+  end
+
+  def ell
+    @ell ||= case raw_ell
+             in /lep student 1st year|LEP student not 1st year|EL Student First Year/i
+               "ELL"
+             in /Does not apply/i
+               "Not ELL"
+             else
+               "Unknown"
+             end
+  end
+
   def value_from(pattern:)
     output = nil
     matches = headers.select do |header|

app/services/survey_responses_data_loader.rb

@@ -1,31 +1,25 @@
 # frozen_string_literal: true
 
 class SurveyResponsesDataLoader
-  def self.load_data(filepath:, rules: [Rule::NoRule])
+  def load_data(filepath:, rules: [Rule::NoRule])
     File.open(filepath) do |file|
       headers = file.first
       headers_array = CSV.parse(headers).first
-      genders = Gender.gender_hash
-      schools = School.school_hash
-      incomes = Income.by_designation
       all_survey_items = survey_items(headers:)
 
       file.lazy.each_slice(500) do |lines|
         survey_item_responses = CSV.parse(lines.join, headers:).map do |row|
           process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:),
-                      rules:, incomes:)
+                      rules:)
         end
         SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500
       end
     end
   end
 
-  def self.from_file(file:, rules: [])
+  def from_file(file:, rules: [])
     headers = file.gets
     headers_array = CSV.parse(headers).first
-    genders = Gender.gender_hash
-    schools = School.school_hash
-    incomes = Income.by_designation
     all_survey_items = survey_items(headers:)
 
     survey_item_responses = []
@@ -36,7 +30,7 @@ class SurveyResponsesDataLoader
 
       CSV.parse(line, headers:).map do |row|
         survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:),
-                                             rules:, incomes:)
+                                             rules:)
       end
 
       row_count += 1
@@ -52,7 +46,23 @@ class SurveyResponsesDataLoader
 
   private
 
-  def self.process_row(row:, rules:, incomes:)
+  def schools
+    @schools = School.school_hash
+  end
+
+  def genders
+    @genders = Gender.by_qualtrics_code
+  end
+
+  def incomes
+    @incomes ||= Income.by_slug
+  end
+
+  def ells
+    @ells ||= Ell.by_designation
+  end
+
+  def process_row(row:, rules:)
     return unless row.dese_id?
     return unless row.school.present?
 
@@ -60,10 +70,10 @@ class SurveyResponsesDataLoader
       return if rule.new(row:).skip_row?
     end
 
-    process_survey_items(row:, incomes:)
+    process_survey_items(row:)
   end
 
-  def self.process_survey_items(row:, incomes:)
+  def process_survey_items(row:)
     row.survey_items.map do |survey_item|
       likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next
 
@@ -72,38 +82,33 @@ class SurveyResponsesDataLoader
         next
       end
       response = row.survey_item_response(survey_item:)
-      create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, incomes:)
+      create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:)
     end.compact
   end
 
-  def self.create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, incomes:)
+  def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:)
     gender = row.gender
     grade = row.grade
-    income = incomes[row.income]
+    income = incomes[row.income.parameterize]
+    ell = ells[row.ell]
     if survey_item_response.present?
-      survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:)
+      survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:)
       []
     else
       SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:,
-                             likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:)
+                             likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:)
     end
   end
 
-  def self.survey_items(headers:)
+  def survey_items(headers:)
     SurveyItem.where(survey_item_id: get_survey_item_ids_from_headers(headers:))
   end
 
-  def self.get_survey_item_ids_from_headers(headers:)
+  def get_survey_item_ids_from_headers(headers:)
     CSV.parse(headers).first
        .filter(&:present?)
        .filter { |header| header.start_with? "t-", "s-" }
   end
-
-  private_class_method :process_row
-  private_class_method :process_survey_items
-  private_class_method :create_or_update_response
-  private_class_method :survey_items
-  private_class_method :get_survey_item_ids_from_headers
 end
 
 module StringMonkeyPatches

app/views/analyze/_group_selectors.html.erb

@@ -21,3 +21,7 @@
 <% @presenter.incomes.each do |income| %>
   <%= render(partial: "checkboxes", locals: {id: "income-#{income.slug}", item: income, selected_items: @presenter.selected_incomes, name: "income", label_text: income.label})  %>
 <% end %>
+
+<% @presenter.ells.each do |ell| %>
+  <%= render(partial: "checkboxes", locals: {id: "ell-#{ell.slug}", item: ell, selected_items: @presenter.selected_ells, name: "ell", label_text: ell.designation})  %>
+<% end %>

app/views/analyze/index.html.erb

@@ -13,7 +13,7 @@
     <%= render partial: "school_years", locals: {available_academic_years: @presenter.academic_years, selected_academic_years: @presenter.selected_academic_years, district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory, measures: @presenter.measures, graph: @presenter.graph} %>
     <%= render partial: "data_filters", locals: {district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory} %>
   </div>
-  <% cache [@presenter.subcategory, @school, @presenter.selected_academic_years,  @presenter.graph, @presenter.selected_races,  @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders]  do %>
+  <% cache [@presenter.subcategory, @school, @presenter.selected_academic_years,  @presenter.graph, @presenter.selected_races,  @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders, @presenter.selected_ells, @presenter.ells]  do %>
     <div class="bg-color-white flex-grow-1 col-9">
       <% @presenter.measures.each do  |measure| %>
         <section class="mb-6">

data/demographics.csv

@@ -1,11 +1,11 @@
-Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income
-1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged - N
-2,Asian or Pacific Islander,1,Female,Economically Disadvantaged - Y
-3,Black or African American,4,Non-Binary,Unknown
-4,Hispanic or Latinx,99,Unknown,
-5,White or Caucasian,,,
-6,Prefer not to disclose,,,
-7,Prefer to self-describe,,,
-8,Middle Eastern,,,
-99,Race/Ethnicity Not Listed,,,
-100,Multiracial,,,
+Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL
+1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged - N,ELL
+2,Asian or Pacific Islander,1,Female,Economically Disadvantaged - Y,Not ELL
+3,Black or African American,4,Non-Binary,Unknown,Unknown
+4,Hispanic or Latinx,99,Unknown,,
+5,White or Caucasian,,,,
+6,Prefer not to disclose,,,,
+7,Prefer to self-describe,,,,
+8,Middle Eastern,,,,
+99,Race/Ethnicity Not Listed,,,,
+100,Multiracial,,,,

db/migrate/20230830213521_create_ell.rb

@@ -0,0 +1,12 @@
+class CreateEll < ActiveRecord::Migration[7.0]
+  def change
+    create_table :ells do |t|
+      t.string :designation
+      t.string :slug
+
+      t.timestamps
+    end
+
+    add_index :ells, :designation, unique: true
+  end
+end

db/migrate/20230912223701_add_ell_to_survey_item_response.rb

@@ -0,0 +1,5 @@
+class AddEllToSurveyItemResponse < ActiveRecord::Migration[7.0]
+  def change
+    add_reference :survey_item_responses, :ell, foreign_key: true
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_08_07_222503) do
+ActiveRecord::Schema[7.0].define(version: 2023_09_12_223701) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -69,6 +69,14 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_07_222503) do
     t.datetime "updated_at", null: false
   end
 
+  create_table "ells", force: :cascade do |t|
+    t.string "designation"
+    t.string "slug"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["designation"], name: "index_ells_on_designation", unique: true
+  end
+
   create_table "genders", force: :cascade do |t|
     t.integer "qualtrics_code"
     t.string "designation"
@@ -439,7 +447,9 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_07_222503) do
     t.bigint "gender_id"
     t.bigint "income_id"
     t.datetime "recorded_date"
+    t.bigint "ell_id"
     t.index ["academic_year_id"], name: "index_survey_item_responses_on_academic_year_id"
+    t.index ["ell_id"], name: "index_survey_item_responses_on_ell_id"
     t.index ["gender_id"], name: "index_survey_item_responses_on_gender_id"
     t.index ["income_id"], name: "index_survey_item_responses_on_income_id"
     t.index ["response_id"], name: "index_survey_item_responses_on_response_id"
@@ -491,6 +501,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_07_222503) do
   add_foreign_key "student_races", "students"
   add_foreign_key "subcategories", "categories"
   add_foreign_key "survey_item_responses", "academic_years"
+  add_foreign_key "survey_item_responses", "ells"
   add_foreign_key "survey_item_responses", "genders"
   add_foreign_key "survey_item_responses", "incomes"
   add_foreign_key "survey_item_responses", "schools"

lib/tasks/clean.rake

@@ -1,37 +1,33 @@
 namespace :clean do
   # These tasks must be run in their respective project so the correct schools are in the database
-  desc 'clean ecp data'
+  desc "clean ecp data"
   task ecp: :environment do
-    input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'raw')
-    output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'clean')
-    log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'removed')
-    disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
-    Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
+    input_filepath = Rails.root.join("tmp", "data", "ecp_data", "raw")
+    output_filepath = Rails.root.join("tmp", "data", "ecp_data", "clean")
+    log_filepath = Rails.root.join("tmp", "data", "ecp_data", "removed")
+    Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
   end
 
-  desc 'clean prepped data'
+  desc "clean prepped data"
   task prepped: :environment do
-    input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped')
-    output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'clean')
-    log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'removed')
-    disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
-    Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
+    input_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped")
+    output_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "clean")
+    log_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "removed")
+    Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
   end
-  desc 'clean mciea data'
+  desc "clean mciea data"
   task mciea: :environment do
-    input_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'raw')
-    output_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'clean')
-    log_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'removed')
-    disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
-    Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
+    input_filepath = Rails.root.join("tmp", "data", "mciea_data", "raw")
+    output_filepath = Rails.root.join("tmp", "data", "mciea_data", "clean")
+    log_filepath = Rails.root.join("tmp", "data", "mciea_data", "removed")
+    Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
   end
 
-  desc 'clean rpp data'
+  desc "clean rpp data"
   task rpp: :environment do
-    input_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'raw')
-    output_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'clean')
-    log_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'removed')
-    disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
-    Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
+    input_filepath = Rails.root.join("tmp", "data", "rpp_data", "raw")
+    output_filepath = Rails.root.join("tmp", "data", "rpp_data", "clean")
+    log_filepath = Rails.root.join("tmp", "data", "rpp_data", "removed")
+    Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
   end
 end

lib/tasks/data.rake

@@ -5,7 +5,7 @@ namespace :data do
     student_count = Student.count
     path = "/data/survey_responses/clean/"
     Sftp::Directory.open(path:) do |file|
-      SurveyResponsesDataLoader.from_file(file:)
+      SurveyResponsesDataLoader.new.from_file(file:)
     end
     puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
 

lib/tasks/one_off.rake

@@ -40,38 +40,6 @@ namespace :one_off do
     end
   end
 
-  desc 'load stoklosa results for 2022-23'
-  task load_stoklosa: :environment do
-    survey_item_response_count = SurveyItemResponse.count
-    school = School.find_by_dese_id(1_600_360)
-    academic_year = AcademicYear.find_by_range('2022-23')
-
-    ['2022-23_stoklosa_student_survey_responses.csv',
-     '2022-23_stoklosa_teacher_survey_responses.csv'].each do |filepath|
-      filepath = Rails.root.join('data', 'survey_responses', filepath)
-      puts "=====================> Loading data from csv at path: #{filepath}"
-      SurveyResponsesDataLoader.load_data filepath:
-    end
-    puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
-
-
-    Dir.glob(Rails.root.join('data', 'survey_responses',
-                             '2022-23_stoklosa_student_survey_responses.csv')).each do |file|
-      puts "=====================> Loading student data from csv at path: #{file}"
-      StudentLoader.load_data filepath: file, rules: [Rule::SkipNonLowellSchools]
-    end
-  end
-
-  desc 'load butler results for 2022-23'
-  task load_butler: :environment do
-    ['2022-23_butler_student_survey_responses.csv',
-     '2022-23_butler_teacher_survey_responses.csv'].each do |filepath|
-      filepath = Rails.root.join('data', 'survey_responses', filepath)
-      puts "=====================> Loading data from csv at path: #{filepath}"
-      SurveyResponsesDataLoader.load_data filepath:
-    end
-  end
-
   desc 'list scales that have no survey responses'
   task list_scales_that_lack_survey_responses: :environment do
     output = AcademicYear.all.map do |academic_year|
@@ -114,23 +82,6 @@ namespace :one_off do
     puts values
   end
 
-  desc 'load survey responses for lowell schools'
-  task load_survey_responses_for_lowell: :environment do
-    survey_item_response_count = SurveyItemResponse.count
-    student_count = Student.count
-    Sftp::Directory.open(path: '/test/survey_responses/') do |file|
-      SurveyResponsesDataLoader.from_file(file:)
-    end
-    puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
-
-    Sftp::Directory.open(path: '/test/survey_responses/') do |file|
-      StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools])
-    end
-    puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
-
-    Rails.cache.clear
-  end
-
   desc 'delete 2022-23 survey responses'
   task delete_survey_responses_2022_23: :environment do
     response_count = SurveyItemResponse.all.count
@@ -148,7 +99,7 @@ namespace :one_off do
     schools = District.find_by_slug('maynard-public-schools').schools
 
     Sftp::Directory.open(path:) do |file|
-      SurveyResponsesDataLoader.from_file(file:)
+      SurveyResponsesDataLoader.new.from_file(file:)
     end
     puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
 

spec/factories.rb

@@ -1,11 +1,15 @@
 FactoryBot.define do
   factory :income do
-    designation { "MyString" }
+    designation { "DefaultIncome" }
+  end
+
+  factory :ell do
+    designation { "DefaultEll" }
   end
 
   factory :gender do
     qualtrics_code { 1 }
-    designation { 'MyString' }
+    designation { "MyString" }
   end
 
   factory :race_score do
@@ -68,22 +72,22 @@ FactoryBot.define do
   end
 
   factory :academic_year do
-    range { '2050-51' }
+    range { "2050-51" }
     initialize_with { AcademicYear.find_or_initialize_by(range:) }
   end
 
-  factory :category, class: 'Category' do
+  factory :category, class: "Category" do
     name { "A #{rand} category" }
     category_id { rand.to_s }
-    description { 'A description of a category' }
+    description { "A description of a category" }
     slug { name.parameterize }
     sort_index { 1 }
   end
 
   factory :subcategory do
-    name { 'A subcategory' }
+    name { "A subcategory" }
     subcategory_id { rand.to_s }
-    description { 'A description of a subcategory' }
+    description { "A description of a subcategory" }
     category
 
     factory :subcategory_with_measures do
@@ -102,7 +106,7 @@ FactoryBot.define do
 
   factory :measure do
     measure_id { rand.to_s }
-    name { 'A Measure' }
+    name { "A Measure" }
     subcategory
     trait :with_student_survey_items do
       after(:create) do |measure|
@@ -136,7 +140,7 @@ FactoryBot.define do
 
   factory :survey_item do
     scale
-    prompt { 'What do YOU think?' }
+    prompt { "What do YOU think?" }
     factory :teacher_survey_item do
       survey_item_id { "t-#{rand}" }
       watch_low_benchmark { 2.0 }

spec/fixtures/sample_demographics.csv

@@ -1,11 +1,11 @@
-Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income
-1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged – N
-2,Asian or Pacific Islander,1,Female,Economically Disadvantaged – Y
-3,Black or African American,4,Non-Binary,Unknown
-4,Hispanic or Latinx,99,Unknown,
-5,White or Caucasian,,,
-6,Prefer not to disclose,,,
-7,Prefer to self-describe,,,
-8,Middle Eastern,,,
-99,Race/Ethnicity Not Listed,,,
-100,Multiracial,,,
+Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL
+1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged – N,ELL
+2,Asian or Pacific Islander,1,Female,Economically Disadvantaged – Y,Not ELL
+3,Black or African American,4,Non-Binary,Unknown,Unknown
+4,Hispanic or Latinx,99,Unknown,,
+5,White or Caucasian,,,,
+6,Prefer not to disclose,,,,
+7,Prefer to self-describe,,,,
+8,Middle Eastern,,,,
+99,Race/Ethnicity Not Listed,,,,
+100,Multiracial,,,,

spec/fixtures/test_2020-21_student_survey_responses.csv

@@ -1,8 +1,8 @@
-Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,LASID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district,school,DESE ID,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,#N/A,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,#N/A,#N/A,s-cure-q3,s-cure-q4,#N/A,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,#N/A,#N/A,s-grmi-q4,s-appa-q1,s-appa-q2,#N/A,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q1-1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-acst-q4,s-acst-q5,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,race,What is your race/ethnicity?(Please select all that apply) - Selected Choice,grade,gender,Raw Income,Income
-2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,1500025,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888,11th,1,Free Lunch,Economically Disadvantaged – Y
-2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,"2,3,4",888,10,,Not Eligible,Economically Disadvantaged – N
-2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1500505,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,6,888,8,2,Reduced Lunch,Economically Disadvantaged – Y
-2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1500505,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888,8,3,,Unknown
-2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888,7,4,Free Lunch,Economically Disadvantaged – Y
-2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888,3,NA,Not Eligible,Economically Disadvantaged – N
-2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,,,4,,Reduced Lunch,Economically Disadvantaged – Y
+Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,LASID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district,school,DESE ID,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,#N/A,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,#N/A,#N/A,s-cure-q3,s-cure-q4,#N/A,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,#N/A,#N/A,s-grmi-q4,s-appa-q1,s-appa-q2,#N/A,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q1-1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-acst-q4,s-acst-q5,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,race,What is your race/ethnicity?(Please select all that apply) - Selected Choice,grade,gender,Raw Income,Income,Raw ELL,ELL
+2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,1500025,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888,11th,1,Free Lunch,Economically Disadvantaged – Y,Does not apply,Not ELL
+2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,"2,3,4",888,10,,Not Eligible,Economically Disadvantaged – N,,Unknown
+2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1500505,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,6,888,8,2,Reduced Lunch,Economically Disadvantaged – Y,,Unknown
+2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1500505,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888,8,3,,Unknown,LEP student not 1st year,ELL
+2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888,7,4,Free Lunch,Economically Disadvantaged – Y,EL Student First Year,ELL
+2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888,3,NA,Not Eligible,Economically Disadvantaged – N,Unknown,Unknown
+2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,,,4,,Reduced Lunch,Economically Disadvantaged – Y,#N/A,Unknown

spec/services/cleaner_spec.rb

@@ -27,14 +27,6 @@ RSpec.describe Cleaner do
     Rails.root.join("tmp", "spec", "removed")
   end
 
-  let(:disaggregation_filepath) do
-    Rails.root.join("spec", "fixtures", "disaggregation")
-  end
-
-  let(:path_to_sample_disaggregation_file) do
-    File.open(Rails.root.join("spec", "fixtures", "disaggregation", "sample_maynard_disaggregation_data.csv"))
-  end
-
   let(:path_to_sample_raw_file) do
     File.open(Rails.root.join("spec", "fixtures", "raw", "sample_maynard_raw_student_survey.csv"))
   end
@@ -99,21 +91,21 @@ RSpec.describe Cleaner do
 
   context "Creating a new Cleaner" do
     it "creates a directory for the clean data" do
-      Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
+      Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
       expect(output_filepath).to exist
     end
 
     it "creates a directory for the removed data" do
-      Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
+      Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
       expect(log_filepath).to exist
     end
   end
 
   context ".process_raw_file" do
     it "sorts data into valid and invalid csvs" do
-      cleaner = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:)
+      cleaner = Cleaner.new(input_filepath:, output_filepath:, log_filepath:)
       processed_data = cleaner.process_raw_file(
-        file: path_to_sample_raw_file, disaggregation_data: cleaner.disaggregation_data
+        file: path_to_sample_raw_file
       )
       processed_data in [headers, clean_csv, log_csv, data]
 
@@ -140,22 +132,6 @@ RSpec.describe Cleaner do
       csv_contains_the_correct_rows(log_csv, invalid_rows)
       invalid_rows_are_rejected_for_the_correct_reasons(data)
     end
-
-    it "adds dissaggregation data to the cleaned file " do
-      cleaner = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:)
-      processed_data = cleaner.process_raw_file(
-        file: path_to_sample_raw_file, disaggregation_data: cleaner.disaggregation_data
-      )
-      processed_data in [headers, clean_csv, log_csv, data]
-      index_of_income = clean_csv.first.index("Income")
-      expect(clean_csv.second[index_of_income]).to eq "Economically Disadvantaged - Y"
-
-      one_thousand = data.find { |row| row.response_id == "1000" }
-      expect(one_thousand.income).to eq "Economically Disadvantaged - Y"
-
-      one_thousand_one = data.find { |row| row.response_id == "1001" }
-      expect(one_thousand_one.income).to eq "Economically Disadvantaged - N"
-    end
   end
 
   context ".filename" do
@@ -166,7 +142,7 @@ RSpec.describe Cleaner do
 
           data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: standard_survey_items, genders: nil, survey_items:,
                                        schools: School.school_hash)]
-          filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename(
+          filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
             headers: standard_survey_items, data:
           )
           expect(filename).to eq "maynard.standard.2022-23.csv"
@@ -178,7 +154,7 @@ RSpec.describe Cleaner do
 
             data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: short_form_survey_items, genders: nil, survey_items:,
                                          schools: School.school_hash)]
-            filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename(
+            filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
               headers: short_form_survey_items, data:
             )
             expect(filename).to eq "maynard.short_form.2022-23.csv"
@@ -191,7 +167,7 @@ RSpec.describe Cleaner do
 
             data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: early_education_survey_items, genders: nil, survey_items:,
                                          schools: School.school_hash)]
-            filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename(
+            filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
               headers: early_education_survey_items, data:
             )
             expect(filename).to eq "maynard.early_education.2022-23.csv"
@@ -203,7 +179,7 @@ RSpec.describe Cleaner do
 
             data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: teacher_survey_items, genders: nil, survey_items:,
                                          schools: School.school_hash)]
-            filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename(
+            filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
               headers: teacher_survey_items, data:
             )
             expect(filename).to eq "maynard.teacher.2022-23.csv"
@@ -217,7 +193,7 @@ RSpec.describe Cleaner do
             data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: teacher_survey_items, genders: nil, survey_items:, schools: School.school_hash),
                     SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "222_222" },
                                          headers: teacher_survey_items, genders: nil, survey_items:, schools: School.school_hash)]
-            filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename(
+            filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
               headers: teacher_survey_items, data:
             )
             expect(filename).to eq "maynard.district2.teacher.2022-23.csv"
@@ -230,7 +206,7 @@ end
 
 def reads_headers_from_raw_csv(processed_data)
   processed_data in [headers, clean_csv, log_csv, data]
-  expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)",
+  expect(headers.to_set.sort).to eq ["StartDate", "EndDate",  "Status", "IPAddress", "Progress", "Duration (in seconds)",
                                      "Finished", "RecordedDate", "ResponseId", "District", "School",
                                      "LASID", "Gender", "Race", "What grade are you in?", "s-emsa-q1", "s-emsa-q2", "s-emsa-q3", "s-tint-q1",
                                      "s-tint-q2", "s-tint-q3", "s-tint-q4", "s-tint-q5", "s-acpr-q1", "s-acpr-q2",
@@ -243,7 +219,7 @@ def reads_headers_from_raw_csv(processed_data)
                                      "s-grit-q1", "s-grit-q2", "s-grit-q3", "s-grit-q4", "s-expa-q1", "s-poaf-q1", "s-poaf-q2", "s-poaf-q3",
                                      "s-poaf-q4", "s-tint-q1-1", "s-tint-q2-1", "s-tint-q3-1", "s-tint-q4-1", "s-tint-q5-1", "s-acpr-q1-1",
                                      "s-acpr-q2-1", "s-acpr-q3-1", "s-acpr-q4-1", "s-peff-q1-1", "s-peff-q2-1", "s-peff-q3-1", "s-peff-q4-1",
-                                     "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income"].to_set.sort
+                                     "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL"].to_set.sort
 end
 
 def invalid_rows_are_rejected_for_the_correct_reasons(data)

spec/services/demographic_loader_spec.rb

@@ -1,20 +1,24 @@
-require 'rails_helper'
+require "rails_helper"
 
 describe DemographicLoader do
-  let(:filepath) { 'spec/fixtures/sample_demographics.csv' }
+  let(:filepath) { "spec/fixtures/sample_demographics.csv" }
   let(:race_codes) do
-    { 'American Indian or Alaskan Native' => 1, 'Asian or Pacific Islander' => 2, 'Black or African American' => 3,
-      'Hispanic or Latinx' => 4, 'White or Caucasian' => 5, 'Race/Ethnicity Not Listed' => 99, 'Middle Eastern' => 8, 'Multiracial' => 100 }
+    { "American Indian or Alaskan Native" => 1, "Asian or Pacific Islander" => 2, "Black or African American" => 3,
+      "Hispanic or Latinx" => 4, "White or Caucasian" => 5, "Race/Ethnicity Not Listed" => 99, "Middle Eastern" => 8, "Multiracial" => 100 }
   end
 
   let(:gender_codes) do
     {
-      'Female' => 1, 'Male' => 2, 'Non-Binary' => 4, 'Unknown' => 99
+      "Female" => 1, "Male" => 2, "Non-Binary" => 4, "Unknown" => 99
     }
   end
 
   let(:incomes) do
-    ['Economically Disadvantaged – N', 'Economically Disadvantaged – Y', 'Unknown']
+    ["Economically Disadvantaged – N", "Economically Disadvantaged – Y", "Unknown"]
+  end
+
+  let(:ells) do
+    ["ELL", "Not ELL", "Unknown"]
   end
 
   before :each do
@@ -25,12 +29,12 @@ describe DemographicLoader do
     DatabaseCleaner.clean
   end
 
-  describe 'self.load_data' do
-    it 'does not load qualtrics categories for `prefer not to disclose` or `prefer to self-describe`' do
+  describe "self.load_data" do
+    it "does not load qualtrics categories for `prefer not to disclose` or `prefer to self-describe`" do
       expect(Race.find_by_qualtrics_code(6)).to be nil
     end
 
-    it 'loads all racial designations' do
+    it "loads all racial designations" do
       expect(Race.all.count).to eq 8
       race_codes.each do |key, value|
         expect(Race.find_by_qualtrics_code(value)).not_to eq nil
@@ -40,7 +44,7 @@ describe DemographicLoader do
       end
     end
 
-    it 'loads all gender designations' do
+    it "loads all gender designations" do
       expect(Gender.all.count).to eq 4
 
       gender_codes.each do |key, value|
@@ -51,11 +55,18 @@ describe DemographicLoader do
       end
     end
 
-    it 'loads all the income designations' do
+    it "loads all the income designations" do
       expect(Income.all.count).to eq 3
       incomes.each do |income|
         expect(Income.find_by_designation(income).designation).to eq income
       end
     end
+
+    it "loads all the ells designations" do
+      expect(Ell.all.count).to eq 3
+      ells.each do |ell|
+        expect(Ell.find_by_designation(ell).designation).to eq ell
+      end
+    end
   end
 end

spec/services/disaggregation_loader_spec.rb

@@ -1,33 +1,37 @@
-require 'rails_helper'
-require 'fileutils'
+require "rails_helper"
+require "fileutils"
 
 RSpec.describe DisaggregationLoader do
   let(:path) do
-    Rails.root.join('spec', 'fixtures', 'disaggregation')
+    Rails.root.join("spec", "fixtures", "disaggregation")
   end
-  let(:academic_year) { create(:academic_year, range: '2022-23') }
-  let(:district) { create(:district, name: 'Maynard Public Schools') }
-  context '.load' do
-    it 'loads data from the file into a hash' do
+  let(:academic_year) { create(:academic_year, range: "2022-23") }
+  let(:district) { create(:district, name: "Maynard Public Schools") }
+  context ".load" do
+    it "loads data from the file into a hash" do
       data = DisaggregationLoader.new(path:).load
-      expect(data.values.first.lasid).to eq('1')
-      expect(data.values.first.academic_year).to eq('2022-23')
-      expect(data.values.first.district).to eq('Maynard Public Schools')
-      expect(data.values.first.income).to eq('Free Lunch')
+      expect(data.values.first.lasid).to eq("1")
+      expect(data.values.first.academic_year).to eq("2022-23")
+      expect(data.values.first.district).to eq("Maynard Public Schools")
 
-      expect(data.values.last.lasid).to eq('500')
-      expect(data.values.last.academic_year).to eq('2022-23')
-      expect(data.values.last.district).to eq('Maynard Public Schools')
-      expect(data.values.last.income).to eq('Not Eligible')
+      expect(data.values.last.lasid).to eq("500")
+      expect(data.values.last.academic_year).to eq("2022-23")
+      expect(data.values.last.district).to eq("Maynard Public Schools")
+    end
+
+    it "loads income data" do
+      data = DisaggregationLoader.new(path:).load
+      expect(data.values.first.raw_income).to eq("Free Lunch")
+      expect(data.values.last.raw_income).to eq("Not Eligible")
 
-      expect(data[['1', 'Maynard Public Schools', '2022-23']].income).to eq('Free Lunch')
-      expect(data[['2', 'Maynard Public Schools', '2022-23']].income).to eq('Not Eligible')
-      expect(data[['3', 'Maynard Public Schools', '2022-23']].income).to eq('Reduced Lunch')
+      expect(data[["1", "Maynard Public Schools", "2022-23"]].raw_income).to eq("Free Lunch")
+      expect(data[["2", "Maynard Public Schools", "2022-23"]].raw_income).to eq("Not Eligible")
+      expect(data[["3", "Maynard Public Schools", "2022-23"]].raw_income).to eq("Reduced Lunch")
     end
   end
 
-  context 'Creating a new loader' do
-    it 'creates a directory for the loader file' do
+  context "Creating a new loader" do
+    it "creates a directory for the loader file" do
       DisaggregationLoader.new(path:)
       expect(path).to exist
     end

spec/services/disaggregation_row_spec.rb

@@ -1,73 +1,107 @@
-require 'rails_helper'
+require "rails_helper"
 
 RSpec.describe DisaggregationRow do
   let(:headers) do
-    ['District', 'Academic Year', 'LASID', 'HispanicLatino', 'Race', 'Gender', 'SpecialEdStatus', 'In 504 Plan',
-     'LowIncome', 'EL Student First Year']
+    ["District", "Academic Year", "LASID", "HispanicLatino", "Race", "Gender", "SpecialEdStatus", "In 504 Plan",
+     "LowIncome", "EL Student First Year"]
   end
 
-  context '.district' do
-    context 'when the column heading is any upper or lowercase variant of the word district' do
-      it 'returns the correct value for district' do
-        row = { 'District' => 'Maynard Public Schools' }
-        expect(DisaggregationRow.new(row:, headers:).district).to eq 'Maynard Public Schools'
+  context ".district" do
+    context "when the column heading is any upper or lowercase variant of the word district" do
+      it "returns the correct value for district" do
+        row = { "District" => "Maynard Public Schools" }
+        expect(DisaggregationRow.new(row:, headers:).district).to eq "Maynard Public Schools"
 
-        headers = ['dISTRICT']
+        headers = ["dISTRICT"]
         headers in [district]
-        row = { district => 'Maynard Public Schools' }
-        expect(DisaggregationRow.new(row:, headers:).district).to eq 'Maynard Public Schools'
+        row = { district => "Maynard Public Schools" }
+        expect(DisaggregationRow.new(row:, headers:).district).to eq "Maynard Public Schools"
       end
     end
   end
 
-  context '.academic_year' do
-    context 'when the column heading is any upper or lowercase variant of the words academic year' do
-      it 'returns the correct value for district' do
-        row = { 'Academic Year' => '2022-23' }
-        expect(DisaggregationRow.new(row:, headers:).academic_year).to eq '2022-23'
+  context ".academic_year" do
+    context "when the column heading is any upper or lowercase variant of the words academic year" do
+      it "returns the correct value for district" do
+        row = { "Academic Year" => "2022-23" }
+        expect(DisaggregationRow.new(row:, headers:).academic_year).to eq "2022-23"
 
-        headers = ['aCADEMIC yEAR']
+        headers = ["aCADEMIC yEAR"]
         headers in [academic_year]
-        row = { academic_year => '2022-23' }
-        expect(DisaggregationRow.new(row:, headers:).academic_year).to eq '2022-23'
+        row = { academic_year => "2022-23" }
+        expect(DisaggregationRow.new(row:, headers:).academic_year).to eq "2022-23"
 
-        headers = ['AcademicYear']
+        headers = ["AcademicYear"]
         headers in [academic_year]
-        row = { academic_year => '2022-23' }
-        expect(DisaggregationRow.new(row:, headers:).academic_year).to eq '2022-23'
+        row = { academic_year => "2022-23" }
+        expect(DisaggregationRow.new(row:, headers:).academic_year).to eq "2022-23"
       end
     end
   end
 
-  context '.income' do
-    context 'when the column heading is any upper or lowercase variant of the words low income' do
-      it 'returns the correct value for low_income' do
-        row = { 'LowIncome' => 'Free Lunch' }
-        expect(DisaggregationRow.new(row:, headers:).income).to eq 'Free Lunch'
+  context ".raw_income" do
+    context "when the column heading is any upper or lowercase variant of the words low income" do
+      it "returns the correct value for low_income" do
+        row = { "LowIncome" => "Free Lunch" }
+        expect(DisaggregationRow.new(row:, headers:).raw_income).to eq "Free Lunch"
 
-        headers = ['Low income']
+        headers = ["Low income"]
         headers in [income]
-        row = { income => 'Free Lunch' }
-        expect(DisaggregationRow.new(row:, headers:).income).to eq 'Free Lunch'
+        row = { income => "Free Lunch" }
+        expect(DisaggregationRow.new(row:, headers:).raw_income).to eq "Free Lunch"
 
-        headers = ['LoW InCOme']
+        headers = ["LoW InCOme"]
         headers in [income]
-        row = { income => 'Free Lunch' }
-        expect(DisaggregationRow.new(row:, headers:).income).to eq 'Free Lunch'
+        row = { income => "Free Lunch" }
+        expect(DisaggregationRow.new(row:, headers:).raw_income).to eq "Free Lunch"
       end
     end
   end
 
-  context '.lasid' do
-    context 'when the column heading is any upper or lowercase variant of the words lasid' do
-      it 'returns the correct value for lasid' do
-        row = { 'LASID' => '2366' }
-        expect(DisaggregationRow.new(row:, headers:).lasid).to eq '2366'
+  context ".lasid" do
+    context "when the column heading is any upper or lowercase variant of the words lasid" do
+      it "returns the correct value for lasid" do
+        row = { "LASID" => "2366" }
+        expect(DisaggregationRow.new(row:, headers:).lasid).to eq "2366"
 
-        headers = ['LaSiD']
+        headers = ["LaSiD"]
         headers in [lasid]
-        row = { lasid => '2366' }
-        expect(DisaggregationRow.new(row:, headers:).lasid).to eq '2366'
+        row = { lasid => "2366" }
+        expect(DisaggregationRow.new(row:, headers:).lasid).to eq "2366"
+      end
+    end
+  end
+
+  context ".ell" do
+    context "when the column heading is any upper or lowercase variant of the words 'ELL' or 'El Student First Year'" do
+      it "returns the correct value for a student" do
+        row = { "EL Student First Year" => "LEP student 1st year" }
+        expect(DisaggregationRow.new(row:, headers:).ell).to eq "ELL"
+
+        headers = ["EL Student First Year"]
+        headers in [ell]
+        row = { ell => "LEP student not 1st year" }
+        expect(DisaggregationRow.new(row:, headers:).ell).to eq "ELL"
+
+        headers = ["EL Student First Year"]
+        headers in [ell]
+        row = { ell => "Does not apply" }
+        expect(DisaggregationRow.new(row:, headers:).ell).to eq "Not ELL"
+
+        headers = ["EL Student First Year"]
+        headers in [ell]
+        row = { ell => "Unknown" }
+        expect(DisaggregationRow.new(row:, headers:).ell).to eq "Unknown"
+
+        headers = ["EL Student First Year"]
+        headers in [ell]
+        row = { ell => "Any other text" }
+        expect(DisaggregationRow.new(row:, headers:).ell).to eq "Unknown"
+
+        headers = ["EL Student First Year"]
+        headers in [ell]
+        row = { ell => "" }
+        expect(DisaggregationRow.new(row:, headers:).ell).to eq "Unknown"
       end
     end
   end

spec/services/student_loader_spec.rb

@@ -88,7 +88,7 @@ describe StudentLoader do
   describe "self.load_data" do
     context "load student data for all schools" do
       before :each do
-        SurveyResponsesDataLoader.load_data filepath: path_to_student_responses
+        SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses
         StudentLoader.load_data filepath: path_to_student_responses
       end
 
@@ -102,7 +102,7 @@ describe StudentLoader do
     # TODO: get this test to run correctly.  Since we are no longer seeding, we need to define schools, and districts; some Lowell, some not
     xcontext "When using the rule to skip non Lowell schools" do
       before :each do
-        SurveyResponsesDataLoader.load_data filepath: path_to_student_responses
+        SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses
         StudentLoader.load_data filepath: path_to_student_responses, rules: [Rule::SkipNonLowellSchools]
       end
 

spec/services/survey_item_values_spec.rb

@@ -7,7 +7,7 @@ RSpec.describe SurveyItemValues, type: :model do
   end
   let(:genders) do
     create(:gender, qualtrics_code: 1)
-    Gender.gender_hash
+    Gender.by_qualtrics_code
   end
   let(:survey_items) { [] }
   let(:district) { create(:district, name: "Attleboro") }
@@ -165,79 +165,90 @@ RSpec.describe SurveyItemValues, type: :model do
   end
 
   context ".income" do
-    context "when no disaggregation data is provided" do
-      it "returns an empty string " do
-        disaggregation_data = {}
-        values = SurveyItemValues.new(row: {}, headers:, genders:, survey_items:, schools:, disaggregation_data:)
-        expect(values.income).to eq "Unknown"
-      end
+    before :each do
+      attleboro
+      ay_2022_23
     end
 
-    context "when disaggregation data is provided" do
-      before :each do
-        attleboro
-        ay_2022_23
-      end
+    it "translates Free Lunch to Economically Disadvantaged - Y" do
+      headers = ["LowIncome"]
+      row = { "LowIncome" => "Free Lunch" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.income).to eq "Economically Disadvantaged - Y"
+    end
 
-      it "translates Free Lunch to Economically Disadvantaged - Y" do
-        headers = ["District", "Academic Year", "LASID", "LowIncome"]
-        row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Free Lunch" }
-        disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) }
+    it "translates Reduced Lunch to Economically Disadvantaged - Y" do
+      headers = ["LowIncome"]
+      row = { "LowIncome" => "Reduced Lunch" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.income).to eq "Economically Disadvantaged - Y"
+    end
 
-        headers = ["LASID", "Dese Id", "RecordedDate"]
-        row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" }
-        values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:,
-                                      disaggregation_data:)
-        expect(values.income).to eq "Economically Disadvantaged - Y"
-      end
+    it "translates LowIncome to Economically Disadvantaged - Y" do
+      headers = ["LowIncome"]
+      row = { "LowIncome" => "LowIncome" }
 
-      it "translates Reduced Lunch to Economically Disadvantaged - Y" do
-        headers = ["District", "Academic Year", "LASID", "LowIncome"]
-        row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Reduced Lunch" }
-        disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.income).to eq "Economically Disadvantaged - Y"
+    end
 
-        headers = ["LASID", "Dese Id", "RecordedDate"]
-        row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" }
-        values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:,
-                                      disaggregation_data:)
-        expect(values.income).to eq "Economically Disadvantaged - Y"
-      end
+    it "translates Not Eligible to Economically Disadvantaged - N" do
+      headers = ["LowIncome"]
+      row = { "LowIncome" => "Not Eligible" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.income).to eq "Economically Disadvantaged - N"
+    end
 
-      it "translates LowIncome to Economically Disadvantaged - Y" do
-        headers = ["District", "Academic Year", "LASID", "LowIncome"]
-        row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "LowIncome" }
-        disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) }
+    it "translates blanks to Unknown" do
+      headers = ["LowIncome"]
+      row = { "LowIncome" => "" }
 
-        headers = ["LASID", "Dese Id", "RecordedDate"]
-        row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" }
-        values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:,
-                                      disaggregation_data:)
-        expect(values.income).to eq "Economically Disadvantaged - Y"
-      end
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.income).to eq "Unknown"
+    end
+  end
 
-      it "translates Not Eligible to Economically Disadvantaged - N" do
-        headers = ["District", "Academic Year", "LASID", "LowIncome"]
-        row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Not Eligible" }
-        disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) }
+  context ".ell" do
+    before :each do
+      attleboro
+      ay_2022_23
+    end
 
-        headers = ["LASID", "Dese Id", "RecordedDate"]
-        row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" }
-        values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:,
-                                      disaggregation_data:)
-        expect(values.income).to eq "Economically Disadvantaged - N"
-      end
+    it 'translates "LEP Student 1st Year" or "LEP Student Not 1st Year" into ELL' do
+      headers = ["Raw ELL"]
+      row = { "Raw ELL" => "LEP Student 1st Year" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.ell).to eq "ELL"
 
-      it "translates blanks to Unknown" do
-        headers = ["District", "Academic Year", "LASID", "LowIncome"]
-        row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "" }
-        disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) }
+      row = { "Raw ELL" => "LEP Student Not 1st Year" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.ell).to eq "ELL"
 
-        headers = ["LASID", "Dese Id", "RecordedDate"]
-        row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" }
-        values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:,
-                                      disaggregation_data:)
-        expect(values.income).to eq "Unknown"
-      end
+      row = { "Raw ELL" => "LEP Student Not 1st Year" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.ell).to eq "ELL"
+    end
+
+    it 'translates "Does not Apply" into "Not ELL"' do
+      headers = ["Raw ELL"]
+      row = { "Raw ELL" => "Does not apply" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.ell).to eq "Not ELL"
+
+      row = { "Raw ELL" => "Does Not APPLY" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.ell).to eq "Not ELL"
+    end
+
+    it 'tranlsates blanks into "Unknown"' do
+      headers = ["Raw ELL"]
+      row = { "Raw ELL" => "" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.ell).to eq "Unknown"
+
+      row = { "Raw ELL" => "Anything else" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.ell).to eq "Unknown"
     end
   end
 

spec/services/survey_responses_data_loader_spec.rb

@@ -54,6 +54,9 @@ describe SurveyResponsesDataLoader do
   let(:low_income) { create(:income, designation: "Economically Disadvantaged – Y") }
   let(:high_income) { create(:income, designation: "Economically Disadvantaged – N") }
   let(:unknown_income) { create(:income, designation: "Unknown") }
+  let(:yes_ell) { create(:ell, designation: "ELL") }
+  let(:not_ell) { create(:ell, designation: "Not ELL") }
+  let(:unknown_ell) { create(:ell, designation: "Unknown") }
 
   let(:setup) do
     ay_2020_21
@@ -92,6 +95,9 @@ describe SurveyResponsesDataLoader do
     low_income
     high_income
     unknown_income
+    yes_ell
+    not_ell
+    unknown_ell
   end
 
   before :each do
@@ -100,7 +106,7 @@ describe SurveyResponsesDataLoader do
 
   describe "loading teacher survey responses" do
     before do
-      SurveyResponsesDataLoader.load_data filepath: path_to_teacher_responses
+      SurveyResponsesDataLoader.new.load_data filepath: path_to_teacher_responses
     end
 
     it "ensures teacher responses load correctly" do
@@ -116,7 +122,7 @@ describe SurveyResponsesDataLoader do
 
   describe "student survey responses" do
     before do
-      SurveyResponsesDataLoader.load_data filepath: path_to_student_responses
+      SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses
     end
 
     it "ensures student responses load correctly" do
@@ -129,13 +135,14 @@ describe SurveyResponsesDataLoader do
       assigns_grade_level_to_responses
       assigns_gender_to_responses
       assigns_income_to_responses
+      assigns_ell_to_responses
       is_idempotent_for_students
     end
 
     context "when updating student survey responses from another csv file" do
       before :each do
-        SurveyResponsesDataLoader.load_data filepath: Rails.root.join("spec", "fixtures",
-                                                                      "secondary_test_2020-21_student_survey_responses.csv")
+        SurveyResponsesDataLoader.new.load_data filepath: Rails.root.join("spec", "fixtures",
+                                                                          "secondary_test_2020-21_student_survey_responses.csv")
       end
       it "updates the likert score to the score on the new csv file" do
         s_emsa_q1 = SurveyItem.find_by_survey_item_id "s-emsa-q1"
@@ -154,8 +161,8 @@ describe SurveyResponsesDataLoader do
   # figure out why this is failing
   describe "when using Lowell rules to skip rows in the csv file" do
     before :each do
-      SurveyResponsesDataLoader.load_data filepath: path_to_student_responses,
-                                          rules: [Rule::SkipNonLowellSchools]
+      SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses,
+                                              rules: [Rule::SkipNonLowellSchools]
     end
 
     it "rejects any non-lowell school" do
@@ -172,8 +179,8 @@ describe SurveyResponsesDataLoader do
 
     context "when loading 22-23 butler survey responses" do
       before :each do
-        SurveyResponsesDataLoader.load_data filepath: path_to_butler_student_responses,
-                                            rules: [Rule::SkipNonLowellSchools]
+        SurveyResponsesDataLoader.new.load_data filepath: path_to_butler_student_responses,
+                                                rules: [Rule::SkipNonLowellSchools]
       end
 
       it "loads all the responses for Butler" do
@@ -235,7 +242,7 @@ end
 def is_idempotent
   number_of_survey_item_responses = SurveyItemResponse.count
 
-  SurveyResponsesDataLoader.load_data filepath: path_to_teacher_responses
+  SurveyResponsesDataLoader.new.load_data filepath: path_to_teacher_responses
 
   expect(SurveyItemResponse.count).to eq number_of_survey_item_responses
 end
@@ -281,7 +288,7 @@ end
 def is_idempotent_for_students
   number_of_survey_item_responses = SurveyItemResponse.count
 
-  SurveyResponsesDataLoader.load_data filepath: path_to_student_responses
+  SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses
 
   expect(SurveyItemResponse.count).to eq number_of_survey_item_responses
 end
@@ -344,6 +351,21 @@ def assigns_income_to_responses
               "student_survey_response_7" => low_income }
 
   results.each do |key, value|
-    expect(SurveyItemResponse.where(response_id: key).first.income).to eq value
+    income = SurveyItemResponse.find_by_response_id(key).income
+    expect(income).to eq value
+  end
+end
+
+def assigns_ell_to_responses
+  results = { "student_survey_response_1" => not_ell,
+              "student_survey_response_3" => unknown_ell,
+              "student_survey_response_4" => yes_ell,
+              "student_survey_response_5" => yes_ell,
+              "student_survey_response_6" => unknown_ell,
+              "student_survey_response_7" => unknown_ell }
+
+  results.each do |key, value|
+    ell = SurveyItemResponse.find_by_response_id(key).ell
+    expect(ell).to eq value
   end
 end