feat: load student responses in the same pass as loading the survey responses

chore: remove student loader since loading students is now done with the survey response loader

app/services/student_loader.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-class StudentLoader
-  def self.load_data(filepath:, rules: [])
-    File.open(filepath) do |file|
-      headers = file.first
-      headers_array = headers.split(",")
-
-      file.lazy.each_slice(1_000) do |lines|
-        CSV.parse(lines.join, headers:).map do |row|
-          row = SurveyItemValues.new(row:, headers: headers_array, genders: nil, survey_items: nil, schools:)
-          next if rules.any? do |rule|
-                    rule.new(row:).skip_row?
-                  end
-
-          process_row(row:)
-        end
-      end
-    end
-  end
-
-  def self.from_file(file:, rules: [])
-    headers = file.gets
-    headers_array = headers.split(",")
-
-    until file.eof?
-      line = file.gets
-      next unless line.present?
-
-      CSV.parse(line, headers:).map do |row|
-        row = SurveyItemValues.new(row:, headers: headers_array, genders: nil, survey_items: nil, schools:)
-        next if rules.any? do |rule|
-                  rule.new(row:).skip_row?
-                end
-
-        process_row(row:)
-      end
-    end
-  end
-
-  def self.process_row(row:)
-    student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid)
-    student.races.delete_all
-    races = row.races
-    races.map do |race|
-      student.races << race
-    end
-    assign_student_to_responses(student:, response_id: row.response_id)
-  end
-
-  def self.schools
-    @schools ||= School.all.map { |school| [school.dese_id, school] }.to_h
-  end
-
-  def self.assign_student_to_responses(student:, response_id:)
-    responses = SurveyItemResponse.where(response_id:)
-    loadable_responses = responses.map do |response|
-      response.student = student
-      response
-    end
-
-    SurveyItemResponse.import(loadable_responses.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
-  end
-end

app/services/survey_item_values.rb

@@ -116,20 +116,24 @@ class SurveyItemValues
   end
 
   def gender
-    gender_code = value_from(pattern: /Gender|What is your gender?|What is your gender? - Selected Choice/i)
+    @gender ||= begin
-    gender_code ||= 99
+      gender_code = value_from(pattern: /Gender|What is your gender?|What is your gender? - Selected Choice/i)
-    gender_code = gender_code.to_i
+      gender_code ||= 99
-    gender_code = 4 if gender_code == 3
+      gender_code = gender_code.to_i
-    gender_code = 99 if gender_code.zero?
+      gender_code = 4 if gender_code == 3
-    genders[gender_code]
+      gender_code = 99 if gender_code.zero?
+      genders[gender_code]
+    end
   end
 
   def races
-    race_codes = value_from(pattern: /RACE/i)
+    @races ||= begin
-    race_codes ||= value_from(pattern: %r{What is your race/ethnicity?(Please select all that apply) - Selected Choice}i)
+      race_codes = value_from(pattern: /RACE/i)
-    race_codes ||= value_from(pattern: /Race Secondary/i) || ""
+      race_codes ||= value_from(pattern: %r{What is your race/ethnicity?(Please select all that apply) - Selected Choice}i)
-    race_codes = race_codes.split(",").map(&:to_i) || []
+      race_codes ||= value_from(pattern: /Race Secondary/i) || ""
-    process_races(codes: race_codes)
+      race_codes = race_codes.split(",").map(&:to_i) || []
+      process_races(codes: race_codes)
+    end
   end
 
   def lasid

app/services/survey_responses_data_loader.rb

@@ -12,7 +12,7 @@ class SurveyResponsesDataLoader
           process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:),
                       rules:)
         end
-        SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500
+        SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all
       end
     end
   end
@@ -23,7 +23,6 @@ class SurveyResponsesDataLoader
     all_survey_items = survey_items(headers:)
 
     survey_item_responses = []
-    row_count = 0
     until file.eof?
       line = file.gets
       next unless line.present?
@@ -32,16 +31,9 @@ class SurveyResponsesDataLoader
         survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:),
                                              rules:)
       end
-
-      row_count += 1
-      next unless row_count == 1000
-
-      SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000
-      survey_item_responses = []
-      row_count = 0
     end
 
-    SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000
+    SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000, on_duplicate_key_update: :all
   end
 
   private
@@ -78,6 +70,13 @@ class SurveyResponsesDataLoader
   end
 
   def process_survey_items(row:)
+    student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid)
+    student.races.delete_all
+    races = row.races
+    races.map do |race|
+      student.races << race
+    end
+
     row.survey_items.map do |survey_item|
       likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next
 
@@ -86,23 +85,30 @@ class SurveyResponsesDataLoader
         next
       end
       response = row.survey_item_response(survey_item:)
-      create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:)
+      create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, student:)
     end.compact
   end
 
-  def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:)
+  def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, student:)
     gender = row.gender
     grade = row.grade
     income = incomes[row.income.parameterize]
     ell = ells[row.ell]
     sped = speds[row.sped]
+
     if survey_item_response.present?
-      survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:,
+      survey_item_response.likert_score = likert_score
-                                   sped:)
+      survey_item_response.grade = grade
-      []
+      survey_item_response.gender = gender
+      survey_item_response.recorded_date = row.recorded_date
+      survey_item_response.income = income
+      survey_item_response.ell = ell
+      survey_item_response.sped = sped
+      survey_item_response.student = student
+      survey_item_response
     else
       SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:,
-                             likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:)
+                             likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:, student:)
     end
   end
 

lib/tasks/data.rake

@@ -9,11 +9,6 @@ namespace :data do
     end
     puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
 
-    Sftp::Directory.open(path:) do |file|
-      StudentLoader.from_file(file:, rules: [])
-    end
-    puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
-
     Rails.cache.clear
   end
 
@@ -27,11 +22,6 @@ namespace :data do
     end
     puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
 
-    Sftp::Directory.open(path:) do |file|
-      StudentLoader.from_file(file:, rules: [])
-    end
-    puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
-
     Rails.cache.clear
   end
 
@@ -50,20 +40,6 @@ namespace :data do
     puts "=====================> Completed loading #{AdminDataValue.count - original_count} admin data values"
   end
 
-  desc "load students"
-  task load_students: :environment do
-    SurveyItemResponse.update_all(student_id: nil)
-    StudentRace.delete_all
-    Student.delete_all
-    Dir.glob(Rails.root.join("data", "survey_responses", "*student*.csv")).each do |file|
-      puts "=====================> Loading student data from csv at path: #{file}"
-      StudentLoader.load_data filepath: file
-    end
-    puts "=====================> Completed loading #{Student.count} students"
-
-    Rails.cache.clear
-  end
-
   desc "reset all cache counters"
   task reset_cache_counters: :environment do
     puts "=====================> Resetting Category counters"

lib/tasks/one_off.rake

@@ -5,10 +5,10 @@ namespace :one_off do
 
     qualtrics_schools = {}
 
-    csv_file = Rails.root.join('data', 'master_list_of_schools_and_districts.csv')
+    csv_file = Rails.root.join("data", "master_list_of_schools_and_districts.csv")
     CSV.parse(File.read(csv_file), headers: true) do |row|
-      district_id = row['District Code'].to_i
+      district_id = row["District Code"].to_i
-      school_id = row['School Code'].to_i
+      school_id = row["School Code"].to_i
 
       if qualtrics_schools[[district_id, school_id]].present?
         puts "Duplicate entry row #{row}"
@@ -24,14 +24,14 @@ namespace :one_off do
       end
 
       if school.nil?
-        school_name = csv_row['School Name'].strip
+        school_name = csv_row["School Name"].strip
         puts "Could not find school '#{school_name}' with district id: #{district_id}, school id: #{school_id}"
-        potential_school_ids = School.where('name like ?', "%#{school_name}%").map(&:id)
+        potential_school_ids = School.where("name like ?", "%#{school_name}%").map(&:id)
         puts "Potential ID matches: #{potential_school_ids}" if potential_school_ids.present?
         next
       end
 
-      school.update!(dese_id: csv_row['DESE School ID'])
+      school.update!(dese_id: csv_row["DESE School ID"])
       updated_schools << school.id
     end
 
@@ -40,7 +40,7 @@ namespace :one_off do
     end
   end
 
-  desc 'list scales that have no survey responses'
+  desc "list scales that have no survey responses"
   task list_scales_that_lack_survey_responses: :environment do
     output = AcademicYear.all.map do |academic_year|
       Scale.all.map do |scale|
@@ -48,11 +48,11 @@ namespace :one_off do
       end
     end
 
-    output = output.map { |year| year.reject { |scale| scale[2] > 0 || scale[1].starts_with?('a-') } }
+    output = output.map { |year| year.reject { |scale| scale[2] > 0 || scale[1].starts_with?("a-") } }
     pp output
   end
 
-  desc 'list survey_items that have no survey responses by district'
+  desc "list survey_items that have no survey responses by district"
   task list_survey_items_that_lack_responses: :environment do
     output = AcademicYear.all.map do |academic_year|
       District.all.map do |district|
@@ -66,14 +66,14 @@ namespace :one_off do
     output = output.map do |year|
       year.map do |district|
         district.reject do |survey_item|
-          survey_item[2] > 0 || survey_item[1].starts_with?('a-')
+          survey_item[2] > 0 || survey_item[1].starts_with?("a-")
         end
       end
     end
     pp output
   end
 
-  desc 'list the most recent admin data values'
+  desc "list the most recent admin data values"
   task list_recent_admin_data_values: :environment do
     range = 4.weeks.ago..1.second.ago
     values = AdminDataValue.where(updated_at: range).group(:admin_data_item).count.map do |item|
@@ -82,240 +82,27 @@ namespace :one_off do
     puts values
   end
 
-  desc 'delete 2022-23 survey responses'
+  desc "delete 2022-23 survey responses"
   task delete_survey_responses_2022_23: :environment do
     response_count = SurveyItemResponse.all.count
-    SurveyItemResponse.where(academic_year: AcademicYear.find_by_range('2022-23')).delete_all
+    SurveyItemResponse.where(academic_year: AcademicYear.find_by_range("2022-23")).delete_all
 
     puts "=====================> Deleted #{response_count - SurveyItemResponse.all.count} survey responses"
     # should be somewhere near 295738
   end
 
-  desc 'load survey responses'
+  desc "load survey responses"
   task load_survey_responses: :environment do
     survey_item_response_count = SurveyItemResponse.count
     student_count = Student.count
-    path = '/data/survey_responses/clean/'
+    path = "/data/survey_responses/clean/"
-    schools = District.find_by_slug('maynard-public-schools').schools
+    schools = District.find_by_slug("maynard-public-schools").schools
 
     Sftp::Directory.open(path:) do |file|
       SurveyResponsesDataLoader.new.from_file(file:)
     end
     puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
 
-    Sftp::Directory.open(path:) do |file|
-      StudentLoader.from_file(file:, rules: [])
-    end
-    puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
-
     Rails.cache.clear
   end
-
-  desc "Generate CSV report of teacher survey item responses"
-  task teacher_survey_questions_csv: :environment do
-    headers = ['School ID', 'Academic Year', 'Survey Item', 'Count','Percentage_diff']
-    output_rows = []
-    counts={}
-    avg=0
-    School.all.each do |sc|
-      AcademicYear.all.each do |ay|
-        sum=0
-        nof_si=0
-        threshold = Respondent.where(school: sc, academic_year: ay).pluck(:total_teachers)
-        SurveyItem.teacher_survey_items.all.each do |si|
-          c = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).group(:school_id).count
-          if c.any? && (c.values.first >= 10 || c.values.first > threshold/4)
-            counts[[sc.id, ay.id, si.id]] = c
-            sum+=c.values.first
-            nof_si+=1
-          end
-        end
-
-
-          avg = sum.to_f/ nof_si
-        counts.each do |key, value|
-          if key[0] == sc.id && key[1] == ay.id
-            count = value.values.first
-            percentage_diff = ((count-avg) / avg) * 100
-            counts[key] = { count: count, percentage_diff: percentage_diff }
-        end
-      end
-    end  end
-
-    counts.each do |key, value|
-      output_rows << [School.where(id:key[0]).pluck(:name) , AcademicYear.where(id:key[1]).pluck(:range) , SurveyItem.where(id:key[2]).pluck(:survey_item_id), SurveyItem.where(id:key[2]).pluck(:prompt), value[:count], value[:percentage_diff]]
-    end
-
-    file = File.new('teacher_survey_questions.csv', 'w')
-    CSV.open(file, 'w', write_headers: true, headers: headers) do |csv|
-      output_rows.each do |row|
-        csv << row
-      end
-    end
-
-    file.close
-    puts "CSV report of teacher survey item responses with removed stray responses.csv"
-  end
-
-
-  desc "Generate CSV report of short survey item responses"
-  task short_survey_questions_csv: :environment do
-    headers = ['School ID', 'Academic Year', 'Survey Item', 'Count','Percentage_diff']
-    output_rows = []
-    counts={}
-    avg=0
-    temp=0
-    count_response_grades_not_in_grades=0
-    School.all.each do |sc|
-      if Respondent.where(school: sc).any?
-      grades_count=Respondent.where(school:sc).first.counts_by_grade
-      grades= grades_count.keys
-      AcademicYear.all.each do |ay|
-        sum=0
-        nof_si=0
-        threshold=0
-        threshold = Respondent.where(school: sc, academic_year: ay).pluck(:total_students)
-        SurveyItem.short_form_items.all.each do |si|
-          c = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).group(:school_id).count
-          response_grades = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).pluck(:grade)
-          count_response_grades_not_in_grades = response_grades.count { |grade| !grades.include?(grade) }
-           if !count_response_grades_not_in_grades.nil? && c.any?
-            c[c.keys.first] = c.values.first - count_response_grades_not_in_grades
-          end
-
-           if threshold.any?
-          if c.any? && (c.values.first >= 10 || c.values.first > threshold.first/4)
-            counts[[sc.id, ay.id, si.id]] = c
-            sum+=c.values.first
-            nof_si+=1
-          end
-        end
-        end
-
-
-          avg = sum.to_f/ nof_si
-        counts.each do |key, value|
-          if key[0] == sc.id && key[1] == ay.id
-            count = value.values.first
-            percentage_diff = ((count-avg) / avg) * 100
-            counts[key] = { count: count, percentage_diff: percentage_diff }
-        end
-      end
-    end  end end
-
-    counts.each do |key, value|
-      output_rows << [School.where(id:key[0]).pluck(:name) , AcademicYear.where(id:key[1]).pluck(:range) , SurveyItem.where(id:key[2]).pluck(:survey_item_id), SurveyItem.where(id:key[2]).pluck(:prompt), value[:count], value[:percentage_diff]]
-    end
-
-    file = File.new('short_survey_questions.csv', 'w')
-    CSV.open(file, 'w', write_headers: true, headers: headers) do |csv|
-      output_rows.each do |row|
-        csv << row
-      end
-    end
-
-    file.close
-    puts "CSV report of short survey item responses with removed stray responses.csv"
-  end
-
-
-    desc "Generate CSV report of early_education_surveysitem responses"
-  task early_education_survey_questions_csv: :environment do
-    headers = ['School ID', 'Academic Year', 'Survey Item', 'Count','Percentage_diff']
-    output_rows = []
-    counts={}
-    avg=0
-    temp=0
-    count_response_grades_not_in_grades=0
-    School.all.each do |sc|
-      if Respondent.where(school: sc).any?
-      grades_count=Respondent.where(school:sc).first.counts_by_grade
-      grades= grades_count.keys
-      AcademicYear.all.each do |ay|
-        sum=0
-        nof_si=0
-        threshold=0
-        threshold = Respondent.where(school: sc, academic_year: ay).pluck(:total_students)
-        SurveyItem.early_education_surveys.all.each do |si|
-          c = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).group(:school_id).count
-          response_grades = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).pluck(:grade)
-          count_response_grades_not_in_grades = response_grades.count { |grade| !grades.include?(grade) }
-           if !count_response_grades_not_in_grades.nil? && c.any?
-            c[c.keys.first] = c.values.first - count_response_grades_not_in_grades
-          end
-
-           if threshold.any?
-          if c.any? && (c.values.first >= 10 || c.values.first > threshold.first/4)
-            counts[[sc.id, ay.id, si.id]] = c
-            sum+=c.values.first
-            nof_si+=1
-          end
-        end
-        end
-
-
-          avg = sum.to_f/ nof_si
-        counts.each do |key, value|
-          if key[0] == sc.id && key[1] == ay.id
-            count = value.values.first
-            percentage_diff = ((count-avg) / avg) * 100
-            counts[key] = { count: count, percentage_diff: percentage_diff }
-        end
-      end
-    end  end end
-
-    counts.each do |key, value|
-      output_rows << [School.where(id:key[0]).pluck(:name) , AcademicYear.where(id:key[1]).pluck(:range) , SurveyItem.where(id:key[2]).pluck(:survey_item_id), SurveyItem.where(id:key[2]).pluck(:prompt), value[:count], value[:percentage_diff]]
-    end
-
-    file = File.new('early_education_surveys_questions.csv', 'w')
-    CSV.open(file, 'w', write_headers: true, headers: headers) do |csv|
-      output_rows.each do |row|
-        csv << row
-      end
-    end
-
-    file.close
-    puts "CSV report of early_education_surveys items with removed stray responses.csv"
-  end
-
-
-  desc "Generate CSV report of survey item responses"
-  task stray_responses: :environment do
-
-    headers = ['School ID', 'Academic Year', 'Survey Item', 'Count','SurveyItemResponse ids']
-
-
-    output_rows = []
-    sir_ids=[]
-
-
-      School.all.each do |sc|
-      AcademicYear.all.each do |ay|
-        SurveyItem.all.each do |si|
-          count = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).count
-          sir_ids= SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).pluck(:response_id)
-
-          if count > 0 && count < 10
-
-            output_rows << [sc.name, ay.range, si.survey_item_id, count,sir_ids]
-          end
-        end
-      end
-    end
-
-
-    file = File.new('stray_responses.csv', 'w')
-
-    CSV.open(file, 'w', write_headers: true, headers: headers) do |csv|
-      output_rows.each do |row|
-        csv << row
-      end
-    end
-
-
-    file.close
-
-    puts "CSV report of survey item responses created in stray_responses.csv"
-  end
 end

spec/services/student_loader_spec.rb

@@ -1,160 +0,0 @@
-require "rails_helper"
-
-describe StudentLoader do
-  let(:path_to_student_responses) { Rails.root.join("spec", "fixtures", "test_2020-21_student_survey_responses.csv") }
-  let(:american_indian) { create(:race, qualtrics_code: 1) }
-  let(:asian)           { create(:race, qualtrics_code: 2) }
-  let(:black)           { create(:race, qualtrics_code: 3) }
-  let(:latinx)          { create(:race, qualtrics_code: 4) }
-  let(:white)           { create(:race, qualtrics_code: 5) }
-  let(:middle_eastern)  { create(:race, qualtrics_code: 8) }
-  let(:unknown_race)    { create(:race, qualtrics_code: 99) }
-  let(:multiracial)     { create(:race, qualtrics_code: 100) }
-  let(:female)          { create(:gender, qualtrics_code: 1) }
-  let(:male)            { create(:gender, qualtrics_code: 2) }
-  let(:another_gender)  { create(:gender, qualtrics_code: 3) }
-  let(:non_binary)      { create(:gender, qualtrics_code: 4) }
-  let(:unknown_gender)  { create(:gender, qualtrics_code: 99) }
-
-  before :each do
-    american_indian
-    asian
-    black
-    latinx
-    white
-    middle_eastern
-    unknown_race
-    multiracial
-    female
-    male
-    another_gender
-    non_binary
-    unknown_gender
-  end
-
-  after :each do
-    DatabaseCleaner.clean
-  end
-  xdescribe "#process_races" do
-    context "as a standalone function" do
-      it "race codes of 6 or 7 get classified as an unknown race" do
-        codes = ["NA"]
-        expect(StudentLoader.process_races(codes:)).to eq [unknown_race]
-        codes = []
-        expect(StudentLoader.process_races(codes:)).to eq [unknown_race]
-        codes = [1]
-        expect(StudentLoader.process_races(codes:)).to eq [american_indian]
-        codes = [2]
-        expect(StudentLoader.process_races(codes:)).to eq [asian]
-        codes = [3]
-        expect(StudentLoader.process_races(codes:)).to eq [black]
-        codes = [4]
-        expect(StudentLoader.process_races(codes:)).to eq [latinx]
-        codes = [5]
-        expect(StudentLoader.process_races(codes:)).to eq [white]
-        codes = [8]
-        expect(StudentLoader.process_races(codes:)).to eq [middle_eastern]
-        codes = [6]
-        expect(StudentLoader.process_races(codes:)).to eq [unknown_race]
-        codes = [7]
-        expect(StudentLoader.process_races(codes:)).to eq [unknown_race]
-        codes = [6, 7]
-        expect(StudentLoader.process_races(codes:)).to eq [unknown_race]
-        codes = [1, 6, 7]
-        expect(StudentLoader.process_races(codes:)).to eq [american_indian]
-        codes = [1, 6, 7, 2]
-        expect(StudentLoader.process_races(codes:)).to eq [american_indian, asian, multiracial]
-        codes = [3, 6, 7, 6, 6, 7, 7, 6, 2]
-        expect(StudentLoader.process_races(codes:)).to eq [black, asian, multiracial]
-        codes = [8, 2]
-        expect(StudentLoader.process_races(codes:)).to eq [middle_eastern, asian, multiracial]
-      end
-    end
-  end
-
-  xdescribe "#add_multiracial_designation" do
-    it "adds the multiracial race code to the list of races" do
-      races = [unknown_race]
-      expect(StudentLoader.add_multiracial_designation(races:)).to eq [unknown_race]
-      races = [american_indian, asian]
-      expect(StudentLoader.add_multiracial_designation(races:)).to eq [american_indian, asian, multiracial]
-      races = [white]
-      expect(StudentLoader.add_multiracial_designation(races:)).to eq [white]
-    end
-  end
-
-  # This fails in CI because github does not know what the key derivation salt is.
-  # I'm not sure how to securely set the key derivation salt as an environment variable in CI
-  describe "self.load_data" do
-    context "load student data for all schools" do
-      before :each do
-        SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses
-        StudentLoader.load_data filepath: path_to_student_responses
-      end
-
-      it "ensures student responses load correctly" do
-        assigns_student_to_the_survey_item_responses
-        assigns_races_to_students
-        is_idempotent_for_students
-      end
-    end
-
-    # TODO: get this test to run correctly.  Since we are no longer seeding, we need to define schools, and districts; some Lowell, some not
-    xcontext "When using the rule to skip non Lowell schools" do
-      before :each do
-        SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses
-        StudentLoader.load_data filepath: path_to_student_responses, rules: [Rule::SkipNonLowellSchools]
-      end
-
-      it "only loads student data for lowell" do
-        expect(Student.find_by_response_id("student_survey_response_1")).to eq nil
-        expect(Student.find_by_response_id("student_survey_response_3").races).to eq [unknown_race]
-        expect(Student.find_by_response_id("student_survey_response_4").races).to eq [unknown_race]
-        expect(Student.find_by_response_id("student_survey_response_5").races).to eq [american_indian, asian, black, latinx, white,
-                                                                                      middle_eastern, multiracial]
-        expect(Student.find_by_response_id("student_survey_response_6").races).to eq [american_indian, asian, black, latinx, white,
-                                                                                      middle_eastern, multiracial]
-        expect(Student.find_by_response_id("student_survey_response_7").races).to eq [unknown_race]
-      end
-    end
-  end
-end
-
-def assigns_student_to_the_survey_item_responses
-  # The csv file has no responses for `student_survey_response_2` so we can't assign a student to nil responses
-  expect(SurveyItemResponse.find_by_response_id("student_survey_response_2")).to eq nil
-
-  response_ids = %w[student_survey_response_1 student_survey_response_3
-                    student_survey_response_4
-                    student_survey_response_5
-                    student_survey_response_6
-                    student_survey_response_7]
-
-  response_ids.each do |response_id|
-    responses = SurveyItemResponse.where(response_id:)
-    responses.each do |response|
-      expect(response.student).not_to eq nil
-      expect(response.student).to eq Student.find_by_response_id(response_id)
-    end
-  end
-end
-
-def assigns_races_to_students
-  expect(Student.find_by_response_id("student_survey_response_1").races).to eq [american_indian]
-  expect(Student.find_by_response_id("student_survey_response_2").races).to eq [asian, black, latinx, multiracial]
-  expect(Student.find_by_response_id("student_survey_response_3").races).to eq [unknown_race]
-  expect(Student.find_by_response_id("student_survey_response_4").races).to eq [unknown_race]
-  expect(Student.find_by_response_id("student_survey_response_5").races).to eq [american_indian, asian, black, latinx, white,
-                                                                                middle_eastern, multiracial]
-  expect(Student.find_by_response_id("student_survey_response_6").races).to eq [american_indian, asian, black, latinx, white,
-                                                                                middle_eastern, multiracial]
-  expect(Student.find_by_response_id("student_survey_response_7").races).to eq [unknown_race]
-end
-
-def is_idempotent_for_students
-  number_of_students = Student.count
-  number_of_responses = SurveyItemResponse.count
-  StudentLoader.load_data filepath: path_to_student_responses
-  expect(Student.count).to eq number_of_students
-  expect(SurveyItemResponse.count).to eq number_of_responses
-end

spec/services/survey_responses_data_loader_spec.rb

@@ -51,13 +51,24 @@ describe SurveyResponsesDataLoader do
   let(:another_gender) { create(:gender, qualtrics_code: 3) }
   let(:non_binary) { create(:gender, qualtrics_code: 4) }
   let(:unknown_gender) { create(:gender, qualtrics_code: 99) }
+
   let(:low_income) { create(:income, designation: "Economically Disadvantaged – Y") }
   let(:high_income) { create(:income, designation: "Economically Disadvantaged – N") }
   let(:unknown_income) { create(:income, designation: "Unknown") }
+
   let(:yes_ell) { create(:ell, designation: "ELL") }
   let(:not_ell) { create(:ell, designation: "Not ELL") }
   let(:unknown_ell) { create(:ell, designation: "Unknown") }
 
+  let(:american_indian) { create(:race, qualtrics_code: 1) }
+  let(:asian)           { create(:race, qualtrics_code: 2) }
+  let(:black)           { create(:race, qualtrics_code: 3) }
+  let(:latinx)          { create(:race, qualtrics_code: 4) }
+  let(:white)           { create(:race, qualtrics_code: 5) }
+  let(:middle_eastern)  { create(:race, qualtrics_code: 8) }
+  let(:unknown_race)    { create(:race, qualtrics_code: 99) }
+  let(:multiracial)     { create(:race, qualtrics_code: 100) }
+
   let(:setup) do
     ay_2020_21
     ay_2022_23
@@ -87,17 +98,29 @@ describe SurveyResponsesDataLoader do
     s_emsa_q1
     s_emsa_q2
     s_emsa_q3
+
     female
     male
     another_gender
     non_binary
     unknown_gender
+
     low_income
     high_income
     unknown_income
+
     yes_ell
     not_ell
     unknown_ell
+
+    american_indian
+    asian
+    black
+    latinx
+    white
+    middle_eastern
+    unknown_race
+    multiracial
   end
 
   before :each do