Add rules to the survey data loader to allow skipping non-lowell schools

app/services/rule/no_rule.rb

@@ -0,0 +1,9 @@
+module Rule
+  class NoRule
+    def initialize(row:); end
+
+    def skip_row?
+      false
+    end
+  end
+end

app/services/rule/skip_non_lowell_schools.rb

@@ -0,0 +1,13 @@
+module Rule
+  class SkipNonLowellSchools
+    attr_reader :row
+
+    def initialize(row:)
+      @row = row
+    end
+
+    def skip_row?
+      row.school.district.name != 'Lowell'
+    end
+  end
+end

app/services/survey_responses_data_loader.rb

@@ -3,7 +3,7 @@
 require 'csv'
 
 class SurveyResponsesDataLoader
-  def self.load_data(filepath:)
+  def self.load_data(filepath:, rules: [Rule::NoRule])
     File.open(filepath) do |file|
       headers = file.first
       genders_hash = genders
@@ -11,7 +11,8 @@ class SurveyResponsesDataLoader
 
       file.lazy.each_slice(500) do |lines|
         survey_item_responses = CSV.parse(lines.join, headers:).map do |row|
-          process_row row: Values.new(row:, headers:, genders: genders_hash, survey_items: all_survey_items)
+          process_row(row: Values.new(row:, headers:, genders: genders_hash, survey_items: all_survey_items),
+                      rules:)
         end
 
         SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500
@@ -21,10 +22,14 @@ class SurveyResponsesDataLoader
 
   private
 
-  def self.process_row(row:)
+  def self.process_row(row:, rules:)
     return unless row.dese_id?
     return unless row.school.present?
 
+    rules.each do |rule|
+      return if rule.new(row:).skip_row?
+    end
+
     process_survey_items(row:)
   end
 

lib/tasks/data.rake

@@ -20,6 +20,25 @@ namespace :data do
     Rails.cache.clear
   end
 
+  desc 'load survey responses for lowell schools'
+  task load_survey_responses_for_lowell: :environment do
+    Dir.glob(Rails.root.join('data', 'survey_responses', '*.csv')).each do |filepath|
+      puts "=====================> Loading data from csv at path: #{filepath}"
+      SurveyResponsesDataLoader.load_data filepath:, rules: [Rule::SkipNonLowellSchools]
+    end
+    puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses"
+
+    puts 'Resetting response rates'
+    ResponseRateLoader.reset
+    puts "=====================> Completed loading #{ResponseRate.count} survey responses"
+
+    puts 'Resetting race scores'
+    RaceScoreLoader.reset(fast_processing: false)
+    puts "=====================> Completed loading #{RaceScore.count} survey responses"
+
+    Rails.cache.clear
+  end
+
   task load_survey_responses_21_22: :environment do
     Dir.glob(Rails.root.join('data', 'survey_responses', '*2021-22*.csv')).each do |filepath|
       puts "=====================> Loading data from csv at path: #{filepath}"

spec/services/survey_responses_data_loader_spec.rb

@@ -14,11 +14,11 @@ describe SurveyResponsesDataLoader do
   let(:s_phys_q1) { SurveyItem.find_by_survey_item_id 's-phys-q1' }
   let(:s_phys_q2) { SurveyItem.find_by_survey_item_id 's-phys-q2' }
 
-  let(:female) {Gender.find_by_qualtrics_code 1}
-  let(:male) {Gender.find_by_qualtrics_code 2}
-  let(:another_gender) {Gender.find_by_qualtrics_code 3}
-  let(:non_binary) {Gender.find_by_qualtrics_code 4}
-  let(:unknown_gender) {Gender.find_by_qualtrics_code 99 }
+  let(:female) { Gender.find_by_qualtrics_code 1 }
+  let(:male) { Gender.find_by_qualtrics_code 2 }
+  let(:another_gender) { Gender.find_by_qualtrics_code 3 }
+  let(:non_binary) { Gender.find_by_qualtrics_code 4 }
+  let(:unknown_gender) { Gender.find_by_qualtrics_code 99 }
 
   before :each do
     Rails.application.load_seed
@@ -75,6 +75,25 @@ describe SurveyResponsesDataLoader do
         end
       end
     end
+
+    context 'when using Lowell rules to skip rows in the csv file' do
+      before :each do
+        SurveyResponsesDataLoader.load_data filepath: path_to_student_responses,
+                                            rules: [Rule::SkipNonLowellSchools]
+      end
+
+      it 'rejects any non-lowell school' do
+        expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').count).to eq 0
+        expect(SurveyItemResponse.count).to eq 128
+      end
+
+      it 'loads the correct number of responses for lowell schools' do
+        expect(SurveyItemResponse.where(response_id: 'student_survey_response_2').count).to eq 0
+        expect(SurveyItemResponse.where(response_id: 'student_survey_response_3').count).to eq 25
+        expect(SurveyItemResponse.where(response_id: 'student_survey_response_4').count).to eq 22
+        expect(SurveyItemResponse.where(response_id: 'student_survey_response_5').count).to eq 27
+      end
+    end
   end
 end