From 9bbba64e9e0088999aa407a11f5a2a2e6eafae1c Mon Sep 17 00:00:00 2001 From: rebuilt Date: Thu, 1 Dec 2022 16:11:23 -0800 Subject: [PATCH] Add rules to the survey data loader to allow skipping non-lowell schools --- app/services/rule/no_rule.rb | 9 ++++++ app/services/rule/skip_non_lowell_schools.rb | 13 +++++++++ app/services/survey_responses_data_loader.rb | 11 +++++-- lib/tasks/data.rake | 19 ++++++++++++ .../survey_responses_data_loader_spec.rb | 29 +++++++++++++++---- 5 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 app/services/rule/no_rule.rb create mode 100644 app/services/rule/skip_non_lowell_schools.rb diff --git a/app/services/rule/no_rule.rb b/app/services/rule/no_rule.rb new file mode 100644 index 00000000..16dd6bf9 --- /dev/null +++ b/app/services/rule/no_rule.rb @@ -0,0 +1,9 @@ +module Rule + class NoRule + def initialize(row:); end + + def skip_row? + false + end + end +end diff --git a/app/services/rule/skip_non_lowell_schools.rb b/app/services/rule/skip_non_lowell_schools.rb new file mode 100644 index 00000000..c791fd63 --- /dev/null +++ b/app/services/rule/skip_non_lowell_schools.rb @@ -0,0 +1,13 @@ +module Rule + class SkipNonLowellSchools + attr_reader :row + + def initialize(row:) + @row = row + end + + def skip_row? + row.school.district.name != 'Lowell' + end + end +end diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index 9f64d00b..45bbe05b 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -3,7 +3,7 @@ require 'csv' class SurveyResponsesDataLoader - def self.load_data(filepath:) + def self.load_data(filepath:, rules: [Rule::NoRule]) File.open(filepath) do |file| headers = file.first genders_hash = genders @@ -11,7 +11,8 @@ class SurveyResponsesDataLoader file.lazy.each_slice(500) do |lines| survey_item_responses = CSV.parse(lines.join, headers:).map do |row| - process_row row: Values.new(row:, headers:, genders: genders_hash, survey_items: all_survey_items) + process_row(row: Values.new(row:, headers:, genders: genders_hash, survey_items: all_survey_items), + rules:) end SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500 @@ -21,10 +22,14 @@ class SurveyResponsesDataLoader private - def self.process_row(row:) + def self.process_row(row:, rules:) return unless row.dese_id? return unless row.school.present? + rules.each do |rule| + return if rule.new(row:).skip_row? + end + process_survey_items(row:) end diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake index c93fc242..feb27e22 100644 --- a/lib/tasks/data.rake +++ b/lib/tasks/data.rake @@ -20,6 +20,25 @@ namespace :data do Rails.cache.clear end + desc 'load survey responses for lowell schools' + task load_survey_responses_for_lowell: :environment do + Dir.glob(Rails.root.join('data', 'survey_responses', '*.csv')).each do |filepath| + puts "=====================> Loading data from csv at path: #{filepath}" + SurveyResponsesDataLoader.load_data filepath:, rules: [Rule::SkipNonLowellSchools] + end + puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses" + + puts 'Resetting response rates' + ResponseRateLoader.reset + puts "=====================> Completed loading #{ResponseRate.count} survey responses" + + puts 'Resetting race scores' + RaceScoreLoader.reset(fast_processing: false) + puts "=====================> Completed loading #{RaceScore.count} survey responses" + + Rails.cache.clear + end + task load_survey_responses_21_22: :environment do Dir.glob(Rails.root.join('data', 'survey_responses', '*2021-22*.csv')).each do |filepath| puts "=====================> Loading data from csv at path: #{filepath}" diff --git a/spec/services/survey_responses_data_loader_spec.rb b/spec/services/survey_responses_data_loader_spec.rb index 8db45f23..824063db 100644 --- a/spec/services/survey_responses_data_loader_spec.rb +++ b/spec/services/survey_responses_data_loader_spec.rb @@ -14,11 +14,11 @@ describe SurveyResponsesDataLoader do let(:s_phys_q1) { SurveyItem.find_by_survey_item_id 's-phys-q1' } let(:s_phys_q2) { SurveyItem.find_by_survey_item_id 's-phys-q2' } - let(:female) {Gender.find_by_qualtrics_code 1} - let(:male) {Gender.find_by_qualtrics_code 2} - let(:another_gender) {Gender.find_by_qualtrics_code 3} - let(:non_binary) {Gender.find_by_qualtrics_code 4} - let(:unknown_gender) {Gender.find_by_qualtrics_code 99 } + let(:female) { Gender.find_by_qualtrics_code 1 } + let(:male) { Gender.find_by_qualtrics_code 2 } + let(:another_gender) { Gender.find_by_qualtrics_code 3 } + let(:non_binary) { Gender.find_by_qualtrics_code 4 } + let(:unknown_gender) { Gender.find_by_qualtrics_code 99 } before :each do Rails.application.load_seed @@ -75,6 +75,25 @@ describe SurveyResponsesDataLoader do end end end + + context 'when using Lowell rules to skip rows in the csv file' do + before :each do + SurveyResponsesDataLoader.load_data filepath: path_to_student_responses, + rules: [Rule::SkipNonLowellSchools] + end + + it 'rejects any non-lowell school' do + expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').count).to eq 0 + expect(SurveyItemResponse.count).to eq 128 + end + + it 'loads the correct number of responses for lowell schools' do + expect(SurveyItemResponse.where(response_id: 'student_survey_response_2').count).to eq 0 + expect(SurveyItemResponse.where(response_id: 'student_survey_response_3').count).to eq 25 + expect(SurveyItemResponse.where(response_id: 'student_survey_response_4').count).to eq 22 + expect(SurveyItemResponse.where(response_id: 'student_survey_response_5').count).to eq 27 + end + end end end