From 063810a9d7c03228759059e8e351987edd8304fb Mon Sep 17 00:00:00 2001 From: Nelson Jovel Date: Wed, 20 Dec 2023 13:39:03 -0800 Subject: [PATCH] chore: make sure to load races in data loader instead of student loader --- app/services/survey_responses_data_loader.rb | 26 +++++--- .../survey_responses_data_loader_spec.rb | 66 +++++++------------ 2 files changed, 40 insertions(+), 52 deletions(-) diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index 035ac73b..d8d3550c 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -12,7 +12,7 @@ class SurveyResponsesDataLoader process_row(row: SurveyItemValues.new(row:, headers: headers_array, survey_items: all_survey_items, schools:), rules:) end - SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500 + SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all end end end @@ -34,14 +34,14 @@ class SurveyResponsesDataLoader end row_count += 1 - next unless row_count == 1000 + next unless row_count == 500 - SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000 + SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all survey_item_responses = [] row_count = 0 end - SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000 + SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all end private @@ -95,7 +95,7 @@ class SurveyResponsesDataLoader next end response = row.survey_item_response(survey_item:) - create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:) + create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, student:) end.compact end @@ -105,13 +105,20 @@ class SurveyResponsesDataLoader income = incomes[row.income.parameterize] ell = ells[row.ell] sped = speds[row.sped] + if survey_item_response.present? - survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, - sped:) - [] + survey_item_response.likert_score = likert_score + survey_item_response.grade = grade + survey_item_response.gender = gender + survey_item_response.recorded_date = row.recorded_date + survey_item_response.income = income + survey_item_response.ell = ell + survey_item_response.sped = sped + survey_item_response.student = student + survey_item_response else SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:, - likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:) + likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:, student:) end end @@ -133,4 +140,3 @@ module StringMonkeyPatches end String.include StringMonkeyPatches - diff --git a/spec/services/survey_responses_data_loader_spec.rb b/spec/services/survey_responses_data_loader_spec.rb index 9ca25874..a2fc203c 100644 --- a/spec/services/survey_responses_data_loader_spec.rb +++ b/spec/services/survey_responses_data_loader_spec.rb @@ -51,13 +51,24 @@ describe SurveyResponsesDataLoader do let(:another_gender) { create(:gender, qualtrics_code: 3) } let(:non_binary) { create(:gender, qualtrics_code: 4) } let(:unknown_gender) { create(:gender, qualtrics_code: 99) } + let(:low_income) { create(:income, designation: "Economically Disadvantaged – Y") } let(:high_income) { create(:income, designation: "Economically Disadvantaged – N") } let(:unknown_income) { create(:income, designation: "Unknown") } + let(:yes_ell) { create(:ell, designation: "ELL") } let(:not_ell) { create(:ell, designation: "Not ELL") } let(:unknown_ell) { create(:ell, designation: "Unknown") } + let(:american_indian) { create(:race, qualtrics_code: 1) } + let(:asian) { create(:race, qualtrics_code: 2) } + let(:black) { create(:race, qualtrics_code: 3) } + let(:latinx) { create(:race, qualtrics_code: 4) } + let(:white) { create(:race, qualtrics_code: 5) } + let(:middle_eastern) { create(:race, qualtrics_code: 8) } + let(:unknown_race) { create(:race, qualtrics_code: 99) } + let(:multiracial) { create(:race, qualtrics_code: 100) } + let(:setup) do ay_2020_21 ay_2022_23 @@ -87,17 +98,29 @@ describe SurveyResponsesDataLoader do s_emsa_q1 s_emsa_q2 s_emsa_q3 + female male another_gender non_binary unknown_gender + low_income high_income unknown_income + yes_ell not_ell unknown_ell + + american_indian + asian + black + latinx + white + middle_eastern + unknown_race + multiracial end before :each do @@ -159,47 +182,6 @@ describe SurveyResponsesDataLoader do end end end - # figure out why this is failing - describe "when using Lowell rules to skip rows in the csv file" do - before :each do - SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses, - rules: [Rule::SkipNonLowellSchools] - end - - it "rejects any non-lowell school" do - expect(SurveyItemResponse.where(response_id: "student_survey_response_1").count).to eq 0 - expect(SurveyItemResponse.count).to eq 69 - end - - it "loads the correct number of responses for lowell schools" do - expect(SurveyItemResponse.where(response_id: "student_survey_response_2").count).to eq 0 - expect(SurveyItemResponse.where(response_id: "student_survey_response_3").count).to eq 12 - expect(SurveyItemResponse.where(response_id: "student_survey_response_4").count).to eq 15 - expect(SurveyItemResponse.where(response_id: "student_survey_response_5").count).to eq 14 - end - - context "when loading 22-23 butler survey responses" do - before :each do - SurveyResponsesDataLoader.new.load_data filepath: path_to_butler_student_responses, - rules: [Rule::SkipNonLowellSchools] - end - - it "loads all the responses for Butler" do - expect(SurveyItemResponse.where(school: butler_school).count).to eq 56 - end - - it "blank entries for grade get loaded as nils, not zero values" do - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_1").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_2").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_3").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_4").first.grade).to eq 5 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_5").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_6").first.grade).to eq 6 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_7").first.grade).to eq nil - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_8").first.grade).to eq 0 - end - end - end end def assigns_academic_year_to_survey_item_responses @@ -314,7 +296,7 @@ def assigns_gender_to_responses "student_survey_response_4" => non_binary, "student_survey_response_5" => non_binary, "student_survey_response_6" => unknown_gender, - "student_survey_response_7" => unknown_gender } + "student_survey_response_7" => non_binary } results.each do |key, value| expect(SurveyItemResponse.where(response_id: key).first.gender).to eq value