From b63c327d33e664dc1359e76665e5ee44a454363a Mon Sep 17 00:00:00 2001 From: Nelson Jovel Date: Mon, 6 Nov 2023 13:13:37 -0800 Subject: [PATCH] chore: when searching for dese id, split up pattern so that to be more explicit about the order in which to search out the columns that might have the dese ID we're looking for. --- app/services/survey_item_values.rb | 10 ++--- spec/services/cleaner_spec.rb | 2 +- spec/services/survey_item_values_spec.rb | 16 ++++---- .../survey_responses_data_loader_spec.rb | 41 ------------------- 4 files changed, 12 insertions(+), 57 deletions(-) diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index 0139009b..b1551ddb 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -81,13 +81,9 @@ class SurveyItemValues def dese_id @dese_id ||= begin - dese_id = nil - dese_headers = ["DESE ID", "Dese ID", "DeseId", "DeseID", "School", "school"] - school_headers = headers.select { |header| /School-\s\w/.match(header) } - dese_headers << school_headers - dese_headers.flatten.each do |header| - dese_id ||= row[header] - end + dese_id = value_from(pattern: /Dese\s*ID/i) + dese_id ||= value_from(pattern: /^School$/i) + dese_id ||= value_from(pattern: /School-\s*\w/i) dese_id.to_i end diff --git a/spec/services/cleaner_spec.rb b/spec/services/cleaner_spec.rb index 7dbb16ed..909a2fc2 100644 --- a/spec/services/cleaner_spec.rb +++ b/spec/services/cleaner_spec.rb @@ -206,7 +206,7 @@ end def reads_headers_from_raw_csv(processed_data) processed_data in [headers, clean_csv, log_csv, data] - expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", + expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", "Finished", "RecordedDate", "ResponseId", "District", "School", "LASID", "Gender", "Race", "What grade are you in?", "s-emsa-q1", "s-emsa-q2", "s-emsa-q3", "s-tint-q1", "s-tint-q2", "s-tint-q3", "s-tint-q4", "s-tint-q5", "s-acpr-q1", "s-acpr-q2", diff --git a/spec/services/survey_item_values_spec.rb b/spec/services/survey_item_values_spec.rb index fed3c86b..3b760c55 100644 --- a/spec/services/survey_item_values_spec.rb +++ b/spec/services/survey_item_values_spec.rb @@ -100,8 +100,8 @@ RSpec.describe SurveyItemValues, type: :model do context ".school" do it "returns the school that maps to the dese id provided" do attleboro - headers = ["Dese ID"] - row = { "Dese ID" => "1234" } + headers = ["DeseID"] + row = { "DeseID" => "1234" } values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) expect(values.school).to eq attleboro @@ -469,15 +469,15 @@ RSpec.describe SurveyItemValues, type: :model do attleboro_respondents end it "returns true for students" do - headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { "grade" => "9", "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:, + headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate DeseID] + values = SurveyItemValues.new(row: { "grade" => "9", "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:, schools:) expect(values.valid_grade?).to eq true end it "returns true for teachers" do - headers = %w[t-sbel-q5 t-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:, + headers = %w[t-sbel-q5 t-phys-q2 grade RecordedDate DeseID] + values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:, schools:) expect(values.valid_grade?).to eq true end @@ -489,8 +489,8 @@ RSpec.describe SurveyItemValues, type: :model do attleboro_respondents end it "returns false" do - headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { "grade" => "2", "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:, + headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate DeseID] + values = SurveyItemValues.new(row: { "grade" => "2", "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:, schools: School.school_hash) expect(values.valid_grade?).to eq false end diff --git a/spec/services/survey_responses_data_loader_spec.rb b/spec/services/survey_responses_data_loader_spec.rb index 9e2931b6..6f332907 100644 --- a/spec/services/survey_responses_data_loader_spec.rb +++ b/spec/services/survey_responses_data_loader_spec.rb @@ -181,47 +181,6 @@ describe SurveyResponsesDataLoader do end end end - # figure out why this is failing - describe "when using Lowell rules to skip rows in the csv file" do - before :each do - SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses, - rules: [Rule::SkipNonLowellSchools] - end - - it "rejects any non-lowell school" do - expect(SurveyItemResponse.where(response_id: "student_survey_response_1").count).to eq 0 - expect(SurveyItemResponse.count).to eq 69 - end - - it "loads the correct number of responses for lowell schools" do - expect(SurveyItemResponse.where(response_id: "student_survey_response_2").count).to eq 0 - expect(SurveyItemResponse.where(response_id: "student_survey_response_3").count).to eq 12 - expect(SurveyItemResponse.where(response_id: "student_survey_response_4").count).to eq 15 - expect(SurveyItemResponse.where(response_id: "student_survey_response_5").count).to eq 14 - end - - context "when loading 22-23 butler survey responses" do - before :each do - SurveyResponsesDataLoader.new.load_data filepath: path_to_butler_student_responses, - rules: [Rule::SkipNonLowellSchools] - end - - it "loads all the responses for Butler" do - expect(SurveyItemResponse.where(school: butler_school).count).to eq 56 - end - - it "blank entries for grade get loaded as nils, not zero values" do - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_1").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_2").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_3").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_4").first.grade).to eq 5 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_5").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_6").first.grade).to eq 6 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_7").first.grade).to eq nil - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_8").first.grade).to eq 0 - end - end - end end def assigns_academic_year_to_survey_item_responses