From a3f9e46414946dce0e3eee656f5f7013fd38c48e Mon Sep 17 00:00:00 2001 From: Nelson Jovel Date: Mon, 6 Nov 2023 13:13:37 -0800 Subject: [PATCH] chore: when searching for dese id, split up pattern so that to be more explicit about the order in which to search out the columns that might have the dese ID we're looking for. --- app/services/survey_item_values.rb | 12 ++---- spec/services/cleaner_spec.rb | 2 +- spec/services/survey_item_values_spec.rb | 15 +++---- .../survey_responses_data_loader_spec.rb | 41 ------------------- 4 files changed, 13 insertions(+), 57 deletions(-) diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index f7eb29b9..b1551ddb 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -81,13 +81,9 @@ class SurveyItemValues def dese_id @dese_id ||= begin - dese_id = nil - dese_headers = ["DESE ID", "Dese ID", "DeseId", "DeseID", "School", "school"] - school_headers = headers.select { |header| /School-\s\w/.match(header) } - dese_headers << school_headers - dese_headers.flatten.each do |header| - dese_id ||= row[header] - end + dese_id = value_from(pattern: /Dese\s*ID/i) + dese_id ||= value_from(pattern: /^School$/i) + dese_id ||= value_from(pattern: /School-\s*\w/i) dese_id.to_i end @@ -260,7 +256,7 @@ class SurveyItemValues def valid_grade? return true if grade.nil? - return true if survey_type == :teacher + return true if respondent_type == :teacher respondents = Respondent.where(school:, academic_year:).first if respondents.present? && respondents.counts_by_grade[grade].present? diff --git a/spec/services/cleaner_spec.rb b/spec/services/cleaner_spec.rb index 6c847dfa..deecf3e1 100644 --- a/spec/services/cleaner_spec.rb +++ b/spec/services/cleaner_spec.rb @@ -206,7 +206,7 @@ end def reads_headers_from_raw_csv(processed_data) processed_data in [headers, clean_csv, log_csv, data] - expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", + expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", "Finished", "RecordedDate", "ResponseId", "District", "School", "LASID", "Gender", "Race", "What grade are you in?", "s-emsa-q1", "s-emsa-q2", "s-emsa-q3", "s-tint-q1", "s-tint-q2", "s-tint-q3", "s-tint-q4", "s-tint-q5", "s-acpr-q1", "s-acpr-q2", diff --git a/spec/services/survey_item_values_spec.rb b/spec/services/survey_item_values_spec.rb index 7e8cb511..3b760c55 100644 --- a/spec/services/survey_item_values_spec.rb +++ b/spec/services/survey_item_values_spec.rb @@ -100,7 +100,8 @@ RSpec.describe SurveyItemValues, type: :model do context ".school" do it "returns the school that maps to the dese id provided" do attleboro - row = { "Dese ID" => "1234" } + headers = ["DeseID"] + row = { "DeseID" => "1234" } values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) expect(values.school).to eq attleboro @@ -468,15 +469,15 @@ RSpec.describe SurveyItemValues, type: :model do attleboro_respondents end it "returns true for students" do - headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { "grade" => "9", "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:, + headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate DeseID] + values = SurveyItemValues.new(row: { "grade" => "9", "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:, schools:) expect(values.valid_grade?).to eq true end it "returns true for teachers" do - headers = %w[t-sbel-q5 t-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:, + headers = %w[t-sbel-q5 t-phys-q2 grade RecordedDate DeseID] + values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:, schools:) expect(values.valid_grade?).to eq true end @@ -488,8 +489,8 @@ RSpec.describe SurveyItemValues, type: :model do attleboro_respondents end it "returns false" do - headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { "grade" => "2", "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:, + headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate DeseID] + values = SurveyItemValues.new(row: { "grade" => "2", "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:, schools: School.school_hash) expect(values.valid_grade?).to eq false end diff --git a/spec/services/survey_responses_data_loader_spec.rb b/spec/services/survey_responses_data_loader_spec.rb index 9e2931b6..6f332907 100644 --- a/spec/services/survey_responses_data_loader_spec.rb +++ b/spec/services/survey_responses_data_loader_spec.rb @@ -181,47 +181,6 @@ describe SurveyResponsesDataLoader do end end end - # figure out why this is failing - describe "when using Lowell rules to skip rows in the csv file" do - before :each do - SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses, - rules: [Rule::SkipNonLowellSchools] - end - - it "rejects any non-lowell school" do - expect(SurveyItemResponse.where(response_id: "student_survey_response_1").count).to eq 0 - expect(SurveyItemResponse.count).to eq 69 - end - - it "loads the correct number of responses for lowell schools" do - expect(SurveyItemResponse.where(response_id: "student_survey_response_2").count).to eq 0 - expect(SurveyItemResponse.where(response_id: "student_survey_response_3").count).to eq 12 - expect(SurveyItemResponse.where(response_id: "student_survey_response_4").count).to eq 15 - expect(SurveyItemResponse.where(response_id: "student_survey_response_5").count).to eq 14 - end - - context "when loading 22-23 butler survey responses" do - before :each do - SurveyResponsesDataLoader.new.load_data filepath: path_to_butler_student_responses, - rules: [Rule::SkipNonLowellSchools] - end - - it "loads all the responses for Butler" do - expect(SurveyItemResponse.where(school: butler_school).count).to eq 56 - end - - it "blank entries for grade get loaded as nils, not zero values" do - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_1").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_2").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_3").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_4").first.grade).to eq 5 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_5").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_6").first.grade).to eq 6 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_7").first.grade).to eq nil - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_8").first.grade).to eq 0 - end - end - end end def assigns_academic_year_to_survey_item_responses