diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index 0139009b..b1551ddb 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -81,13 +81,9 @@ class SurveyItemValues def dese_id @dese_id ||= begin - dese_id = nil - dese_headers = ["DESE ID", "Dese ID", "DeseId", "DeseID", "School", "school"] - school_headers = headers.select { |header| /School-\s\w/.match(header) } - dese_headers << school_headers - dese_headers.flatten.each do |header| - dese_id ||= row[header] - end + dese_id = value_from(pattern: /Dese\s*ID/i) + dese_id ||= value_from(pattern: /^School$/i) + dese_id ||= value_from(pattern: /School-\s*\w/i) dese_id.to_i end diff --git a/spec/services/cleaner_spec.rb b/spec/services/cleaner_spec.rb index 7dbb16ed..909a2fc2 100644 --- a/spec/services/cleaner_spec.rb +++ b/spec/services/cleaner_spec.rb @@ -206,7 +206,7 @@ end def reads_headers_from_raw_csv(processed_data) processed_data in [headers, clean_csv, log_csv, data] - expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", + expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", "Finished", "RecordedDate", "ResponseId", "District", "School", "LASID", "Gender", "Race", "What grade are you in?", "s-emsa-q1", "s-emsa-q2", "s-emsa-q3", "s-tint-q1", "s-tint-q2", "s-tint-q3", "s-tint-q4", "s-tint-q5", "s-acpr-q1", "s-acpr-q2", diff --git a/spec/services/survey_item_values_spec.rb b/spec/services/survey_item_values_spec.rb index fed3c86b..3b760c55 100644 --- a/spec/services/survey_item_values_spec.rb +++ b/spec/services/survey_item_values_spec.rb @@ -100,8 +100,8 @@ RSpec.describe SurveyItemValues, type: :model do context ".school" do it "returns the school that maps to the dese id provided" do attleboro - headers = ["Dese ID"] - row = { "Dese ID" => "1234" } + headers = ["DeseID"] + row = { "DeseID" => "1234" } values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) expect(values.school).to eq attleboro @@ -469,15 +469,15 @@ RSpec.describe SurveyItemValues, type: :model do attleboro_respondents end it "returns true for students" do - headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { "grade" => "9", "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:, + headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate DeseID] + values = SurveyItemValues.new(row: { "grade" => "9", "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:, schools:) expect(values.valid_grade?).to eq true end it "returns true for teachers" do - headers = %w[t-sbel-q5 t-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:, + headers = %w[t-sbel-q5 t-phys-q2 grade RecordedDate DeseID] + values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:, schools:) expect(values.valid_grade?).to eq true end @@ -489,8 +489,8 @@ RSpec.describe SurveyItemValues, type: :model do attleboro_respondents end it "returns false" do - headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { "grade" => "2", "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:, + headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate DeseID] + values = SurveyItemValues.new(row: { "grade" => "2", "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:, schools: School.school_hash) expect(values.valid_grade?).to eq false end diff --git a/spec/services/survey_responses_data_loader_spec.rb b/spec/services/survey_responses_data_loader_spec.rb index 9e2931b6..6f332907 100644 --- a/spec/services/survey_responses_data_loader_spec.rb +++ b/spec/services/survey_responses_data_loader_spec.rb @@ -181,47 +181,6 @@ describe SurveyResponsesDataLoader do end end end - # figure out why this is failing - describe "when using Lowell rules to skip rows in the csv file" do - before :each do - SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses, - rules: [Rule::SkipNonLowellSchools] - end - - it "rejects any non-lowell school" do - expect(SurveyItemResponse.where(response_id: "student_survey_response_1").count).to eq 0 - expect(SurveyItemResponse.count).to eq 69 - end - - it "loads the correct number of responses for lowell schools" do - expect(SurveyItemResponse.where(response_id: "student_survey_response_2").count).to eq 0 - expect(SurveyItemResponse.where(response_id: "student_survey_response_3").count).to eq 12 - expect(SurveyItemResponse.where(response_id: "student_survey_response_4").count).to eq 15 - expect(SurveyItemResponse.where(response_id: "student_survey_response_5").count).to eq 14 - end - - context "when loading 22-23 butler survey responses" do - before :each do - SurveyResponsesDataLoader.new.load_data filepath: path_to_butler_student_responses, - rules: [Rule::SkipNonLowellSchools] - end - - it "loads all the responses for Butler" do - expect(SurveyItemResponse.where(school: butler_school).count).to eq 56 - end - - it "blank entries for grade get loaded as nils, not zero values" do - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_1").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_2").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_3").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_4").first.grade).to eq 5 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_5").first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_6").first.grade).to eq 6 - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_7").first.grade).to eq nil - expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_8").first.grade).to eq 0 - end - end - end end def assigns_academic_year_to_survey_item_responses