chore: when searching for dese id, split up pattern so that to be more explicit about the order in which to search out the columns that might have the dese ID we're looking for.

app/services/survey_item_values.rb

@@ -81,13 +81,9 @@ class SurveyItemValues
 
   def dese_id
     @dese_id ||= begin
-      dese_id = nil
-      dese_headers = ["DESE ID", "Dese ID", "DeseId", "DeseID", "School", "school"]
-      school_headers = headers.select { |header| /School-\s\w/.match(header) }
-      dese_headers << school_headers
-      dese_headers.flatten.each do |header|
-        dese_id ||= row[header]
-      end
+      dese_id = value_from(pattern: /Dese\s*ID/i)
+      dese_id ||= value_from(pattern: /^School$/i)
+      dese_id ||= value_from(pattern: /School-\s*\w/i)
 
       dese_id.to_i
     end
@@ -260,7 +256,7 @@ class SurveyItemValues
   def valid_grade?
     return true if grade.nil?
 
-    return true if survey_type == :teacher
+    return true if respondent_type == :teacher
 
     respondents = Respondent.where(school:, academic_year:).first
     if respondents.present? && respondents.counts_by_grade[grade].present?

spec/services/cleaner_spec.rb

@@ -206,7 +206,7 @@ end
 
 def reads_headers_from_raw_csv(processed_data)
   processed_data in [headers, clean_csv, log_csv, data]
-  expect(headers.to_set.sort).to eq ["StartDate", "EndDate",  "Status", "IPAddress", "Progress", "Duration (in seconds)",
+  expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)",
                                      "Finished", "RecordedDate", "ResponseId", "District", "School",
                                      "LASID", "Gender", "Race", "What grade are you in?", "s-emsa-q1", "s-emsa-q2", "s-emsa-q3", "s-tint-q1",
                                      "s-tint-q2", "s-tint-q3", "s-tint-q4", "s-tint-q5", "s-acpr-q1", "s-acpr-q2",

spec/services/survey_item_values_spec.rb

@@ -100,7 +100,8 @@ RSpec.describe SurveyItemValues, type: :model do
   context ".school" do
     it "returns the school that maps to the dese id provided" do
       attleboro
-      row = { "Dese ID" => "1234" }
+      headers = ["DeseID"]
+      row = { "DeseID" => "1234" }
       values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
       expect(values.school).to eq attleboro
 
@@ -468,15 +469,15 @@ RSpec.describe SurveyItemValues, type: :model do
         attleboro_respondents
       end
       it "returns true for students" do
-        headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate]
-        values = SurveyItemValues.new(row: { "grade" => "9", "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:,
+        headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate DeseID]
+        values = SurveyItemValues.new(row: { "grade" => "9", "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:,
                                       schools:)
 
         expect(values.valid_grade?).to eq true
       end
       it "returns true for teachers" do
-        headers = %w[t-sbel-q5 t-phys-q2 grade RecordedDate]
-        values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:,
+        headers = %w[t-sbel-q5 t-phys-q2 grade RecordedDate DeseID]
+        values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:,
                                       schools:)
         expect(values.valid_grade?).to eq true
       end
@@ -488,8 +489,8 @@ RSpec.describe SurveyItemValues, type: :model do
         attleboro_respondents
       end
       it "returns false" do
-        headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate]
-        values = SurveyItemValues.new(row: { "grade" => "2", "RecordedDate" => recorded_date, "Dese ID" => "1234" }, headers:, genders:, survey_items:,
+        headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate DeseID]
+        values = SurveyItemValues.new(row: { "grade" => "2", "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, genders:, survey_items:,
                                       schools: School.school_hash)
         expect(values.valid_grade?).to eq false
       end

spec/services/survey_responses_data_loader_spec.rb

@@ -181,47 +181,6 @@ describe SurveyResponsesDataLoader do
       end
     end
   end
-  # figure out why this is failing
-  describe "when using Lowell rules to skip rows in the csv file" do
-    before :each do
-      SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses,
-                                              rules: [Rule::SkipNonLowellSchools]
-    end
-
-    it "rejects any non-lowell school" do
-      expect(SurveyItemResponse.where(response_id: "student_survey_response_1").count).to eq 0
-      expect(SurveyItemResponse.count).to eq 69
-    end
-
-    it "loads the correct number of responses for lowell schools" do
-      expect(SurveyItemResponse.where(response_id: "student_survey_response_2").count).to eq 0
-      expect(SurveyItemResponse.where(response_id: "student_survey_response_3").count).to eq 12
-      expect(SurveyItemResponse.where(response_id: "student_survey_response_4").count).to eq 15
-      expect(SurveyItemResponse.where(response_id: "student_survey_response_5").count).to eq 14
-    end
-
-    context "when loading 22-23 butler survey responses" do
-      before :each do
-        SurveyResponsesDataLoader.new.load_data filepath: path_to_butler_student_responses,
-                                                rules: [Rule::SkipNonLowellSchools]
-      end
-
-      it "loads all the responses for Butler" do
-        expect(SurveyItemResponse.where(school: butler_school).count).to eq 56
-      end
-
-      it "blank entries for grade get loaded as nils, not zero values" do
-        expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_1").first.grade).to eq 7
-        expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_2").first.grade).to eq 7
-        expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_3").first.grade).to eq 7
-        expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_4").first.grade).to eq 5
-        expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_5").first.grade).to eq 7
-        expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_6").first.grade).to eq 6
-        expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_7").first.grade).to eq nil
-        expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_8").first.grade).to eq 0
-      end
-    end
-  end
 end
 
 def assigns_academic_year_to_survey_item_responses