chore: since it's now possible for there to be multiple district and dese id columns separated by a dash and a number, be more explicit when we only want to filter out survey item ids that end in a -1

app/services/cleaner.rb

@@ -43,7 +43,7 @@ class Cleaner
     log_csv = []
     data = []
 
-    headers = CSV.parse(file.first).first.push("Raw Income").push("Income").push("Raw ELL").push("ELL").push("Raw SpEd").push("SpEd").push("Progress Count")
+    headers = CSV.parse(file.first).first.push("Raw Income").push("Income").push("Raw ELL").push("ELL").push("Raw SpEd").push("SpEd").push("Progress Count").uniq
     filtered_headers = include_all_headers(headers:)
     filtered_headers = remove_unwanted_headers(headers: filtered_headers)
     log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?",
@@ -70,7 +70,7 @@ class Cleaner
 
   def include_all_headers(headers:)
     alternates = headers.filter(&:present?)
-                        .filter { |header| header.end_with? "-1" }
+                        .filter {  |header| header.match? /^[st]-\w*-\w*-1$/i }
     alternates.each do |header|
       main = header.sub(/-1\z/, "")
       headers.push(main) unless headers.include?(main)
@@ -86,7 +86,7 @@ class Cleaner
   def remove_unwanted_headers(headers:)
     headers.to_set.to_a.compact.reject do |item|
       item.start_with? "Q"
-    end.reject { |item| item.end_with? "-1" }
+    end.reject {  |header| header.match? /^[st]-\w*-\w*-1$/i }
   end
 
   def write_csv(data:, output_filepath:, filename:, prefix: "")

app/services/survey_item_values.rb

@@ -32,7 +32,7 @@ class SurveyItemValues
   # We don't ensure that ids in the form of s-tint-q1 have a matching pair because not all questions have variants
   def include_all_headers(headers:)
     alternates = headers.filter(&:present?)
-                        .filter { |header| header.end_with? "-1" }
+                        .filter { |header| header.match?(/^[st]-\w*-\w*-1$/i) }
     alternates.each do |header|
       main = header.sub(/-1\z/, "")
       headers.push(main) unless headers.include?(main)
@@ -192,11 +192,14 @@ class SurveyItemValues
     output
   end
 
-  def to_a
+  def sanitized_headers
-    headers.select(&:present?)
+    @sanitized_headers ||= headers.select(&:present?)
                                   .reject { |key, _value| key.start_with? "Q" }
-           .reject { |key, _value| key.end_with? "-1" }
+                                  .reject { |key, _value| key.match?(/^[st]-\w*-\w*-1$/i) }
-           .map { |header| row[header] }
+  end
+
+  def to_a
+    sanitized_headers.map { |header| row[header] }
   end
 
   def duration
@@ -216,12 +219,11 @@ class SurveyItemValues
   end
 
   def survey_type
-    survey_item_ids = headers
+    @survey_type ||= SurveyItem.survey_type(survey_item_ids:)
-                      .filter(&:present?)
+  end
-                      .reject { |header| header.end_with?("-1") }
-                      .filter { |header| header.start_with?("t-", "s-") }
 
-    SurveyItem.survey_type(survey_item_ids:)
+  def survey_item_ids
+    @survey_item_ids ||= sanitized_headers.filter { |header| header.start_with?("t-", "s-") }
   end
 
   def valid_duration?
@@ -236,10 +238,7 @@ class SurveyItemValues
   end
 
   def progress
-    headers.filter(&:present?)
+    survey_item_ids.reject { |header| row[header].nil? }.count
-           .reject { |header| header.end_with?("-1") }
-           .filter { |header| header.start_with?("t-", "s-") }
-           .reject { |header| row[header].nil? }.count
   end
 
   def valid_progress?