From 1b82ff941391279e0f170868e5c515880eda4119 Mon Sep 17 00:00:00 2001
From: Nelson Jovel <memoryman51@hotmail.com>
Date: Mon, 6 Nov 2023 14:51:56 -0800
Subject: [PATCH] chore: since it's now possible for there to be multiple
 district and dese id columns separated by a dash and a number, be more
 explicit when we only want to filter out survey item ids that end in a -1

---
 app/services/cleaner.rb            |  6 +++---
 app/services/survey_item_values.rb | 27 +++++++++++++--------------
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/app/services/cleaner.rb b/app/services/cleaner.rb
index 65e7414b..76067e39 100644
--- a/app/services/cleaner.rb
+++ b/app/services/cleaner.rb
@@ -43,7 +43,7 @@ class Cleaner
     log_csv = []
     data = []
 
-    headers = CSV.parse(file.first).first.push("Raw Income").push("Income").push("Raw ELL").push("ELL").push("Raw SpEd").push("SpEd").push("Progress Count")
+    headers = CSV.parse(file.first).first.push("Raw Income").push("Income").push("Raw ELL").push("ELL").push("Raw SpEd").push("SpEd").push("Progress Count").uniq
     filtered_headers = include_all_headers(headers:)
     filtered_headers = remove_unwanted_headers(headers: filtered_headers)
     log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?",
@@ -70,7 +70,7 @@ class Cleaner
 
   def include_all_headers(headers:)
     alternates = headers.filter(&:present?)
-                        .filter { |header| header.end_with? "-1" }
+                        .filter {  |header| header.match? /^[st]-\w*-\w*-1$/i }
     alternates.each do |header|
       main = header.sub(/-1\z/, "")
       headers.push(main) unless headers.include?(main)
@@ -86,7 +86,7 @@ class Cleaner
   def remove_unwanted_headers(headers:)
     headers.to_set.to_a.compact.reject do |item|
       item.start_with? "Q"
-    end.reject { |item| item.end_with? "-1" }
+    end.reject {  |header| header.match? /^[st]-\w*-\w*-1$/i }
   end
 
   def write_csv(data:, output_filepath:, filename:, prefix: "")
diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb
index b1551ddb..7764a756 100644
--- a/app/services/survey_item_values.rb
+++ b/app/services/survey_item_values.rb
@@ -32,7 +32,7 @@ class SurveyItemValues
   # We don't ensure that ids in the form of s-tint-q1 have a matching pair because not all questions have variants
   def include_all_headers(headers:)
     alternates = headers.filter(&:present?)
-                        .filter { |header| header.end_with? "-1" }
+                        .filter { |header| header.match?(/^[st]-\w*-\w*-1$/i) }
     alternates.each do |header|
       main = header.sub(/-1\z/, "")
       headers.push(main) unless headers.include?(main)
@@ -192,11 +192,14 @@ class SurveyItemValues
     output
   end
 
+  def sanitized_headers
+    @sanitized_headers ||= headers.select(&:present?)
+                                  .reject { |key, _value| key.start_with? "Q" }
+                                  .reject { |key, _value| key.match?(/^[st]-\w*-\w*-1$/i) }
+  end
+
   def to_a
-    headers.select(&:present?)
-           .reject { |key, _value| key.start_with? "Q" }
-           .reject { |key, _value| key.end_with? "-1" }
-           .map { |header| row[header] }
+    sanitized_headers.map { |header| row[header] }
   end
 
   def duration
@@ -216,12 +219,11 @@ class SurveyItemValues
   end
 
   def survey_type
-    survey_item_ids = headers
-                      .filter(&:present?)
-                      .reject { |header| header.end_with?("-1") }
-                      .filter { |header| header.start_with?("t-", "s-") }
+    @survey_type ||= SurveyItem.survey_type(survey_item_ids:)
+  end
 
-    SurveyItem.survey_type(survey_item_ids:)
+  def survey_item_ids
+    @survey_item_ids ||= sanitized_headers.filter { |header| header.start_with?("t-", "s-") }
   end
 
   def valid_duration?
@@ -236,10 +238,7 @@ class SurveyItemValues
   end
 
   def progress
-    headers.filter(&:present?)
-           .reject { |header| header.end_with?("-1") }
-           .filter { |header| header.start_with?("t-", "s-") }
-           .reject { |header| row[header].nil? }.count
+    survey_item_ids.reject { |header| row[header].nil? }.count
   end
 
   def valid_progress?