chore: add test for checking duplicate headers during cleaning process

spec/fixtures/raw/bak/prepped_2022_23_student_survey_responses.csv

@@ -1,4 +1,4 @@
-StartDate,EndDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,RecipientLastName,RecipientFirstName,RecipientEmail,ExternalReference,LocationLatitude,LocationLongitude,DistributionChannel,UserLanguage,District,School- Lee,School- Maynard,Q145_1,Q145_2,Q145_3,LASID,Grade,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,s-tint-q3,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,s-cure-q1,s-cure-q2,s-cure-q3,s-cure-q4,s-sten-q1,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,s-grmi-q2,s-grmi-q3,s-grmi-q4,s-appa-q1,s-appa-q2,s-appa-q3,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-sust-q1,s-sust-q2,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,s-expa-q1,s-poaf-q1,s-poaf-q2,s-poaf-q3,s-poaf-q4,s-tint-q1-1,s-tint-q2-1,s-tint-q3-1,s-tint-q4-1,s-tint-q5-1,s-acpr-q1-1,s-acpr-q2-1,s-acpr-q3-1,s-acpr-q4-1,s-peff-q1-1,s-peff-q2-1,s-peff-q3-1,s-peff-q4-1,s-peff-q5-1,s-peff-q6-1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q168_1,Q168_2,Q168_3,Q168_4,Q168_5,Q168_6,Q168_7,Q169,Q3,Q4,Gender,Gender_3_TEXT,Race,Race_7_TEXT,Q_Language,MathTeacher,ScienceTeacher,SocialTeacher,EnglishTeacher
+StartDate,EndDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,RecipientLastName,RecipientFirstName,RecipientEmail,ExternalReference,LocationLatitude,LocationLongitude,DistributionChannel,UserLanguage,District,School- Lee,School- Maynard,Q145_1,Q145_2,Q145_3,LASID,Grade,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,s-tint-q3,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,s-cure-q1,s-cure-q2,s-cure-q3,s-cure-q4,s-sten-q1,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,s-grmi-q2,s-grmi-q3,s-grmi-q4,s-appa-q1,s-appa-q2,s-appa-q3,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-sust-q1,s-sust-q2,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,s-expa-q1,s-poaf-q1,s-poaf-q2,s-poaf-q3,s-poaf-q4,s-tint-q1-1,s-tint-q2-1,s-tint-q3-1,s-tint-q4-1,s-tint-q5-1,s-acpr-q1-1,s-acpr-q2-1,s-acpr-q3-1,s-acpr-q4-1,s-peff-q1-1,s-peff-q2-1,s-peff-q3-1,s-peff-q4-1,s-peff-q5-1,s-peff-q6-1,Q2,QA3,Qa4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q168_1,Q168_2,Q168_3,Q168_4,Q168_5,Q168_6,Q168_7,Q169,Q3,Q4,Gender,Gender_3_TEXT,Race,Race_7_TEXT,Q_Language,MathTeacher,ScienceTeacher,SocialTeacher,EnglishTeacher
 2023-03-15 10:07:47,2023-03-15 10:27:46,0,71.174.81.214,27,1198,0,2023-03-16 10:27:51,R_oZf5P5qFxiJFmFP,,,,,,,anonymous,EN,2,,1740505,,,,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,Social Studies teacher,
 2023-03-16 12:27:21,2023-03-16 12:29:46,0,71.174.81.214,100,145,1,2023-03-16 12:29:47,R_eVzCTxBTDvPFFiV,,,,,42.4337,-71.4568,anonymous,EN,2,,1740505,1,1,109,,5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,
 2023-03-17 7:57:04,2023-03-17 8:00:28,0,71.174.81.214,100,203,1,2023-03-17 8:00:30,R_2ccT2ouzOsZVwLn,,,,,42.4337,-71.4568,anonymous,EN,2,,1740505,10,22,105,jb2349,12,,,,,,,,,,,,,,,,,,,,3,4,3,3,,,,,,,,,1,1,1,,,,,,,3,3,3,4,2,5,5,4,5,3,2,2,3,1,3,1,3,3,2,1,1,1,1,2,2,2,1,,,,,,4,4,4,4,4,4,4,4,2,4,,,,,,,,,,,,,,,,,,,,,1,,"4,5",,EN,,,Social Studies teacher,

spec/fixtures/raw/sample_file_with_duplicate_headers.csv

@@ -0,0 +1,2 @@
+StartDate,StartDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,District,School,LASID,Gender,Race,What grade are you in?,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,s-tint-q3,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,s-cure-q1,s-cure-q2,s-cure-q3,s-cure-q4,s-sten-q1,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,s-grmi-q2,s-grmi-q3,s-grmi-q4,s-appa-q1,s-appa-q2,s-appa-q3,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-sust-q1,s-sust-q2,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,s-expa-q1,s-poaf-q1,s-poaf-q2,s-poaf-q3,s-poaf-q4,s-tint-q1-1,s-tint-q2-1,s-tint-q3-1,s-tint-q4-1,s-tint-q5-1,s-acpr-q1-1,s-acpr-q2-1,s-acpr-q3-1,s-acpr-q4-1,s-peff-q1-1,s-peff-q2-1,s-peff-q3-1,s-peff-q4-1,s-peff-q5-1,s-peff-q6-1
+2023-03-17 7:57:47,2023-03-17 8:09:15,0,71.174.81.214,100,1000,1,2023-03-17T8:9:15,1000,2,1740505,1,2,4,9,3,5,5,,,,,,,,,,,,,,,,,,,,,,,,,4,4,3,5,,,,,,,,,4,4,2,3,2,5,5,5,5,4,2,2,4,3,2,3,3,5,4,4,3,5,2,3,3,4,4,4,1,2,5,5,,,,,4,4,4,3,4,5

spec/services/cleaner_spec.rb

@@ -31,6 +31,10 @@ RSpec.describe Cleaner do
     File.open(Rails.root.join("spec", "fixtures", "raw", "sample_maynard_raw_student_survey.csv"))
   end
 
+  let(:path_to_file_with_duplicate_headers) do
+    File.open(Rails.root.join("spec", "fixtures", "raw", "sample_file_with_duplicate_headers.csv"))
+  end
+
   let(:common_headers) do
     ["Recorded Date", "Dese ID", "ResponseID"]
   end
@@ -89,6 +93,13 @@ RSpec.describe Cleaner do
     respondents
   end
 
+  context "When duplicate headers exist" do
+    it "outputs a message to stdout" do
+      output = capture_stdout { Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean }
+      expect(output).to match "\n>>>>>>>>>>>>>>>>>>    Duplicate header found.  This will misalign column headings.  Please delete or rename the duplicate column: StartDate \n>>>>>>>>>>>>>> \n"
+    end
+  end
+
   context "Creating a new Cleaner" do
     it "creates a directory for the clean data" do
       Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean

spec/spec_helper.rb

@@ -122,3 +122,14 @@ RSpec.configure do |config|
   #   # as the one that triggered the failure.
   #   Kernel.srand config.seed
 end
+
+def capture_stdout
+  original_stdout = $stdout
+  $stdout = fake = StringIO.new
+  begin
+    yield
+  ensure
+    $stdout = original_stdout
+  end
+  fake.string
+end