diff --git a/spec/fixtures/raw/bak/prepped_2022_23_student_survey_responses.csv b/spec/fixtures/raw/bak/prepped_2022_23_student_survey_responses.csv index 1ac3e039..870272ba 100644 --- a/spec/fixtures/raw/bak/prepped_2022_23_student_survey_responses.csv +++ b/spec/fixtures/raw/bak/prepped_2022_23_student_survey_responses.csv @@ -1,4 +1,4 @@ -StartDate,EndDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,RecipientLastName,RecipientFirstName,RecipientEmail,ExternalReference,LocationLatitude,LocationLongitude,DistributionChannel,UserLanguage,District,School- Lee,School- Maynard,Q145_1,Q145_2,Q145_3,LASID,Grade,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,s-tint-q3,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,s-cure-q1,s-cure-q2,s-cure-q3,s-cure-q4,s-sten-q1,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,s-grmi-q2,s-grmi-q3,s-grmi-q4,s-appa-q1,s-appa-q2,s-appa-q3,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-sust-q1,s-sust-q2,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,s-expa-q1,s-poaf-q1,s-poaf-q2,s-poaf-q3,s-poaf-q4,s-tint-q1-1,s-tint-q2-1,s-tint-q3-1,s-tint-q4-1,s-tint-q5-1,s-acpr-q1-1,s-acpr-q2-1,s-acpr-q3-1,s-acpr-q4-1,s-peff-q1-1,s-peff-q2-1,s-peff-q3-1,s-peff-q4-1,s-peff-q5-1,s-peff-q6-1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q168_1,Q168_2,Q168_3,Q168_4,Q168_5,Q168_6,Q168_7,Q169,Q3,Q4,Gender,Gender_3_TEXT,Race,Race_7_TEXT,Q_Language,MathTeacher,ScienceTeacher,SocialTeacher,EnglishTeacher +StartDate,EndDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,RecipientLastName,RecipientFirstName,RecipientEmail,ExternalReference,LocationLatitude,LocationLongitude,DistributionChannel,UserLanguage,District,School- Lee,School- Maynard,Q145_1,Q145_2,Q145_3,LASID,Grade,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,s-tint-q3,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,s-cure-q1,s-cure-q2,s-cure-q3,s-cure-q4,s-sten-q1,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,s-grmi-q2,s-grmi-q3,s-grmi-q4,s-appa-q1,s-appa-q2,s-appa-q3,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-sust-q1,s-sust-q2,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,s-expa-q1,s-poaf-q1,s-poaf-q2,s-poaf-q3,s-poaf-q4,s-tint-q1-1,s-tint-q2-1,s-tint-q3-1,s-tint-q4-1,s-tint-q5-1,s-acpr-q1-1,s-acpr-q2-1,s-acpr-q3-1,s-acpr-q4-1,s-peff-q1-1,s-peff-q2-1,s-peff-q3-1,s-peff-q4-1,s-peff-q5-1,s-peff-q6-1,Q2,QA3,Qa4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q168_1,Q168_2,Q168_3,Q168_4,Q168_5,Q168_6,Q168_7,Q169,Q3,Q4,Gender,Gender_3_TEXT,Race,Race_7_TEXT,Q_Language,MathTeacher,ScienceTeacher,SocialTeacher,EnglishTeacher 2023-03-15 10:07:47,2023-03-15 10:27:46,0,71.174.81.214,27,1198,0,2023-03-16 10:27:51,R_oZf5P5qFxiJFmFP,,,,,,,anonymous,EN,2,,1740505,,,,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,Social Studies teacher, 2023-03-16 12:27:21,2023-03-16 12:29:46,0,71.174.81.214,100,145,1,2023-03-16 12:29:47,R_eVzCTxBTDvPFFiV,,,,,42.4337,-71.4568,anonymous,EN,2,,1740505,1,1,109,,5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,, 2023-03-17 7:57:04,2023-03-17 8:00:28,0,71.174.81.214,100,203,1,2023-03-17 8:00:30,R_2ccT2ouzOsZVwLn,,,,,42.4337,-71.4568,anonymous,EN,2,,1740505,10,22,105,jb2349,12,,,,,,,,,,,,,,,,,,,,3,4,3,3,,,,,,,,,1,1,1,,,,,,,3,3,3,4,2,5,5,4,5,3,2,2,3,1,3,1,3,3,2,1,1,1,1,2,2,2,1,,,,,,4,4,4,4,4,4,4,4,2,4,,,,,,,,,,,,,,,,,,,,,1,,"4,5",,EN,,,Social Studies teacher, diff --git a/spec/fixtures/raw/sample_file_with_duplicate_headers.csv b/spec/fixtures/raw/sample_file_with_duplicate_headers.csv new file mode 100644 index 00000000..305af074 --- /dev/null +++ b/spec/fixtures/raw/sample_file_with_duplicate_headers.csv @@ -0,0 +1,2 @@ +StartDate,StartDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,District,School,LASID,Gender,Race,What grade are you in?,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,s-tint-q3,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,s-cure-q1,s-cure-q2,s-cure-q3,s-cure-q4,s-sten-q1,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,s-grmi-q2,s-grmi-q3,s-grmi-q4,s-appa-q1,s-appa-q2,s-appa-q3,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-sust-q1,s-sust-q2,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,s-expa-q1,s-poaf-q1,s-poaf-q2,s-poaf-q3,s-poaf-q4,s-tint-q1-1,s-tint-q2-1,s-tint-q3-1,s-tint-q4-1,s-tint-q5-1,s-acpr-q1-1,s-acpr-q2-1,s-acpr-q3-1,s-acpr-q4-1,s-peff-q1-1,s-peff-q2-1,s-peff-q3-1,s-peff-q4-1,s-peff-q5-1,s-peff-q6-1 +2023-03-17 7:57:47,2023-03-17 8:09:15,0,71.174.81.214,100,1000,1,2023-03-17T8:9:15,1000,2,1740505,1,2,4,9,3,5,5,,,,,,,,,,,,,,,,,,,,,,,,,4,4,3,5,,,,,,,,,4,4,2,3,2,5,5,5,5,4,2,2,4,3,2,3,3,5,4,4,3,5,2,3,3,4,4,4,1,2,5,5,,,,,4,4,4,3,4,5 diff --git a/spec/services/cleaner_spec.rb b/spec/services/cleaner_spec.rb index 909a2fc2..d66286e2 100644 --- a/spec/services/cleaner_spec.rb +++ b/spec/services/cleaner_spec.rb @@ -31,6 +31,10 @@ RSpec.describe Cleaner do File.open(Rails.root.join("spec", "fixtures", "raw", "sample_maynard_raw_student_survey.csv")) end + let(:path_to_file_with_duplicate_headers) do + File.open(Rails.root.join("spec", "fixtures", "raw", "sample_file_with_duplicate_headers.csv")) + end + let(:common_headers) do ["Recorded Date", "Dese ID", "ResponseID"] end @@ -89,6 +93,13 @@ RSpec.describe Cleaner do respondents end + context "When duplicate headers exist" do + it "outputs a message to stdout" do + output = capture_stdout { Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean } + expect(output).to match "\n>>>>>>>>>>>>>>>>>> Duplicate header found. This will misalign column headings. Please delete or rename the duplicate column: StartDate \n>>>>>>>>>>>>>> \n" + end + end + context "Creating a new Cleaner" do it "creates a directory for the clean data" do Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 26a6aa16..0f06f0c5 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -122,3 +122,14 @@ RSpec.configure do |config| # # as the one that triggered the failure. # Kernel.srand config.seed end + +def capture_stdout + original_stdout = $stdout + $stdout = fake = StringIO.new + begin + yield + ensure + $stdout = original_stdout + end + fake.string +end