feat: Support two date formats: ISO 8601 and the standard US date format

used in google sheets

app/models/survey_item.rb

@@ -67,7 +67,7 @@ class SurveyItem < ActiveRecord::Base
   end
 
   def self.survey_type(survey_item_ids:)
-    survey_item_ids = survey_item_ids.reject { |id| id.ends_with?('-1') }.to_set
+    survey_item_ids = survey_item_ids.reject { |id| id.ends_with?("-1") }.to_set
     return :short_form if survey_item_ids.subset? short_form_survey_items.map(&:survey_item_id).to_set
     return :early_education if survey_item_ids.subset? early_education_survey_items.map(&:survey_item_id).to_set
     return :teacher if survey_item_ids.subset? teacher_survey_items.map(&:survey_item_id).to_set

app/services/survey_item_values.rb

@@ -48,12 +48,10 @@ class SurveyItemValues
   def recorded_date
     @recorded_date ||= begin
       recorded_date = value_from(pattern: /Recorded\s*Date/i)
-      puts recorded_date
+      date = if recorded_date.include?("T")
-      date = nil
+               Date.parse(recorded_date)
-      begin
+             else
-        date = Date.parse(recorded_date)
+               Date.strptime(recorded_date, "%m/%d/%Y")
-      rescue StandardError => e
-        date = Date.strptime(recorded_date, "%m/%d/%Y")
              end
     end
   end

spec/fixtures/test_2020-21_teacher_survey_responses.csv

@@ -1,6 +1,6 @@
 Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,District,school,,DESE ID,t-prep-q1,t-prep-q2,t-prep-q3,t-pcom-q1,t-pcom-q2,t-pcom-q3,t-pcom-q4,t-pcom-q5,t-ieff-q1,t-ieff-q2,t-ieff-q3,t-ieff-q4,t-qupd-q3,t-qupd-q2,t-qupd-q1,t-qupd-q4,t-coll-q1,t-coll-q2,t-coll-q3,t-prtr-q1,t-prtr-q2,t-prtr-q3,t-inle-q1,t-inle-q2,t-inle-q3,t-pvic-q1,t-pvic-q2,t-pvic-q3,t-psup-q1,t-psup-q2,t-psup-q3,t-psup-q4,t-acch-q1,t-acch-q2,t-acch-q3,t-reso-q1,t-reso-q2,t-reso-q3,t-reso-q4,t-reso-q5,t-sust-q1,t-sust-q2,t-sust-q3,t-sust-q4,t-curv-q1,t-curv-q2,t-curv-q3,t-curv-q4,t-cure-q1,t-cure-q2,t-cure-q3,t-cure-q4,t-peng-q1,t-peng-q2,t-peng-q3,t-peng-q4,t-ceng-q1,t-ceng-q2,t-ceng-q3,t-ceng-q4,t-sach-q1,t-sach-q2,t-sach-q3,t-psol-q1,t-psol-q2,t-psol-q3,t-expa-q2,t-expa-q3,t-phya-q2,t-phya-q3,,,
-2020-10-09 9:42:45,2020-10-09 11:08:57,0,50.204.125.194,93,5172,0,2020-10-16 11:09:03,teacher_survey_response_1,,,,,,,anonymous,EN,1,1,4,1500025,,,,,,3,,,,,,,,,,,3,3,3,NA,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,,,,,,,,,,,
+2020-10-09 9:42:45,2020-10-09 11:08:57,0,50.204.125.194,93,5172,0,2020-10-16T11:09:03,teacher_survey_response_1,,,,,,,anonymous,EN,1,1,4,1500025,,,,,,3,,,,,,,,,,,3,3,3,NA,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,,,,,,,,,,,
-2020-10-09 9:21:30,2020-10-09 14:28:09,0,50.204.125.194,24,18398,0,2020-10-16 14:28:18,teacher_survey_response_2,,,,,,,anonymous,EN,1,1,4,1500025,,,,,,,,,,,,,,,,,,,NA,,,,,NA,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+2020-10-09 9:21:30,2020-10-09 14:28:09,0,50.204.125.194,24,18398,0,2020-10-16T14:28:18,teacher_survey_response_2,,,,,,,anonymous,EN,1,1,4,1500025,,,,,,,,,,,,,,,,,,,NA,,,,,NA,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-2020-12-06 8:28:00,2020-12-06 8:36:52,0,73.61.140.32,100,532,1,2020-12-06 8:36:52,teacher_survey_response_3,,,,,41.92610168,-71.30110168,anonymous,EN,1,2,2,1500505,5,5,5,5,5,5,5,1,5,5,5,5,3,2,1,5,4,5,5,4,4,5,5,5,5,3,3,3,3,1,3,3,4,3,2,4,4,4,4,3,3,2,2,2,4,4,4,3,2,3,2,5,4,5,5,3,4,4,3,5,2,1,3,4,3,3,4,4,3,3,,9,9
+2020-12-06 8:28:00,2020-12-06 8:36:52,0,73.61.140.32,100,532,1,2020-12-06T8:36:52,teacher_survey_response_3,,,,,41.92610168,-71.30110168,anonymous,EN,1,2,2,1500505,5,5,5,5,5,5,5,1,5,5,5,5,3,2,1,5,4,5,5,4,4,5,5,5,5,3,3,3,3,1,3,3,4,3,2,4,4,4,4,3,3,2,2,2,4,4,4,3,2,3,2,5,4,5,5,3,4,4,3,5,2,1,3,4,3,3,4,4,3,3,,9,9
-2020-12-06 8:41:41,2020-12-06 8:51:25,0,96.230.183.162,100,584,1,2020-12-06 8:51:25,teacher_survey_response_4,,,,,41.98530579,-71.5184021,anonymous,EN,1,2,2,1500505,5,5,5,4,4,4,4,1,4,4,4,4,3,3,3,2,5,2,4,5,4,5,4,4,4,3,3,3,3,4,4,3,3,3,3,3,4,3,3,4,4,3,2,4,4,4,4,4,3,2,3,4,5,3,2,2,4,4,3,3,4,3,3,3,3,3,3,4,3,3,,9,6
+2020-12-06 8:41:41,2020-12-06 8:51:25,0,96.230.183.162,100,584,1,2020-12-06T8:51:25,teacher_survey_response_4,,,,,41.98530579,-71.5184021,anonymous,EN,1,2,2,1500505,5,5,5,4,4,4,4,1,4,4,4,4,3,3,3,2,5,2,4,5,4,5,4,4,4,3,3,3,3,4,4,3,3,3,3,3,4,3,3,4,4,3,2,4,4,4,4,4,3,2,3,4,5,3,2,2,4,4,3,3,4,3,3,3,3,3,3,4,3,3,,9,6
-2020-12-06 8:44:42,2020-12-06 8:55:58,0,216.41.82.178,100,675,1,2020-12-06 8:55:58,teacher_survey_response_5,,,,,41.92610168,-71.30110168,anonymous,EN,1,2,2,1500505,5,4,4,2,2,4,1,2,4,4,4,4,3,3,3,2,4,3,5,4,3,4,3,3,4,2,2,4,4,2,4,3,4,4,3,3,2,3,2,3,3,2,4,4,3,4,4,3,2,2,4,4,5,2,3,1,4,4,3,3,3,2,2,4,4,3,3,3,3,3,,7,5
+2020-12-06 8:44:42,2020-12-06 8:55:58,0,216.41.82.178,100,675,1,2020-12-06T8:55:58,teacher_survey_response_5,,,,,41.92610168,-71.30110168,anonymous,EN,1,2,2,1500505,5,4,4,2,2,4,1,2,4,4,4,4,3,3,3,2,4,3,5,4,3,4,3,3,4,2,2,4,4,2,4,3,4,4,3,3,2,3,2,3,3,2,4,4,3,4,4,3,2,2,4,4,5,2,3,1,4,4,3,3,3,2,2,4,4,3,3,3,3,3,,7,5

spec/services/cleaner_spec.rb

@@ -14,7 +14,8 @@ RSpec.describe Cleaner do
     create(:respondent, school: second_school, academic_year:, one: 0, four: 40, five: 40, six: 40, seven: 40,
                         eight: 40)
   end
-  let(:recorded_date) { "2023-04-01" }
+  let(:recorded_date) { "4/1/2023" }
+
   let(:input_filepath) do
     Rails.root.join("spec", "fixtures", "raw")
   end

spec/services/survey_item_values_spec.rb

@@ -36,7 +36,7 @@ RSpec.describe SurveyItemValues, type: :model do
     create(:respondent, school: attleboro, academic_year: ay_2022_23, nine: 40, ten: 40, eleven: 40, twelve: 40)
   end
   let(:schools) { School.school_hash }
-  let(:recorded_date) { "2023-04-01" }
+  let(:recorded_date) { "2023-04-01T12:12:12" }
   let(:ay_2022_23) do
     create(:academic_year, range: "2022-23")
   end
@@ -104,14 +104,14 @@ RSpec.describe SurveyItemValues, type: :model do
 
   context ".recorded_date" do
     it "returns the recorded date" do
-      row = { "RecordedDate" => "2017-01-01" }
+      row = { "RecordedDate" => "2017-01-01T12:12:121" }
       values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
-      expect(values.recorded_date).to eq Date.parse("2017-01-01")
+      expect(values.recorded_date).to eq Date.parse("2017-01-01T12:12:12")
 
       headers = ["Recorded Date"]
-      row = { "Recorded Date" => "2017-01-02" }
+      row = { "Recorded Date" => "2017-01-02T12:12:122" }
       values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
-      expect(values.recorded_date).to eq Date.parse("2017-01-02")
+      expect(values.recorded_date).to eq Date.parse("2017-01-02T12:12:12")
     end
   end
 
@@ -826,4 +826,27 @@ RSpec.describe SurveyItemValues, type: :model do
       end
     end
   end
+
+  context ".academic_year" do
+    before do
+      create(:academic_year, range: "2020-21")
+      create(:academic_year, range: "2021-22")
+    end
+
+    it "parses the date correctly when the date is in standard date format for google sheets" do
+      recorded_date = "1/10/2022 14:21:45"
+      values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, survey_items:,
+                                    schools:)
+      ay_21_22 = AcademicYear.find_by_range "2021-22"
+      expect(values.academic_year).to eq ay_21_22
+    end
+
+    it "parses the date correctly when the date is in iso standard 8601" do
+      recorded_date = "2022-1-10T14:21:45"
+      values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, survey_items:,
+                                    schools:)
+      ay_21_22 = AcademicYear.find_by_range "2021-22"
+      expect(values.academic_year).to eq ay_21_22
+    end
+  end
 end

spec/services/survey_responses_data_loader_spec.rb

@@ -316,10 +316,10 @@ def assigns_recorded_date_to_student_responses
 end
 
 def assigns_recorded_date_to_teacher_responses
-  results = { "teacher_survey_response_1" => "2020-10-16 11:09:03",
+  results = { "teacher_survey_response_1" => "2020-10-16T11:09:03",
-              "teacher_survey_response_3" => "2020-12-06 8:36:52",
+              "teacher_survey_response_3" => "2020-12-06T8:36:52",
-              "teacher_survey_response_4" => "2020-12-06 8:51:25",
+              "teacher_survey_response_4" => "2020-12-06T8:51:25",
-              "teacher_survey_response_5" => "2020-12-06 8:55:58" }
+              "teacher_survey_response_5" => "2020-12-06T8:55:58" }
   results.each do |key, value|
     expect(SurveyItemResponse.find_by_response_id(key).recorded_date).to eq Date.parse(value)
   end