Missing progress or duration information does not result in a row removed in the cleaning process

app/services/survey_item_values.rb

@@ -110,7 +110,7 @@ class SurveyItemValues
   end
 
   def duration
-    @duration ||= value_from(pattern: /Duration|Duration \(in seconds\)|Duration\.\.\(in\.seconds\)/i).to_i
+    @duration ||= value_from(pattern: /Duration|Duration \(in seconds\)|Duration\.\.\(in\.seconds\)/i)
   end
 
   def valid?
@@ -134,15 +134,22 @@ class SurveyItemValues
   end
 
   def valid_duration?
-    return duration >= 300 if survey_type == :teacher
+    return true if duration.nil? || duration == '' || duration.downcase == 'n/a' || duration.downcase == 'na'
-    return duration >= 240 if survey_type == :standard
+
-    return duration >= 100 if survey_type == :short_form
+    span_in_seconds = duration.to_i
+    return span_in_seconds >= 300 if survey_type == :teacher
+    return span_in_seconds >= 240 if survey_type == :standard
+    return span_in_seconds >= 100 if survey_type == :short_form
 
     true
   end
 
   def valid_progress?
-    row['Progress'].to_i >= 25
+    progress = row['Progress']
+    return true if progress.nil? || progress == '' || progress.downcase == 'n/a' || progress.downcase == 'na'
+
+    progress = progress.to_i
+    progress.to_i >= 25
   end
 
   def valid_grade?

spec/services/survey_item_values_spec.rb

@@ -178,6 +178,27 @@ RSpec.describe SurveyItemValues, type: :model do
         values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '300' }, headers:, genders:, survey_items:,
                                       schools:)
         expect(values.valid_duration?).to eq true
+
+        headers = short_form_survey_items
+        values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '100' }, headers:, genders:, survey_items:,
+                                      schools:)
+        expect(values.valid_duration?).to eq true
+
+        # When duration is blank or N/A or NA, we don't have enough information to kick out the row as invalid so we keep it in
+        headers = short_form_survey_items
+        values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '' }, headers:, genders:, survey_items:,
+                                      schools:)
+        expect(values.valid_duration?).to eq true
+
+        headers = short_form_survey_items
+        values = SurveyItemValues.new(row: { 'Duration (in seconds)' => 'N/A' }, headers:, genders:, survey_items:,
+                                      schools:)
+        expect(values.valid_duration?).to eq true
+
+        headers = short_form_survey_items
+        values = SurveyItemValues.new(row: { 'Duration (in seconds)' => 'NA' }, headers:, genders:, survey_items:,
+                                      schools:)
+        expect(values.valid_duration?).to eq true
       end
     end
 
@@ -207,6 +228,22 @@ RSpec.describe SurveyItemValues, type: :model do
         values = SurveyItemValues.new(row: { 'Progress' => '25' }, headers:, genders:, survey_items:,
                                       schools:)
         expect(values.valid_progress?).to eq true
+
+        # When progress is blank or N/A or NA, we don't have enough information to kick out the row as invalid so we keep it in
+        headers = %w[s-sbel-q5 s-phys-q2 RecordedDate]
+        values = SurveyItemValues.new(row: { 'Progress' => '' }, headers:, genders:, survey_items:,
+                                      schools:)
+        expect(values.valid_progress?).to eq true
+
+        headers = %w[s-sbel-q5 s-phys-q2 RecordedDate]
+        values = SurveyItemValues.new(row: { 'Progress' => 'N/A' }, headers:, genders:, survey_items:,
+                                      schools:)
+        expect(values.valid_progress?).to eq true
+
+        headers = %w[s-sbel-q5 s-phys-q2 RecordedDate]
+        values = SurveyItemValues.new(row: { 'Progress' => 'NA' }, headers:, genders:, survey_items:,
+                                      schools:)
+        expect(values.valid_progress?).to eq true
       end
     end