chore: ECP-152 Reduce batch size of survey uploads to 100 at a time from 500 to reduce memory usage and also explicitly run garbage collection so we stay under the 500 mb limit of heroku

app/services/survey_responses_data_loader.rb

@@ -30,6 +30,8 @@ class SurveyResponsesDataLoader
 
     survey_item_responses = []
     row_count = 0
+    batch_size = 100
+
     until file.eof?
       line = file.gets
       next unless line.present?
@@ -40,22 +42,15 @@ class SurveyResponsesDataLoader
       end
 
       row_count += 1
-      next unless row_count == BATCH_SIZE
+      next unless row_count == batch_size
 
-      SurveyItemResponse.import(
-        survey_item_responses.compact.flatten,
-        batch_size: BATCH_SIZE,
-        on_duplicate_key_update: :all
-      )
-      survey_item_responses = []
+      SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all)
+      survey_item_responses = [] unless file.eof?
+      GC.start
       row_count = 0
     end
 
-    SurveyItemResponse.import(
-      survey_item_responses.compact.flatten,
-      batch_size: BATCH_SIZE,
-      on_duplicate_key_update: :all
-    )
+    SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all)
   end
 
   private