chore: ECP-152 Reduce batch size of survey uploads to 100 at a time from 500 to reduce memory usage and also explicitly run garbage collection so we stay under the 500 mb limit of heroku

app/services/survey_responses_data_loader.rb

@@ -29,6 +29,8 @@ class SurveyResponsesDataLoader
 
     survey_item_responses = []
     row_count = 0
+    batch_size = 100
+
     until file.eof?
       line = file.gets
       next unless line.present?
@@ -39,14 +41,15 @@ class SurveyResponsesDataLoader
       end
 
       row_count += 1
-      next unless row_count == 500
+      next unless row_count == batch_size
 
-      SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all)
-      survey_item_responses = []
+      SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all)
+      survey_item_responses = [] unless file.eof?
+      GC.start
       row_count = 0
     end
 
-    SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all)
+    SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all)
   end
 
   private