From 31e9779debcde89c4336c1a09eb5f31eb8b88d9c Mon Sep 17 00:00:00 2001 From: rebuilt Date: Mon, 12 May 2025 19:15:33 -0700 Subject: [PATCH] chore: ECP-152 Reduce batch size of survey uploads to 100 at a time from 500 to reduce memory usage and also explicitly run garbage collection so we stay under the 500 mb limit of heroku --- app/services/survey_responses_data_loader.rb | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index a0d5c36a..63b0d027 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -30,6 +30,8 @@ class SurveyResponsesDataLoader survey_item_responses = [] row_count = 0 + batch_size = 100 + until file.eof? line = file.gets next unless line.present? @@ -40,22 +42,15 @@ class SurveyResponsesDataLoader end row_count += 1 - next unless row_count == BATCH_SIZE + next unless row_count == batch_size - SurveyItemResponse.import( - survey_item_responses.compact.flatten, - batch_size: BATCH_SIZE, - on_duplicate_key_update: :all - ) - survey_item_responses = [] + SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all) + survey_item_responses = [] unless file.eof? + GC.start row_count = 0 end - SurveyItemResponse.import( - survey_item_responses.compact.flatten, - batch_size: BATCH_SIZE, - on_duplicate_key_update: :all - ) + SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all) end private