From 76f2467b97ebeccc644b598db66c1f9989107f32 Mon Sep 17 00:00:00 2001 From: rebuilt Date: Mon, 12 May 2025 19:15:33 -0700 Subject: [PATCH] chore: ECP-152 Reduce batch size of survey uploads to 100 at a time from 500 to reduce memory usage and also explicitly run garbage collection so we stay under the 500 mb limit of heroku --- app/services/survey_responses_data_loader.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index ea303679..c8e57917 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -29,6 +29,8 @@ class SurveyResponsesDataLoader survey_item_responses = [] row_count = 0 + batch_size = 100 + until file.eof? line = file.gets next unless line.present? @@ -39,14 +41,15 @@ class SurveyResponsesDataLoader end row_count += 1 - next unless row_count == 500 + next unless row_count == batch_size - SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all) - survey_item_responses = [] + SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all) + survey_item_responses = [] unless file.eof? + GC.start row_count = 0 end - SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all) + SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all) end private