From 47c1856281345f7a2005ad5ab150fcc6f918e247 Mon Sep 17 00:00:00 2001 From: rebuilt Date: Tue, 3 Jan 2023 16:27:16 -0800 Subject: [PATCH] Process 1000 rows at a time to limit memory usage in production --- app/services/survey_responses_data_loader.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index 2cea151d..b08e4dd3 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -26,6 +26,7 @@ class SurveyResponsesDataLoader all_survey_items = survey_items(headers:) survey_item_responses = [] + row_count = 0 until file.eof? line = file.gets next unless line.present? @@ -35,8 +36,13 @@ class SurveyResponsesDataLoader rules:) end + row_count += 1 + next unless row_count == 1000 + + SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000 + survey_item_responses = [] + row_count = 0 end - SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000 end private