From 02b335ef63ea53cc0dcf03ba9ed0a41a5f7cc223 Mon Sep 17 00:00:00 2001 From: rebuilt Date: Thu, 2 Nov 2023 12:21:07 -0700 Subject: [PATCH] feat: reduce number of rows to process at one time to reduce memory use --- app/services/survey_responses_data_loader.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index 0917d73c..d863b0e4 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -23,6 +23,7 @@ class SurveyResponsesDataLoader all_survey_items = survey_items(headers:) survey_item_responses = [] + row_count = 0 until file.eof? line = file.gets next unless line.present? @@ -31,9 +32,16 @@ class SurveyResponsesDataLoader survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:), rules:) end + + row_count += 1 + next unless row_count == 500 + + SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all + survey_item_responses = [] + row_count = 0 end - SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000, on_duplicate_key_update: :all + SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all end private