gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 21:48:16 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: ECP-152 Reduce batch size of survey uploads to 100 at a time from 500 to reduce memory usage and also explicitly run garbage collection so we stay under the 500 mb limit of heroku

Browse source
This commit is contained in:
rebuilt 2025-05-12 19:15:33 -07:00
parent 545bb85b1c
commit 76f2467b97
1 changed files with 7 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

11
app/services/survey_responses_data_loader.rb
View file

@ -29,6 +29,8 @@ class SurveyResponsesDataLoader
survey_item_responses = []
row_count = 0
batch_size = 100
until file.eof?
line = file.gets
next unless line.present?
@ -39,14 +41,15 @@ class SurveyResponsesDataLoader
end
row_count += 1
next unless row_count == 500
next unless row_count == batch_size
SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all)
survey_item_responses = []
SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all)
survey_item_responses = [] unless file.eof?
GC.start
row_count = 0
end
SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all)
SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all)
end
private