gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 21:48:16 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: ECP-152 Reduce batch size of survey uploads to 100 at a time from 500 to reduce memory usage and also explicitly run garbage collection so we stay under the 500 mb limit of heroku

Browse source
This commit is contained in:
rebuilt 2025-05-12 19:15:33 -07:00
parent a6c68df349
commit 31e9779deb
1 changed files with 7 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

19
app/services/survey_responses_data_loader.rb
View file

@ -30,6 +30,8 @@ class SurveyResponsesDataLoader
survey_item_responses = []
row_count = 0
batch_size = 100
until file.eof?
line = file.gets
next unless line.present?
@ -40,22 +42,15 @@ class SurveyResponsesDataLoader
end
row_count += 1
next unless row_count == BATCH_SIZE
next unless row_count == batch_size
SurveyItemResponse.import(
survey_item_responses.compact.flatten,
batch_size: BATCH_SIZE,
on_duplicate_key_update: :all
)
survey_item_responses = []
SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all)
survey_item_responses = [] unless file.eof?
GC.start
row_count = 0
end
SurveyItemResponse.import(
survey_item_responses.compact.flatten,
batch_size: BATCH_SIZE,
on_duplicate_key_update: :all
)
SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all)
end
private