gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-08 23:18:18 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: reduce number of rows to process at one time to reduce memory use

Browse source
This commit is contained in:
rebuilt 2023-11-02 12:21:07 -07:00
parent 1a707eb6bc
commit cddea60c8b
1 changed files with 9 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

10
app/services/survey_responses_data_loader.rb
View file

@ -23,6 +23,7 @@ class SurveyResponsesDataLoader
all_survey_items = survey_items(headers:)
survey_item_responses = []
row_count = 0
until file.eof?
line = file.gets
next unless line.present?
@ -31,9 +32,16 @@ class SurveyResponsesDataLoader
survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:),
rules:)
end
row_count += 1
next unless row_count == 500
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all
survey_item_responses = []
row_count = 0
end
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000, on_duplicate_key_update: :all
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all
end
private