gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 21:48:16 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: update how survey data is loaded because some data was missing from previous uploads

Browse source
This commit is contained in:
rebuilt 2025-09-17 19:22:31 -07:00
parent 4331c01789
commit 65acd68d84
1 changed files with 22 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

30
app/services/survey_responses_data_loader.rb
View file

@ -29,32 +29,34 @@ class SurveyResponsesDataLoader
all_survey_items = survey_items(headers:) all_survey_items = survey_items(headers:)
survey_item_responses = [] survey_item_responses = []
row_count = 0 batch_size = 1000
batch_size = 100
lines = []
until file.eof? until file.eof?
line = file.gets line = file.gets
next unless line.present? next unless line.present?
lines << line
end
lines.each_slice(batch_size) do |slice|
slice.each do |line|
CSV.parse(line, headers:).map do |row| CSV.parse(line, headers:).map do |row|
survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, row = process_row(row: SurveyItemValues.new(row:, headers: headers_array,
survey_items: all_survey_items, schools:, academic_years:)) survey_items: all_survey_items, schools:, academic_years:))
survey_item_responses.concat(row)
end end
row_count += 1 survey_item_responses = survey_item_responses.compact.flatten
next unless row_count == batch_size
survey_item_responses = survey_item_responses.compact.flatten.uniq do |response|
response.response_id
end
SurveyItemResponse.import(survey_item_responses, batch_size:, on_duplicate_key_update: :all) SurveyItemResponse.import(survey_item_responses, batch_size:, on_duplicate_key_update: :all)
survey_item_responses = [] unless file.eof?
GC.start survey_item_responses = []
row_count = 0 end
end end
SurveyItemResponse.import(survey_item_responses.compact.flatten, batch_size:, on_duplicate_key_update: :all) GC.start
end end
private private
@ -156,7 +158,7 @@ class SurveyResponsesDataLoader
likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next
unless likert_score.valid_likert_score? unless likert_score.valid_likert_score?
puts("Response ID: #{row.response_id}, Likert score: #{likert_score} rejected") unless likert_score == "NA" # puts("Response ID: #{row.response_id}, Likert score: #{likert_score} rejected") unless likert_score == "NA"
next next
end end