gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-13 09:20:38 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix regression in student loader

Browse source
This commit is contained in:
rebuilt 2023-05-19 13:48:16 -07:00
parent bd4e64167d
commit 37e932e078
2 changed files with 8 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

6
app/services/student_loader.rb
View file

@ -6,11 +6,12 @@ class StudentLoader
def self.load_data(filepath:, rules: []) def self.load_data(filepath:, rules: [])
File.open(filepath) do |file| File.open(filepath) do |file|
headers = file.first headers = file.first
headers_array = headers.split(',')
file.lazy.each_slice(1_000) do |lines| file.lazy.each_slice(1_000) do |lines|
CSV.parse(lines.join, headers:).map do |row| CSV.parse(lines.join, headers:).map do |row|
next if rules.any? do |rule| next if rules.any? do |rule|
rule.new(row: SurveyItemValues.new(row:, headers:, genders: nil, survey_items: nil, rule.new(row: SurveyItemValues.new(row:, headers: headers_array, genders: nil, survey_items: nil,
schools:)).skip_row? schools:)).skip_row?
end end
@ -22,6 +23,7 @@ class StudentLoader
def self.from_file(file:, rules: []) def self.from_file(file:, rules: [])
headers = file.gets headers = file.gets
headers_array = headers.split(',')
survey_item_responses = [] survey_item_responses = []
until file.eof? until file.eof?
@ -30,7 +32,7 @@ class StudentLoader
CSV.parse(line, headers:).map do |row| CSV.parse(line, headers:).map do |row|
next if rules.any? do |rule| next if rules.any? do |rule|
rule.new(row: SurveyItemValues.new(row:, headers:, genders: nil, survey_items: nil, rule.new(row: SurveyItemValues.new(row:, headers: headers_array, genders: nil, survey_items: nil,
schools:)).skip_row? schools:)).skip_row?
end end

6
app/services/survey_responses_data_loader.rb
View file

@ -4,12 +4,13 @@ class SurveyResponsesDataLoader
def self.load_data(filepath:, rules: [Rule::NoRule]) def self.load_data(filepath:, rules: [Rule::NoRule])
File.open(filepath) do |file| File.open(filepath) do |file|
headers = file.first headers = file.first
headers_array = headers.split(',')
genders_hash = genders genders_hash = genders
all_survey_items = survey_items(headers:) all_survey_items = survey_items(headers:)
file.lazy.each_slice(500) do |lines| file.lazy.each_slice(500) do |lines|
survey_item_responses = CSV.parse(lines.join, headers:).map do |row| survey_item_responses = CSV.parse(lines.join, headers:).map do |row|
process_row(row: SurveyItemValues.new(row:, headers: headers.split(','), genders: genders_hash, survey_items: all_survey_items, schools:), process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders: genders_hash, survey_items: all_survey_items, schools:),
rules:) rules:)
end end
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500 SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500
@ -19,6 +20,7 @@ class SurveyResponsesDataLoader
def self.from_file(file:, rules: []) def self.from_file(file:, rules: [])
headers = file.gets headers = file.gets
headers_array = headers.split(',')
genders_hash = genders genders_hash = genders
all_survey_items = survey_items(headers:) all_survey_items = survey_items(headers:)
@ -29,7 +31,7 @@ class SurveyResponsesDataLoader
next unless line.present? next unless line.present?
CSV.parse(line, headers:).map do |row| CSV.parse(line, headers:).map do |row|
survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers.split(','), genders: genders_hash, survey_items: all_survey_items, schools:), survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders: genders_hash, survey_items: all_survey_items, schools:),
rules:) rules:)
end end