diff --git a/app/services/student_loader.rb b/app/services/student_loader.rb index d2fdc157..a3f03fff 100644 --- a/app/services/student_loader.rb +++ b/app/services/student_loader.rb @@ -6,11 +6,12 @@ class StudentLoader def self.load_data(filepath:, rules: []) File.open(filepath) do |file| headers = file.first + headers_array = headers.split(',') file.lazy.each_slice(1_000) do |lines| CSV.parse(lines.join, headers:).map do |row| next if rules.any? do |rule| - rule.new(row: SurveyItemValues.new(row:, headers:, genders: nil, survey_items: nil, + rule.new(row: SurveyItemValues.new(row:, headers: headers_array, genders: nil, survey_items: nil, schools:)).skip_row? end @@ -22,6 +23,7 @@ class StudentLoader def self.from_file(file:, rules: []) headers = file.gets + headers_array = headers.split(',') survey_item_responses = [] until file.eof? @@ -30,7 +32,7 @@ class StudentLoader CSV.parse(line, headers:).map do |row| next if rules.any? do |rule| - rule.new(row: SurveyItemValues.new(row:, headers:, genders: nil, survey_items: nil, + rule.new(row: SurveyItemValues.new(row:, headers: headers_array, genders: nil, survey_items: nil, schools:)).skip_row? end diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index c2f367ed..1ca0d91a 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -4,12 +4,13 @@ class SurveyResponsesDataLoader def self.load_data(filepath:, rules: [Rule::NoRule]) File.open(filepath) do |file| headers = file.first + headers_array = headers.split(',') genders_hash = genders all_survey_items = survey_items(headers:) file.lazy.each_slice(500) do |lines| survey_item_responses = CSV.parse(lines.join, headers:).map do |row| - process_row(row: SurveyItemValues.new(row:, headers: headers.split(','), genders: genders_hash, survey_items: all_survey_items, schools:), + process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders: genders_hash, survey_items: all_survey_items, schools:), rules:) end SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500 @@ -19,6 +20,7 @@ class SurveyResponsesDataLoader def self.from_file(file:, rules: []) headers = file.gets + headers_array = headers.split(',') genders_hash = genders all_survey_items = survey_items(headers:) @@ -29,7 +31,7 @@ class SurveyResponsesDataLoader next unless line.present? CSV.parse(line, headers:).map do |row| - survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers.split(','), genders: genders_hash, survey_items: all_survey_items, schools:), + survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders: genders_hash, survey_items: all_survey_items, schools:), rules:) end