gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 21:48:16 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: load student responses in the same pass as loading the survey responses

Browse source 
chore: remove student loader since loading students is now done with the survey response loader
This commit is contained in:
rebuilt 2023-11-02 09:45:30 -07:00
parent 2d90516f9f
commit 1a707eb6bc
7 changed files with 76 additions and 504 deletions
Download patch file Download diff file Expand all files Collapse all files

64
app/services/student_loader.rb
View file

@ -1,64 +0,0 @@
# frozen_string_literal: true
class StudentLoader
def self.load_data(filepath:, rules: [])
File.open(filepath) do |file|
headers = file.first
headers_array = headers.split(",")
file.lazy.each_slice(1_000) do |lines|
CSV.parse(lines.join, headers:).map do |row|
row = SurveyItemValues.new(row:, headers: headers_array, genders: nil, survey_items: nil, schools:)
next if rules.any? do |rule|
rule.new(row:).skip_row?
end
process_row(row:)
end
end
end
end
def self.from_file(file:, rules: [])
headers = file.gets
headers_array = headers.split(",")
until file.eof?
line = file.gets
next unless line.present?
CSV.parse(line, headers:).map do |row|
row = SurveyItemValues.new(row:, headers: headers_array, genders: nil, survey_items: nil, schools:)
next if rules.any? do |rule|
rule.new(row:).skip_row?
end
process_row(row:)
end
end
end
def self.process_row(row:)
student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid)
student.races.delete_all
races = row.races
races.map do |race|
student.races << race
end
assign_student_to_responses(student:, response_id: row.response_id)
end
def self.schools
@schools ||= School.all.map { |school| [school.dese_id, school] }.to_h
end
def self.assign_student_to_responses(student:, response_id:)
responses = SurveyItemResponse.where(response_id:)
loadable_responses = responses.map do |response|
response.student = student
response
end
SurveyItemResponse.import(loadable_responses.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
end
end

26
app/services/survey_item_values.rb
View file

@ -116,20 +116,24 @@ class SurveyItemValues
end
def gender
gender_code = value_from(pattern: /Gender|What is your gender?|What is your gender? - Selected Choice/i)
gender_code ||= 99
gender_code = gender_code.to_i
gender_code = 4 if gender_code == 3
gender_code = 99 if gender_code.zero?
genders[gender_code]
@gender ||= begin
gender_code = value_from(pattern: /Gender|What is your gender?|What is your gender? - Selected Choice/i)
gender_code ||= 99
gender_code = gender_code.to_i
gender_code = 4 if gender_code == 3
gender_code = 99 if gender_code.zero?
genders[gender_code]
end
end
def races
race_codes = value_from(pattern: /RACE/i)
race_codes ||= value_from(pattern: %r{What is your race/ethnicity?(Please select all that apply) - Selected Choice}i)
race_codes ||= value_from(pattern: /Race Secondary/i) || ""
race_codes = race_codes.split(",").map(&:to_i) || []
process_races(codes: race_codes)
@races ||= begin
race_codes = value_from(pattern: /RACE/i)
race_codes ||= value_from(pattern: %r{What is your race/ethnicity?(Please select all that apply) - Selected Choice}i)
race_codes ||= value_from(pattern: /Race Secondary/i) || ""
race_codes = race_codes.split(",").map(&:to_i) || []
process_races(codes: race_codes)
end
end
def lasid

38
app/services/survey_responses_data_loader.rb
View file

@ -12,7 +12,7 @@ class SurveyResponsesDataLoader
process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:),
rules:)
end
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all
end
end
end
@ -23,7 +23,6 @@ class SurveyResponsesDataLoader
all_survey_items = survey_items(headers:)
survey_item_responses = []
row_count = 0
until file.eof?
line = file.gets
next unless line.present?
@ -32,16 +31,9 @@ class SurveyResponsesDataLoader
survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:),
rules:)
end
row_count += 1
next unless row_count == 1000
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000
survey_item_responses = []
row_count = 0
end
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000, on_duplicate_key_update: :all
end
private
@ -78,6 +70,13 @@ class SurveyResponsesDataLoader
end
def process_survey_items(row:)
student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid)
student.races.delete_all
races = row.races
races.map do |race|
student.races << race
end
row.survey_items.map do |survey_item|
likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next
@ -86,23 +85,30 @@ class SurveyResponsesDataLoader
next
end
response = row.survey_item_response(survey_item:)
create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:)
create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, student:)
end.compact
end
def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:)
def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, student:)
gender = row.gender
grade = row.grade
income = incomes[row.income.parameterize]
ell = ells[row.ell]
sped = speds[row.sped]
if survey_item_response.present?
survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:,
sped:)
[]
survey_item_response.likert_score = likert_score
survey_item_response.grade = grade
survey_item_response.gender = gender
survey_item_response.recorded_date = row.recorded_date
survey_item_response.income = income
survey_item_response.ell = ell
survey_item_response.sped = sped
survey_item_response.student = student
survey_item_response
else
SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:,
likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:)
likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:, student:)
end
end