gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 21:48:16 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Load a single year of student demographic data and race scores at a time

Browse source
This commit is contained in:
rebuilt 2022-08-22 11:29:38 -07:00
parent 4a32ea3332
commit 3f493727b3
3 changed files with 31 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

40
app/services/race_score_loader.rb
View file

@ -1,25 +1,39 @@
class RaceScoreLoader
def self.reset(schools: School.all, academic_years: AcademicYear.all, measures: Measure.all, races: Race.all)
def self.reset(schools: School.all, academic_years: AcademicYear.all, measures: Measure.all, races: Race.all, fast_processing: true)
RaceScore.where(school: schools, academic_year: academic_years, measure: measures, race: races).delete_all
measures.each do |measure|
schools.each do |school|
loadable_race_scores = []
loadable_race_scores = academic_years.map do |academic_year|
races.map do |race|
process_score(measure:, school:, academic_year:, race:)
end
end
RaceScore.import(loadable_race_scores.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
@grouped_responses = nil
@total_responses = nil
@response_rate = nil
@sufficient_responses = nil
if fast_processing
large_memory_use(measure:, schools:, academic_years:, races:)
else
slow_loading_time(measure:, schools:, academic_years:, races:)
end
end
end
private
def self.large_memory_use(measure:, schools:, academic_years:, races:)
loadable_race_scores = schools.map do |school|
academic_years.map do |academic_year|
races.map do |race|
process_score(measure:, school:, academic_year:, race:)
end
end
end
RaceScore.import(loadable_race_scores.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
end
def self.slow_loading_time(measure:, schools:, academic_years:, races:)
schools.each do |school|
loadable_race_scores = academic_years.map do |academic_year|
races.map do |race|
process_score(measure:, school:, academic_year:, race:)
end
end
RaceScore.import(loadable_race_scores.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
end
end
def self.process_score(measure:, school:, academic_year:, race:)
score = race_score(measure:, school:, academic_year:, race:)
{ measure_id: measure.id, school_id: school.id, academic_year_id: academic_year.id, race_id: race.id, average: score.average,

2
app/services/student_loader.rb
View file

@ -6,7 +6,7 @@
require 'csv'
class StudentLoader
def self.load_data(filepath:, reinitialize: false)
def self.load_data(filepath:, reinitialize: true)
destroy_students if reinitialize
File.open(filepath) do |file|

5
lib/tasks/one_off.rake
View file

@ -71,7 +71,7 @@ namespace :one_off do
desc 'load students'
task load_students: :environment do
Dir.glob(Rails.root.join('data', 'survey_responses', '2019-20_*student*.csv')).each do |file|
Dir.glob(Rails.root.join('data', 'survey_responses', '2021-22_*student*.csv')).each do |file|
puts "=====================> Loading student data from csv at path: #{file}"
StudentLoader.load_data filepath: file
end
@ -119,7 +119,8 @@ namespace :one_off do
desc 'reset race score calculations'
task reset_race_scores: :environment do
puts 'Resetting race scores'
RaceScoreLoader.reset(schools: [School.find_by_slug('a-irvin-studley-elementary-school')])
academic_years = [AcademicYear.find_by_range('2021-22')]
RaceScoreLoader.reset(academic_years:)
Rails.cache.clear
puts "=====================> Completed loading #{RaceScore.count} race scores"
end