gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 13:38:18 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Memoize admin data loader

Browse source
This commit is contained in:
Gabe Farrell 2024-04-18 17:13:03 -04:00
parent f0024d019d
commit 04d61c0b1a
2 changed files with 28 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

11
lib/tasks/data.rake
View file

@ -28,15 +28,14 @@ namespace :data do
desc "load admin_data"
task load_admin_data: :environment do
original_count = AdminDataValue.count
Dir.glob(Rails.root.join("data", "admin_data", "dese", "*.csv")).each do |filepath|
puts "=====================> Loading data from csv at path: #{filepath}"
Dese::Loader.load_data filepath:
end
Dir.glob(Rails.root.join("data", "admin_data", "out_of_state", "*.csv")).each do |filepath|
jobs = Queue.new
Dir.glob(Rails.root.join("data", "admin_data", "dese", "*.csv")).each { |filepath| jobs << filepath }
Dir.glob(Rails.root.join("data", "admin_data", "out_of_state", "*.csv")).each { |filepath| jobs << filepath }
while filepath = jobs.pop(true)
puts "=====================> Loading data from csv at path: #{filepath}"
Dese::Loader.load_data filepath:
end
rescue ThreadError
puts "=====================> Completed loading #{AdminDataValue.count - original_count} admin data values"
end