gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-09 07:28:41 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

candidate ~1.1s

Browse source
This commit is contained in:
Gabe Farrell 2024-04-18 02:13:38 -04:00
parent f6c6d26a2c
commit 9b0cc1527f
2 changed files with 30 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

21
lib/tasks/data.rake
View file

@ -28,15 +28,20 @@ namespace :data do
desc "load admin_data"
task load_admin_data: :environment do
original_count = AdminDataValue.count
Dir.glob(Rails.root.join("data", "admin_data", "dese", "*.csv")).each do |filepath|
puts "=====================> Loading data from csv at path: #{filepath}"
Dese::Loader.load_data filepath:
end
Dir.glob(Rails.root.join("data", "admin_data", "out_of_state", "*.csv")).each do |filepath|
puts "=====================> Loading data from csv at path: #{filepath}"
Dese::Loader.load_data filepath:
pool_size = 2
jobs = Queue.new
Dir.glob(Rails.root.join("data", "admin_data", "dese", "*.csv")).each { |filepath| jobs << filepath }
Dir.glob(Rails.root.join("data", "admin_data", "out_of_state", "*.csv")).each { |filepath| jobs << filepath }
workers = pool_size.times.map do
Thread.new do
while filepath = jobs.pop(true)
puts "=====================> Loading data from csv at path: #{filepath}"
Dese::Loader.load_data filepath:
end
rescue ThreadError
end
end
workers.each(&:join)
puts "=====================> Completed loading #{AdminDataValue.count - original_count} admin data values"
end