gabehf/Dashboard
1
0
Fork 0
mirror of https://github.com/edcommonwealth/Dashboard.git synced 2026-03-07 21:38:14 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: get admin data scraper running

Browse source
This commit is contained in:
Nelson Jovel 2024-02-07 11:47:25 -08:00
parent 1810ee0074
commit a142afe022
102 changed files with 41933 additions and 8756 deletions
Download patch file Download diff file Expand all files Collapse all files

4
lib/tasks/dashboard_tasks.rake
View file

@ -31,12 +31,12 @@ namespace :dashboard do
desc "load admin_data"
task load_admin_data: :environment do
original_count = AdminDataValue.count
Dir.glob(Dashboard::Engine.root.join("data", "dashboard", "admin_data", "dese", "*.csv")).each do |filepath|
Dir.glob(Dashboard::Engine.root.join("data", "admin_data", "dese", "*.csv")).each do |filepath|
puts "=====================> Loading data from csv at path: #{filepath}"
Dashboard::Dese::Loader.load_data filepath:
end
Dir.glob(Dashboard::Engine.root.join("data", "dashboard", "admin_data", "out_of_state",
Dir.glob(Dashboard::Engine.root.join("data", "admin_data", "out_of_state",
"*.csv")).each do |filepath|
puts "=====================> Loading data from csv at path: #{filepath}"
Dashboard::Dese::Loader.load_data filepath:

6
lib/tasks/db.rake
View file

@ -6,10 +6,10 @@ namespace :dashboard do
seeder.seed_academic_years "2016-17", "2017-18", "2018-19", "2019-20", "2020-21", "2021-22", "2022-23",
"2023-24"
seeder.seed_districts_and_schools Dashboard::Engine.root.join("data", "dashboard",
seeder.seed_districts_and_schools Dashboard::Engine.root.join("data",
"master_list_of_schools_and_districts.csv")
seeder.seed_sqm_framework Dashboard::Engine.root.join("data", "dashboard", "sqm_framework.csv")
seeder.seed_demographics Dashboard::Engine.root.join("data", "dashboard", "demographics.csv")
seeder.seed_sqm_framework Dashboard::Engine.root.join("data", "sqm_framework.csv")
seeder.seed_demographics Dashboard::Engine.root.join("data", "demographics.csv")
Dir.glob("#{Dashboard::Engine.root}/data/dashboard/enrollment/*.csv").each do |file|
seeder.seed_enrollment file

8
lib/tasks/scrape.rake
View file

@ -3,8 +3,8 @@ namespace :dashboard do
desc "scrape dese site for admin data"
task admin: :environment do
puts "scraping data from dese"
scrapers = [Dese::OneAOne, Dese::OneAThree, Dese::TwoAOne, Dese::TwoCOne, Dese::ThreeAOne, Dese::ThreeATwo,
Dese::ThreeBOne, Dese::ThreeBTwo, Dese::FourAOne, Dese::FourBTwo, Dese::FourDOne, Dese::FiveCOne, Dese::FiveDTwo]
scrapers = [Dashboard::Dese::OneAOne, Dashboard::Dese::OneAThree, Dashboard::Dese::TwoAOne, Dashboard::Dese::TwoCOne, Dashboard::Dese::ThreeAOne, Dashboard::Dese::ThreeATwo,
Dashboard::Dese::ThreeBOne, Dashboard::Dese::ThreeBTwo, Dashboard::Dese::FourAOne, Dashboard::Dese::FourBTwo, Dashboard::Dese::FourDOne, Dashboard::Dese::FiveCOne, Dashboard::Dese::FiveDTwo]
scrapers.each do |scraper|
scraper.new.run_all
end
@ -12,8 +12,8 @@ namespace :dashboard do
desc "scrape dese site for teacher staffing information"
task enrollment: :environment do
Dese::ThreeATwo.new.scrape_enrollments(filepath: Dashboard::Engine.root.join("data", "dashboard", "enrollment",
"enrollment.csv"))
Dese::ThreeATwo.new.scrape_enrollments(filepath: Dashboard::Engine.root.join("data", "enrollment",
"enrollments.csv"))
end
desc "scrape dese site for student staffing information"