gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-09 15:38:21 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Scrape enrollment and staffing information. Seed enrollment and staffing information. Update DatabaseCleaner so it cleans up leftover information in the database. Remove old admin csvs from codebase.

Browse source
This commit is contained in:
rebuilt 2023-03-05 16:15:35 -08:00
parent 20d4f966e7
commit 06f9d2f0e9
27 changed files with 48177 additions and 4843 deletions
Download patch file Download diff file Expand all files Collapse all files

16
lib/tasks/data.rake
View file

@ -29,10 +29,8 @@ namespace :data do
seeder.seed_surveys Rails.root.join('data', 'master_list_of_schools_and_districts.csv')
seeder.seed_sqm_framework Rails.root.join('data', 'sqm_framework.csv')
seeder.seed_demographics Rails.root.join('data', 'demographics.csv')
Dir.glob(Rails.root.join('data', 'enrollment', '*')).each do |file|
seeder.seed_enrollment(file)
end
seeder.seed_enrollment Rails.root.join('data', 'enrollment', 'enrollment.csv')
seeder.seed_staffing Rails.root.join('data', 'staffing', 'staffing.csv')
end
desc 'load survey responses for lowell schools'
@ -206,14 +204,4 @@ namespace :data do
SurveyItem.reset_counters(survey_item.id, :survey_item_responses)
end
end
desc 'scrape dese site for admin data'
task scrape_all: :environment do
puts 'scraping data from dese'
scrapers = [Dese::OneAOne, Dese::OneAThree, Dese::TwoAOne, Dese::TwoCOne, Dese::ThreeAOne, Dese::ThreeATwo,
Dese::ThreeBOne, Dese::ThreeBTwo, Dese::FourAOne, Dese::FourBTwo, Dese::FourDOne, Dese::FiveCOne, Dese::FiveDTwo]
scrapers.each do |scraper|
scraper.new.run_all
end
end
end

21
lib/tasks/scrape.rake Normal file
View file

@ -0,0 +1,21 @@
namespace :scrape do
desc 'scrape dese site for admin data'
task admin: :environment do
puts 'scraping data from dese'
scrapers = [Dese::OneAOne, Dese::OneAThree, Dese::TwoAOne, Dese::TwoCOne, Dese::ThreeAOne, Dese::ThreeATwo,
Dese::ThreeBOne, Dese::ThreeBTwo, Dese::FourAOne, Dese::FourBTwo, Dese::FourDOne, Dese::FiveCOne, Dese::FiveDTwo]
scrapers.each do |scraper|
scraper.new.run_all
end
end
desc 'scrape dese site for teacher staffing information'
task enrollment: :environment do
Dese::ThreeATwo.new.scrape_enrollments(filepath: Rails.root.join('data', 'enrollment', 'enrollment.csv'))
end
desc 'scrape dese site for student staffing information'
task staffing: :environment do
Dese::OneAThree.new(filepaths: ['not used', Rails.root.join('data', 'staffing', 'staffing.csv')]).run_a_pcom_i3
end
end