gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-08 23:18:18 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: clean up rake tasks

Browse source
This commit is contained in:
Nelson Jovel 2023-12-20 13:47:38 -08:00
parent bbefdcb5bb
commit 76bd79e8a2
2 changed files with 4 additions and 184 deletions
Download patch file Download diff file Expand all files Collapse all files

47
lib/tasks/data.rake
View file

@ -1,5 +1,3 @@
require "csv"
namespace :data do
desc "load survey responses"
task load_survey_responses: :environment do
@ -11,11 +9,6 @@ namespace :data do
end
puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
Sftp::Directory.open(path:) do |file|
StudentLoader.from_file(file:, rules: [])
end
puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
Rails.cache.clear
end
@ -29,22 +22,9 @@ namespace :data do
end
puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
Sftp::Directory.open(path:) do |file|
StudentLoader.from_file(file:, rules: [])
end
puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
Rails.cache.clear
end
desc "reset response rate values"
task reset_response_rates: :environment do
puts "Resetting response rates"
ResponseRateLoader.reset
Rails.cache.clear
puts "=====================> Completed loading #{ResponseRate.count} survey responses"
end
desc "load admin_data"
task load_admin_data: :environment do
original_count = AdminDataValue.count
@ -52,21 +32,12 @@ namespace :data do
puts "=====================> Loading data from csv at path: #{filepath}"
Dese::Loader.load_data filepath:
end
puts "=====================> Completed loading #{AdminDataValue.count - original_count} admin data values"
end
desc "load students"
task load_students: :environment do
SurveyItemResponse.update_all(student_id: nil)
StudentRace.delete_all
Student.delete_all
Dir.glob(Rails.root.join("data", "survey_responses", "*student*.csv")).each do |file|
puts "=====================> Loading student data from csv at path: #{file}"
StudentLoader.load_data filepath: file
Dir.glob(Rails.root.join("data", "admin_data", "out_of_state", "*.csv")).each do |filepath|
puts "=====================> Loading data from csv at path: #{filepath}"
Dese::Loader.load_data filepath:
end
puts "=====================> Completed loading #{Student.count} students"
Rails.cache.clear
puts "=====================> Completed loading #{AdminDataValue.count - original_count} admin data values"
end
desc "reset all cache counters"
@ -92,14 +63,4 @@ namespace :data do
SurveyItem.reset_counters(survey_item.id, :survey_item_responses)
end
end
desc "scrape dese site for admin data"
task scrape_all: :environment do
puts "scraping data from dese"
scrapers = [Dese::OneAOne, Dese::OneAThree, Dese::TwoAOne, Dese::TwoCOne, Dese::ThreeAOne, Dese::ThreeATwo,
Dese::ThreeBOne, Dese::ThreeBTwo, Dese::FourAOne, Dese::FourBTwo, Dese::FourDOne, Dese::FiveCOne, Dese::FiveDTwo]
scrapers.each do |scraper|
scraper.new.run_all
end
end
end