gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-13 17:30:38 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add disaggregation by ELL

Browse source
This commit is contained in:
rebuilt 2023-08-30 15:18:38 -07:00
parent 490522eb1e
commit 2fd56047d4
43 changed files with 947 additions and 429 deletions
Download patch file Download diff file Expand all files Collapse all files

32
lib/tasks/clean.rake
View file

@ -1,33 +1,33 @@
namespace :clean do
# These tasks must be run in their respective project so the correct schools are in the database
desc 'clean ecp data'
desc "clean ecp data"
task ecp: :environment do
input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'raw')
output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'removed')
input_filepath = Rails.root.join("tmp", "data", "ecp_data", "raw")
output_filepath = Rails.root.join("tmp", "data", "ecp_data", "clean")
log_filepath = Rails.root.join("tmp", "data", "ecp_data", "removed")
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
desc 'clean prepped data'
desc "clean prepped data"
task prepped: :environment do
input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped')
output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'removed')
input_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped")
output_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "clean")
log_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "removed")
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
desc 'clean mciea data'
desc "clean mciea data"
task mciea: :environment do
input_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'raw')
output_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'removed')
input_filepath = Rails.root.join("tmp", "data", "mciea_data", "raw")
output_filepath = Rails.root.join("tmp", "data", "mciea_data", "clean")
log_filepath = Rails.root.join("tmp", "data", "mciea_data", "removed")
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
desc 'clean rpp data'
desc "clean rpp data"
task rpp: :environment do
input_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'raw')
output_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'removed')
input_filepath = Rails.root.join("tmp", "data", "rpp_data", "raw")
output_filepath = Rails.root.join("tmp", "data", "rpp_data", "clean")
log_filepath = Rails.root.join("tmp", "data", "rpp_data", "removed")
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
end

2
lib/tasks/data.rake
View file

@ -29,7 +29,7 @@ namespace :data do
student_count = Student.count
path = '/data/survey_responses/clean/'
Sftp::Directory.open(path:) do |file|
SurveyResponsesDataLoader.from_file(file:)
SurveyResponsesDataLoader.new.from_file(file:)
end
puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"

23
lib/tasks/one_off.rake
View file

@ -82,27 +82,6 @@ namespace :one_off do
puts values
end
desc "load survey responses for lowell schools"
task load_survey_responses_for_lowell: :environment do
survey_item_response_count = SurveyItemResponse.count
student_count = Student.count
Sftp::Directory.open(path: "/test/survey_responses/") do |file|
SurveyResponsesDataLoader.from_file(file:)
end
puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
Sftp::Directory.open(path: "/test/survey_responses/") do |file|
StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools])
end
puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
puts "Resetting race scores"
RaceScoreLoader.reset(fast_processing: false)
puts "=====================> Completed loading #{RaceScore.count} race scores"
Rails.cache.clear
end
desc "delete 2022-23 survey responses"
task delete_survey_responses_2022_23: :environment do
response_count = SurveyItemResponse.all.count
@ -127,7 +106,7 @@ namespace :one_off do
path = "/data/survey_responses/2022-23/"
Sftp::Directory.open(path:) do |file|
SurveyResponsesDataLoader.from_file(file:)
SurveyResponsesDataLoader.new.from_file(file:)
end
puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"