gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 21:48:16 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add disaggregation by ELL

Browse source
This commit is contained in:
rebuilt 2023-08-30 15:18:38 -07:00
parent 8d33095a48
commit 060d7aa55a
41 changed files with 707 additions and 376 deletions
Download patch file Download diff file Expand all files Collapse all files

44
lib/tasks/clean.rake
View file

@ -1,37 +1,33 @@
namespace :clean do
# These tasks must be run in their respective project so the correct schools are in the database
desc 'clean ecp data'
desc "clean ecp data"
task ecp: :environment do
input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'raw')
output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'removed')
disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
input_filepath = Rails.root.join("tmp", "data", "ecp_data", "raw")
output_filepath = Rails.root.join("tmp", "data", "ecp_data", "clean")
log_filepath = Rails.root.join("tmp", "data", "ecp_data", "removed")
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
desc 'clean prepped data'
desc "clean prepped data"
task prepped: :environment do
input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped')
output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'removed')
disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
input_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped")
output_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "clean")
log_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "removed")
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
desc 'clean mciea data'
desc "clean mciea data"
task mciea: :environment do
input_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'raw')
output_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'removed')
disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
input_filepath = Rails.root.join("tmp", "data", "mciea_data", "raw")
output_filepath = Rails.root.join("tmp", "data", "mciea_data", "clean")
log_filepath = Rails.root.join("tmp", "data", "mciea_data", "removed")
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
desc 'clean rpp data'
desc "clean rpp data"
task rpp: :environment do
input_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'raw')
output_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'removed')
disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
input_filepath = Rails.root.join("tmp", "data", "rpp_data", "raw")
output_filepath = Rails.root.join("tmp", "data", "rpp_data", "clean")
log_filepath = Rails.root.join("tmp", "data", "rpp_data", "removed")
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
end

2
lib/tasks/data.rake
View file

@ -5,7 +5,7 @@ namespace :data do
student_count = Student.count
path = "/data/survey_responses/clean/"
Sftp::Directory.open(path:) do |file|
SurveyResponsesDataLoader.from_file(file:)
SurveyResponsesDataLoader.new.from_file(file:)
end
puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"

51
lib/tasks/one_off.rake
View file

@ -40,38 +40,6 @@ namespace :one_off do
end
end
desc 'load stoklosa results for 2022-23'
task load_stoklosa: :environment do
survey_item_response_count = SurveyItemResponse.count
school = School.find_by_dese_id(1_600_360)
academic_year = AcademicYear.find_by_range('2022-23')
['2022-23_stoklosa_student_survey_responses.csv',
'2022-23_stoklosa_teacher_survey_responses.csv'].each do |filepath|
filepath = Rails.root.join('data', 'survey_responses', filepath)
puts "=====================> Loading data from csv at path: #{filepath}"
SurveyResponsesDataLoader.load_data filepath:
end
puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
Dir.glob(Rails.root.join('data', 'survey_responses',
'2022-23_stoklosa_student_survey_responses.csv')).each do |file|
puts "=====================> Loading student data from csv at path: #{file}"
StudentLoader.load_data filepath: file, rules: [Rule::SkipNonLowellSchools]
end
end
desc 'load butler results for 2022-23'
task load_butler: :environment do
['2022-23_butler_student_survey_responses.csv',
'2022-23_butler_teacher_survey_responses.csv'].each do |filepath|
filepath = Rails.root.join('data', 'survey_responses', filepath)
puts "=====================> Loading data from csv at path: #{filepath}"
SurveyResponsesDataLoader.load_data filepath:
end
end
desc 'list scales that have no survey responses'
task list_scales_that_lack_survey_responses: :environment do
output = AcademicYear.all.map do |academic_year|
@ -114,23 +82,6 @@ namespace :one_off do
puts values
end
desc 'load survey responses for lowell schools'
task load_survey_responses_for_lowell: :environment do
survey_item_response_count = SurveyItemResponse.count
student_count = Student.count
Sftp::Directory.open(path: '/test/survey_responses/') do |file|
SurveyResponsesDataLoader.from_file(file:)
end
puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
Sftp::Directory.open(path: '/test/survey_responses/') do |file|
StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools])
end
puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
Rails.cache.clear
end
desc 'delete 2022-23 survey responses'
task delete_survey_responses_2022_23: :environment do
response_count = SurveyItemResponse.all.count
@ -148,7 +99,7 @@ namespace :one_off do
schools = District.find_by_slug('maynard-public-schools').schools
Sftp::Directory.open(path:) do |file|
SurveyResponsesDataLoader.from_file(file:)
SurveyResponsesDataLoader.new.from_file(file:)
end
puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"