gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 21:48:16 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix rake task for uploading responses so that it also loads student racial information

Browse source
This commit is contained in:
rebuilt 2023-01-06 17:40:09 -08:00
parent b3ded7e7be
commit e8825c6796
1 changed files with 14 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

22
lib/tasks/data.rake
View file

@ -34,11 +34,17 @@ namespace :data do
desc 'load survey responses for lowell schools' desc 'load survey responses for lowell schools'
task load_survey_responses_for_lowell: :environment do task load_survey_responses_for_lowell: :environment do
Dir.glob(Rails.root.join('data', 'survey_responses', '*.csv')).each do |filepath| survey_item_response_count = SurveyItemResponse.count
puts "=====================> Loading data from csv at path: #{filepath}" student_count = Student.count
SurveyResponsesDataLoader.load_data filepath:, rules: [Rule::SkipNonLowellSchools] Sftp::Directory.open(path: '/data/survey_responses/clean/') do |file|
SurveyResponsesDataLoader.from_file(file:)
end end
puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses" puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
Sftp::Directory.open(path: '/data/survey_responses/clean/') do |file|
StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools])
end
puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
puts 'Resetting response rates' puts 'Resetting response rates'
ResponseRateLoader.reset ResponseRateLoader.reset
@ -56,11 +62,11 @@ namespace :data do
SurveyItemResponse.update_all(student_id: nil) SurveyItemResponse.update_all(student_id: nil)
StudentRace.delete_all StudentRace.delete_all
Student.delete_all Student.delete_all
Dir.glob(Rails.root.join('data', 'survey_responses', '*student*.csv')).each do |file|
puts "=====================> Loading student data from csv at path: #{file}" Sftp::Directory.open(path: '/data/survey_responses/clean/') do |file|
StudentLoader.load_data filepath: file, rules: [Rule::SkipNonLowellSchools] StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools])
end end
puts "=====================> Completed loading #{Student.count} students" puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
puts 'Resetting race scores' puts 'Resetting race scores'
RaceScoreLoader.reset(fast_processing: false) RaceScoreLoader.reset(fast_processing: false)