gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 21:48:16 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Clean up rake tasks to get data:load_survey_responses working

Browse source
This commit is contained in:
rebuilt 2023-04-26 15:56:17 -07:00
parent 4b7d2e6feb
commit fb963f63e0
1 changed files with 12 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

25
lib/tasks/data.rake
View file

@ -1,19 +1,22 @@
namespace :data do namespace :data do
desc 'load survey responses' desc 'load survey responses'
task load_survey_responses: :environment do task load_survey_responses: :environment do
Dir.glob(Rails.root.join('data', 'survey_responses', '*.csv')).each do |filepath| survey_item_response_count = SurveyItemResponse.count
puts "=====================> Loading data from csv at path: #{filepath}" student_count = Student.count
SurveyResponsesDataLoader.load_data filepath: path = '/data/survey_responses/clean/'
Sftp::Directory.open(path:) do |file|
SurveyResponsesDataLoader.from_file(file:)
end end
puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses" puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
puts 'Resetting response rates' Sftp::Directory.open(path:) do |file|
ResponseRateLoader.reset StudentLoader.from_file(file:, rules: [])
puts "=====================> Completed loading #{ResponseRate.count} survey responses" end
puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
puts 'Resetting race scores' puts 'Resetting race scores'
RaceScoreLoader.reset(fast_processing: false) RaceScoreLoader.reset(fast_processing: true)
puts "=====================> Completed loading #{RaceScore.count} survey responses" puts "=====================> Completed loading #{RaceScore.count} race scores"
Rails.cache.clear Rails.cache.clear
end end
@ -46,10 +49,6 @@ namespace :data do
end end
puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
puts 'Resetting response rates'
ResponseRateLoader.reset
puts "=====================> Completed loading #{ResponseRate.count} response rates"
puts 'Resetting race scores' puts 'Resetting race scores'
RaceScoreLoader.reset(fast_processing: false) RaceScoreLoader.reset(fast_processing: false)
puts "=====================> Completed loading #{RaceScore.count} race scores" puts "=====================> Completed loading #{RaceScore.count} race scores"