diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake index 991c297f..c8447718 100644 --- a/lib/tasks/data.rake +++ b/lib/tasks/data.rake @@ -3,34 +3,27 @@ require 'csv' namespace :data do desc 'load survey responses' task load_survey_responses: :environment do - survey_item_response_count = SurveyItemResponse.count - student_count = Student.count - - Dir.glob(Rails.root.join('data', 'survey_responses', '*.csv')).each do |filepath| - puts "=====================> Loading data from csv at path: #{filepath}" - SurveyResponsesDataLoader.load_data filepath: + survey_item_response_count = surveyitemresponse.count + student_count = student.count + sftp.directory.open(path: '/data/survey_responses/clean/') do |file| + surveyresponsesdataloader.from_file(file:) end - puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" - - SurveyItemResponse.update_all(student_id: nil) - StudentRace.delete_all - Student.delete_all + puts "=====================> completed loading #{surveyitemresponse.count - survey_item_response_count} survey responses. #{surveyitemresponse.count} total responses in the database" - Dir.glob(Rails.root.join('data', 'survey_responses', '*student*.csv')).each do |file| - puts "=====================> Loading student data from csv at path: #{file}" - StudentLoader.load_data filepath: file + sftp.directory.open(path: '/data/survey_responses/clean/') do |file| + studentloader.from_file(file:, rules: [rule.skipnonlowellschools]) end - puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" + puts "=====================> completed loading #{student.count - student_count} students. #{student.count} total students" - puts 'Resetting response rates' - ResponseRateLoader.reset - puts "=====================> Completed loading #{ResponseRate.count} survey responses" + puts 'resetting response rates' + responserateloader.reset + puts "=====================> completed loading #{responserate.count} response rates" - puts 'Resetting race scores' - RaceScoreLoader.reset(fast_processing: false) - puts "=====================> Completed loading #{RaceScore.count} survey responses" + puts 'resetting race scores' + racescoreloader.reset(fast_processing: false) + puts "=====================> completed loading #{racescore.count} race scores" - Rails.cache.clear + rails.cache.clear end desc 'seed only lowell' @@ -47,27 +40,27 @@ namespace :data do desc 'load survey responses for lowell schools' task load_survey_responses_for_lowell: :environment do - survey_item_response_count = SurveyItemResponse.count - student_count = Student.count - Sftp::Directory.open(path: '/data/survey_responses/clean/') do |file| - SurveyResponsesDataLoader.from_file(file:) + survey_item_response_count = surveyitemresponse.count + student_count = student.count + sftp.directory.open(path: '/data/survey_responses/clean/') do |file| + surveyresponsesdataloader.from_file(file:) end - puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" + puts "=====================> completed loading #{surveyitemresponse.count - survey_item_response_count} survey responses. #{surveyitemresponse.count} total responses in the database" - Sftp::Directory.open(path: '/data/survey_responses/clean/') do |file| - StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools]) + sftp.directory.open(path: '/data/survey_responses/clean/') do |file| + studentloader.from_file(file:, rules: [rule.skipnonlowellschools]) end - puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" + puts "=====================> completed loading #{student.count - student_count} students. #{student.count} total students" - puts 'Resetting response rates' - ResponseRateLoader.reset - puts "=====================> Completed loading #{ResponseRate.count} response rates" + puts 'resetting response rates' + responserateloader.reset + puts "=====================> completed loading #{responserate.count} response rates" - puts 'Resetting race scores' - RaceScoreLoader.reset(fast_processing: false) - puts "=====================> Completed loading #{RaceScore.count} race scores" + puts 'resetting race scores' + racescoreloader.reset(fast_processing: false) + puts "=====================> completed loading #{racescore.count} race scores" - Rails.cache.clear + rails.cache.clear end desc 'load students for lowell' diff --git a/lib/tasks/one_off.rake b/lib/tasks/one_off.rake index 8e98e6cc..c875afc4 100644 --- a/lib/tasks/one_off.rake +++ b/lib/tasks/one_off.rake @@ -45,7 +45,7 @@ namespace :one_off do filepath = Rails.root.join('data', 'survey_responses', '2021-22_revere_somerville_wareham_student_survey_responses.csv') puts "=====================> Loading data from csv at path: #{filepath}" - SurveyResponsesDataLoader.load_data filepath: filepath + SurveyResponsesDataLoader.load_data(filepath:) puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses" puts 'Resetting response rates' ResponseRateLoader.reset @@ -137,4 +137,12 @@ namespace :one_off do end puts values end + + desc 'delete survey item responses for 2016-2017' + task delete_survey_item_responses_2016_17: :environment do + academic_years = AcademicYear.where(range: %w[2016-17 2017-18]) + response_count = SurveyItemResponse.where(academic_year: academic_years).count + SurveyItemResponse.where(academic_year: academic_years).delete_all + puts "Deleted #{response_count} survey item responses" + end end