diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake index 1642bb0a..6c5ed502 100644 --- a/lib/tasks/data.rake +++ b/lib/tasks/data.rake @@ -14,142 +14,9 @@ namespace :data do end puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - puts "Resetting race scores" - RaceScoreLoader.reset(fast_processing: true) - puts "=====================> Completed loading #{RaceScore.count} race scores" - Rails.cache.clear end - desc "seed only lowell" - task seed_only_lowell: :environment do - seeder = Seeder.new rules: [Rule::SeedOnlyLowell] - - seeder.seed_academic_years "2016-17", "2017-18", "2018-19", "2019-20", "2020-21", "2021-22", "2022-23" - seeder.seed_districts_and_schools Rails.root.join("data", "master_list_of_schools_and_districts.csv") - seeder.seed_surveys Rails.root.join("data", "master_list_of_schools_and_districts.csv") - seeder.seed_sqm_framework Rails.root.join("data", "sqm_framework.csv") - seeder.seed_demographics Rails.root.join("data", "demographics.csv") - seeder.seed_enrollment Rails.root.join("data", "enrollment", "enrollment.csv") - seeder.seed_staffing Rails.root.join("data", "staffing", "staffing.csv") - end - - desc "load survey responses for lowell schools" - task load_survey_responses_for_lowell: :environment do - survey_item_response_count = SurveyItemResponse.count - student_count = Student.count - path = "/data/survey_responses/clean/" - Sftp::Directory.open(path:) do |file| - SurveyResponsesDataLoader.from_file(file:) - end - puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" - - Sftp::Directory.open(path:) do |file| - StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools]) - end - puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - - puts "Resetting race scores" - RaceScoreLoader.reset(fast_processing: false) - puts "=====================> Completed loading #{RaceScore.count} race scores" - - Rails.cache.clear - end - - desc "load students for lowell" - task load_students_for_lowell: :environment do - student_count = Student.count - SurveyItemResponse.update_all(student_id: nil) - StudentRace.delete_all - Student.delete_all - - Sftp::Directory.open(path: "/data/survey_responses/clean/") do |file| - StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools]) - end - puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - - puts "Resetting race scores" - RaceScoreLoader.reset(fast_processing: false) - puts "=====================> Completed loading #{RaceScore.count} survey responses" - - Rails.cache.clear - end - - desc "delete non-lowell schools and districts" - task delete_non_lowell: :environment do - schools = School.all.reject { |s| s.district.name == "Lowell" } - ResponseRate.where(school: schools).delete_all - Respondent.where(school: schools).delete_all - Survey.where(school: schools).delete_all - schools.each { |school| school.delete } - districts = District.all.reject { |district| district.name == "Lowell" } - districts.each { |district| district.delete } - end - - task load_survey_responses_21_22: :environment do - Dir.glob(Rails.root.join("data", "survey_responses", "*2021-22*.csv")).each do |filepath| - puts "=====================> Loading data from csv at path: #{filepath}" - SurveyResponsesDataLoader.load_data filepath: - end - puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses" - end - - task load_survey_responses_20_21: :environment do - Dir.glob(Rails.root.join("data", "survey_responses", "*2020-21*.csv")).each do |filepath| - puts "=====================> Loading data from csv at path: #{filepath}" - SurveyResponsesDataLoader.load_data filepath: - end - puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses" - end - - task load_survey_responses_19_20: :environment do - Dir.glob(Rails.root.join("data", "survey_responses", "*2019-20*.csv")).each do |filepath| - puts "=====================> Loading data from csv at path: #{filepath}" - SurveyResponsesDataLoader.load_data filepath: - end - puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses" - end - - task load_survey_responses_18_19: :environment do - Dir.glob(Rails.root.join("data", "survey_responses", "*2018-19*.csv")).each do |filepath| - puts "=====================> Loading data from csv at path: #{filepath}" - SurveyResponsesDataLoader.load_data filepath: - end - puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses" - end - - task load_survey_responses_17_18: :environment do - Dir.glob(Rails.root.join("data", "survey_responses", "*2017-18*.csv")).each do |filepath| - puts "=====================> Loading data from csv at path: #{filepath}" - SurveyResponsesDataLoader.load_data filepath: - end - puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses" - end - - task load_survey_responses_16_17: :environment do - Dir.glob(Rails.root.join("data", "survey_responses", "*2016-17*.csv")).each do |filepath| - puts "=====================> Loading data from csv at path: #{filepath}" - SurveyResponsesDataLoader.load_data filepath: - end - puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses" - end - - desc "reset response rate values" - task reset_response_rates: :environment do - puts "Resetting response rates" - ResponseRateLoader.reset - Rails.cache.clear - puts "=====================> Completed loading #{ResponseRate.count} survey responses" - end - - desc "reset race score calculations" - task reset_race_scores: :environment do - puts "Resetting race scores" - RaceScoreLoader.reset(fast_processing: false) - Rails.cache.clear - puts "=====================> Completed loading #{RaceScore.count} survey responses" - end - desc "load admin_data" task load_admin_data: :environment do original_count = AdminDataValue.count @@ -176,10 +43,6 @@ namespace :data do end puts "=====================> Completed loading #{Student.count} students" - puts "Resetting race scores" - RaceScoreLoader.reset(fast_processing: false) - puts "=====================> Completed loading #{RaceScore.count} survey responses" - Rails.cache.clear end diff --git a/lib/tasks/one_off.rake b/lib/tasks/one_off.rake index f19c8459..d1cce35c 100644 --- a/lib/tasks/one_off.rake +++ b/lib/tasks/one_off.rake @@ -54,18 +54,12 @@ namespace :one_off do end puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" - puts 'Resetting response rates' - ResponseRateLoader.reset(schools: [school], - academic_years: [academic_year]) Dir.glob(Rails.root.join('data', 'survey_responses', '2022-23_stoklosa_student_survey_responses.csv')).each do |file| puts "=====================> Loading student data from csv at path: #{file}" StudentLoader.load_data filepath: file, rules: [Rule::SkipNonLowellSchools] end - puts 'Resetting race scores' - RaceScoreLoader.reset(fast_processing: true, schools: [school], academic_years: [academic_year]) - puts "=====================> Completed recalculating #{ResponseRate.count} response rates" end desc 'load butler results for 2022-23' @@ -76,18 +70,6 @@ namespace :one_off do puts "=====================> Loading data from csv at path: #{filepath}" SurveyResponsesDataLoader.load_data filepath: end - puts 'Resetting response rates' - ResponseRateLoader.reset - puts "=====================> Completed recalculating #{ResponseRate.count} response rates" - end - - desc 'reset race score calculations' - task reset_race_scores_2021: :environment do - puts 'Resetting race scores' - academic_years = [AcademicYear.find_by_range('2021-22')] - RaceScoreLoader.reset(academic_years:, fast_processing: true) - Rails.cache.clear - puts "=====================> Completed loading #{RaceScore.count} race scores" end desc 'list scales that have no survey responses' @@ -146,14 +128,6 @@ namespace :one_off do end puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - puts 'Resetting response rates' - ResponseRateLoader.reset - puts "=====================> Completed loading #{ResponseRate.count} response rates" - - puts 'Resetting race scores' - RaceScoreLoader.reset(fast_processing: false) - puts "=====================> Completed loading #{RaceScore.count} race scores" - Rails.cache.clear end @@ -183,29 +157,9 @@ namespace :one_off do end puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - puts 'Resetting race scores' - RaceScoreLoader.reset(fast_processing: false, academic_years: [AcademicYear.find_by_range('2022-23')], schools:) - puts "=====================> Completed loading #{RaceScore.count} race scores" - Rails.cache.clear end - desc 'set response rates for lee schools to 100' - task set_response_rates_for_lee: :environment do - lee = District.find_by(name: 'Lee Public Schools') - academic_year = AcademicYear.find_by_range('2022-23') - sufficient_response_rate = ResponseRate.where(academic_year:, school: lee.schools).select do |rate| - rate.student_response_rate > 0 - end.map(&:id) - sufficient_response_rate = ResponseRate.where(id: sufficient_response_rate) - sufficient_response_rate.update_all(student_response_rate: 100, meets_student_threshold: true) - - sufficient_response_rate = ResponseRate.where(academic_year:, school: lee.schools).select do |rate| - rate.teacher_response_rate > 0 - end.map(&:id) - sufficient_response_rate = ResponseRate.where(id: sufficient_response_rate) - sufficient_response_rate.update_all(teacher_response_rate: 100, meets_teacher_threshold: true) - end desc "Generate CSV report of teacher survey item responses" task teacher_survey_questions_csv: :environment do headers = ['School ID', 'Academic Year', 'Survey Item', 'Count','Percentage_diff']