diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake index eaab62bb..9a62a5ed 100644 --- a/lib/tasks/data.rake +++ b/lib/tasks/data.rake @@ -22,58 +22,6 @@ namespace :data do Rails.cache.clear end - desc 'seed only lowell' - task seed_only_lowell: :environment do - seeder = Seeder.new rules: [Rule::SeedOnlyLowell] - - seeder.seed_academic_years '2016-17', '2017-18', '2018-19', '2019-20', '2020-21', '2021-22', '2022-23' - seeder.seed_districts_and_schools Rails.root.join('data', 'master_list_of_schools_and_districts.csv') - seeder.seed_enrollment Rails.root.join('data', 'enrollment', 'enrollment.csv') - seeder.seed_staffing Rails.root.join('data', 'staffing', 'staffing.csv') - seeder.seed_sqm_framework Rails.root.join('data', 'sqm_framework.csv') - seeder.seed_demographics Rails.root.join('data', 'demographics.csv') - end - - desc 'load survey responses for lowell schools' - task load_survey_responses_for_lowell: :environment do - survey_item_response_count = SurveyItemResponse.count - student_count = Student.count - path = '/data/survey_responses/clean/' - Sftp::Directory.open(path:) do |file| - SurveyResponsesDataLoader.from_file(file:) - end - puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" - - Sftp::Directory.open(path:) do |file| - StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools]) - end - puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - - puts 'Resetting race scores' - RaceScoreLoader.reset(fast_processing: false) - puts "=====================> Completed loading #{RaceScore.count} race scores" - - Rails.cache.clear - end - - desc 'load students for lowell' - task load_students_for_lowell: :environment do - SurveyItemResponse.update_all(student_id: nil) - StudentRace.delete_all - Student.delete_all - - Sftp::Directory.open(path: '/data/survey_responses/clean/') do |file| - StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools]) - end - puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - - puts 'Resetting race scores' - RaceScoreLoader.reset(fast_processing: false) - puts "=====================> Completed loading #{RaceScore.count} survey responses" - - Rails.cache.clear - end - desc 'seed only lowell' task seed_only_lowell: :environment do seeder = Seeder.new rules: [Rule::SeedOnlyLowell] @@ -101,10 +49,6 @@ namespace :data do end puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - puts 'Resetting response rates' - ResponseRateLoader.reset - puts "=====================> Completed loading #{ResponseRate.count} response rates" - puts 'Resetting race scores' RaceScoreLoader.reset(fast_processing: false) puts "=====================> Completed loading #{RaceScore.count} race scores" @@ -136,7 +80,6 @@ namespace :data do schools = School.all.reject { |s| s.district.name == 'Lowell' } ResponseRate.where(school: schools).delete_all Respondent.where(school: schools).delete_all - Survey.where(school: schools).delete_all schools.each { |school| school.delete } districts = District.all.reject { |district| district.name == 'Lowell' } districts.each { |district| district.delete } diff --git a/lib/tasks/one_off.rake b/lib/tasks/one_off.rake index deaa370e..cb4ac63b 100644 --- a/lib/tasks/one_off.rake +++ b/lib/tasks/one_off.rake @@ -146,10 +146,6 @@ namespace :one_off do end puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - puts 'Resetting response rates' - ResponseRateLoader.reset - puts "=====================> Completed loading #{ResponseRate.count} response rates" - puts 'Resetting race scores' RaceScoreLoader.reset(fast_processing: false) puts "=====================> Completed loading #{RaceScore.count} race scores" @@ -182,10 +178,6 @@ namespace :one_off do end puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - puts 'Resetting response rates' - ResponseRateLoader.reset academic_years: [AcademicYear.find_by_range('2022-23')] - puts "=====================> Completed loading #{ResponseRate.count} response rates" - puts 'Resetting race scores' RaceScoreLoader.reset(fast_processing: false, academic_years: [AcademicYear.find_by_range('2022-23')]) puts "=====================> Completed loading #{RaceScore.count} race scores" @@ -217,30 +209,30 @@ namespace :one_off do sum+=c.values.first nof_si+=1 end - end - - + end + + avg = sum.to_f/ nof_si counts.each do |key, value| if key[0] == sc.id && key[1] == ay.id count = value.values.first percentage_diff = ((count-avg) / avg) * 100 - counts[key] = { count: count, percentage_diff: percentage_diff } + counts[key] = { count: count, percentage_diff: percentage_diff } end end end end - + counts.each do |key, value| output_rows << [School.where(id:key[0]).pluck(:name) , AcademicYear.where(id:key[1]).pluck(:range) , SurveyItem.where(id:key[2]).pluck(:survey_item_id), SurveyItem.where(id:key[2]).pluck(:prompt), value[:count], value[:percentage_diff]] end - + file = File.new('teacher_survey_questions.csv', 'w') CSV.open(file, 'w', write_headers: true, headers: headers) do |csv| output_rows.each do |row| csv << row end end - + file.close puts "CSV report of teacher survey item responses with removed stray responses.csv" end @@ -258,7 +250,7 @@ namespace :one_off do if Respondent.where(school: sc,academic_year:ay).any? grades_count=Respondent.where(school:sc,academic_year:ay).first.counts_by_grade grades= grades_count.keys - else + else grades=[] end sum=0 @@ -281,31 +273,31 @@ namespace :one_off do sum+=c.values.first nof_si+=1 end - end end + end + - avg = sum.to_f/ nof_si counts.each do |key, value| if key[0] == sc.id && key[1] == ay.id count = value.values.first percentage_diff = ((count-avg) / avg) * 100 - counts[key] = { count: count, percentage_diff: percentage_diff } + counts[key] = { count: count, percentage_diff: percentage_diff } end end - end end - + end end + counts.each do |key, value| output_rows << [School.where(id:key[0]).pluck(:name) , AcademicYear.where(id:key[1]).pluck(:range) , SurveyItem.where(id:key[2]).pluck(:survey_item_id), SurveyItem.where(id:key[2]).pluck(:prompt), value[:count], value[:percentage_diff]] end - + file = File.new('regular_survey_questions.csv', 'w') CSV.open(file, 'w', write_headers: true, headers: headers) do |csv| output_rows.each do |row| csv << row end end - + file.close puts "CSV report of regular survey item responses with removed stray responses.csv" end @@ -323,7 +315,7 @@ namespace :one_off do if Respondent.where(school: sc,academic_year:ay).any? grades_count=Respondent.where(school:sc,academic_year:ay).first.counts_by_grade grades= grades_count.keys - else + else grades=[] end sum=0 @@ -346,31 +338,31 @@ namespace :one_off do sum+=c.values.first nof_si+=1 end - end end + end + - avg = sum.to_f/ nof_si counts.each do |key, value| if key[0] == sc.id && key[1] == ay.id count = value.values.first percentage_diff = ((count-avg) / avg) * 100 - counts[key] = { count: count, percentage_diff: percentage_diff } + counts[key] = { count: count, percentage_diff: percentage_diff } end end - end end - + end end + counts.each do |key, value| output_rows << [School.where(id:key[0]).pluck(:name) , AcademicYear.where(id:key[1]).pluck(:range) , SurveyItem.where(id:key[2]).pluck(:survey_item_id), SurveyItem.where(id:key[2]).pluck(:prompt), value[:count], value[:percentage_diff]] end - + file = File.new('short_survey_questions.csv', 'w') CSV.open(file, 'w', write_headers: true, headers: headers) do |csv| output_rows.each do |row| csv << row end end - + file.close puts "CSV report of short survey item responses with removed stray responses.csv" end @@ -389,7 +381,7 @@ namespace :one_off do if Respondent.where(school: sc,academic_year:ay).any? grades_count=Respondent.where(school:sc,academic_year:ay).first.counts_by_grade grades= grades_count.keys - else + else grades=[] end sum=0 @@ -412,31 +404,31 @@ namespace :one_off do sum+=c.values.first nof_si+=1 end - end end + end + - avg = sum.to_f/ nof_si counts.each do |key, value| if key[0] == sc.id && key[1] == ay.id count = value.values.first percentage_diff = ((count-avg) / avg) * 100 - counts[key] = { count: count, percentage_diff: percentage_diff } + counts[key] = { count: count, percentage_diff: percentage_diff } end end end end - + counts.each do |key, value| output_rows << [School.where(id:key[0]).pluck(:name) , AcademicYear.where(id:key[1]).pluck(:range) , SurveyItem.where(id:key[2]).pluck(:survey_item_id), SurveyItem.where(id:key[2]).pluck(:prompt), value[:count], value[:percentage_diff]] end - + file = File.new('early_education_surveys_questions.csv', 'w') CSV.open(file, 'w', write_headers: true, headers: headers) do |csv| output_rows.each do |row| csv << row end end - + file.close puts "CSV report of early_education_surveys items with removed stray responses.csv" end @@ -444,19 +436,19 @@ namespace :one_off do desc "Generate CSV report of survey item responses" task stray_responses: :environment do - + headers = ['School ID', 'Academic Year', 'Survey Item', 'Count','SurveyItemResponse ids'] - + output_rows = [] sir_ids=[] - + School.all.each do |sc| AcademicYear.all.each do |ay| SurveyItem.all.each do |si| count = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).count - sir_ids= SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).pluck(:response_id) + sir_ids= SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).pluck(:response_id) if count > 0 && count < 10 @@ -466,7 +458,7 @@ namespace :one_off do end end - + file = File.new('stray_responses.csv', 'w') CSV.open(file, 'w', write_headers: true, headers: headers) do |csv| @@ -475,7 +467,7 @@ namespace :one_off do end end - + file.close puts "CSV report of survey item responses created in stray_responses.csv"