diff --git a/app/models/report/beyond_learning_loss_school_response_rates.rb b/app/models/report/beyond_learning_loss_school_response_rates.rb new file mode 100644 index 00000000..3b6a3dd3 --- /dev/null +++ b/app/models/report/beyond_learning_loss_school_response_rates.rb @@ -0,0 +1,156 @@ +module Report + class BeyondLearningLossSchoolResponseRates + def self.create_report(schools: School.all.includes(:district), academic_years: AcademicYear.all, filename: "bll_response_rate_report.csv") + data = to_csv(schools:, academic_years:) + FileUtils.mkdir_p Rails.root.join("tmp", "reports") + filepath = Rails.root.join("tmp", "reports", filename) + write_csv(data:, filepath:) + data + end + + def self.to_csv(schools:, academic_years:) + data = [] + mutex = Thread::Mutex.new + headers = ["District", "School", "School Code", "Academic Year", "Recorded Date Range", "Grades"] + (0..12).each do |grade| + headers << "Number of responses for grade #{grade}" + headers << "Number of questions given to grade #{grade}" + headers << "Average number of responses per question for grade #{grade}" + headers << "Number of students in grade #{grade}" + headers << "Response rate for grade #{grade} (Rate per grade for all questions regardless of subcategory)" + headers << "Response rate for grade #{grade} (Rate per grade grouped by subcategory, then averaged)" + headers << "Number of students that participated in the survey. (grade #{grade})" + headers << "Participation rate for grade #{grade} (Percentage of students who participated in survey)" + end + + subcategories_with_student_survey_items = + ::Subcategory.select do |subcategory| + subcategory.student_survey_items.count.positive? + end + student_survey_items = ::SurveyItem.student_survey_items + early_education_survey_items = ::SurveyItem.early_education_survey_items + + data << headers + pool_size = 2 + jobs = Queue.new + schools.each { |school| jobs << school } + + workers = pool_size.times.map do + Thread.new do + while school = jobs.pop(true) + academic_years.each do |academic_year| + respondents = Respondent.by_school_and_year(school:, academic_year:) + next if respondents.nil? + + begin_date = ::SurveyItemResponse.where(school:, + academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date + end_date = ::SurveyItemResponse.where(school:, + academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date + date_range = "#{begin_date} - #{end_date}" + + all_grades = respondents.enrollment_by_grade.keys + grades = "#{all_grades.first}-#{all_grades.last}" + + mutex.synchronize do + data_row = [school.district.name, + school.name, + school.dese_id, + academic_year.range, + date_range, + grades] + + (0..12).each do |grade| + response_rate_for_grade = + subcategories_with_student_survey_items.map do |subcategory| + calc = ::StudentResponseRateCalculator.new(subcategory:, school:, + academic_year:) + calc.rates_by_grade[grade] + end.remove_blanks.average + + # response_rate_for_grade = response_rate_for_grade.remove_blanks.average + response_rate_for_grade = "" if response_rate_for_grade.nan? + + respondent_count = respondents.for_grade(grade).to_f + + threshold = 10 + quarter_of_grade = if respondent_count.nil? + 10 + else + respondent_count / 4.0 + end + threshold = threshold > quarter_of_grade ? quarter_of_grade : threshold + + survey_items_with_sufficient_responses = ::SurveyItem.where(id: ::SurveyItem.joins("inner join survey_item_responses on survey_item_responses.survey_item_id = survey_items.id") + .student_survey_items + .where("survey_item_responses.school": school, + "survey_item_responses.academic_year": academic_year, + "survey_item_responses.survey_item_id": student_survey_items, + "survey_item_responses.grade": grade) + .group("survey_items.id") + .having("count(*) >= #{threshold}") + .count.keys) + + student_responses = ::SurveyItemResponse.where(school:, academic_year:, + survey_item: survey_items_with_sufficient_responses, grade:) + survey_item_count = student_responses.pluck(:survey_item_id).uniq.count.to_f + average_number_of_responses_per_question = if survey_item_count.positive? + student_responses.count.to_f / survey_item_count + else + 0 + end + + simple_response_rate_calculation = if respondent_count.nil? + 0.to_f + else + average_number_of_responses_per_question.to_f / respondent_count * 100 + + end + simple_response_rate_calculation = "" if simple_response_rate_calculation.nan? + + non_early_ed_items = student_survey_items - early_education_survey_items + non_early_ed_count = ::SurveyItemResponse.where(school:, academic_year:, + survey_item: non_early_ed_items, grade:).select(:response_id).distinct.count || 0 + + early_ed_items = early_education_survey_items + early_ed_count = ::SurveyItemResponse.where(school:, academic_year:, + survey_item: early_ed_items, grade:) + .group(:survey_item) + .select(:response_id) + .distinct + .count + .values.max || 0 + + participation_count = non_early_ed_count + early_ed_count + participation_rate = participation_count / respondent_count * 100 + participation_rate = 0 if participation_rate.nil? || participation_rate.nan? || participation_rate < 5 + + data_row << student_responses.count + data_row << survey_item_count + data_row << average_number_of_responses_per_question + data_row << respondent_count + data_row << simple_response_rate_calculation + data_row << response_rate_for_grade + data_row << participation_count + data_row << participation_rate + end + data << data_row + end + end + end + rescue ThreadError + end + end + + workers.each(&:join) + CSV.generate do |csv| + data.each do |row| + csv << row + end + end + end + + def self.write_csv(data:, filepath:) + File.write(filepath, data) + end + end +end diff --git a/app/models/respondent.rb b/app/models/respondent.rb index 8afd9c03..55ce9249 100644 --- a/app/models/respondent.rb +++ b/app/models/respondent.rb @@ -5,6 +5,8 @@ class Respondent < ApplicationRecord belongs_to :academic_year validates :school, uniqueness: { scope: :academic_year } + GRADE_SYMBOLS = { -1 => :pk, 0 => :k, 1 => :one, 2 => :two, 3 => :three, 4 => :four, 5 => :five, 6 => :six, + 7 => :seven, 8 => :eight, 9 => :nine, 10 => :ten, 11 => :eleven, 12 => :twelve } def enrollment_by_grade @enrollment_by_grade ||= {}.tap do |row| @@ -24,4 +26,8 @@ class Respondent < ApplicationRecord @by_school_and_year[[school, academic_year]] end + + def for_grade(grade) + send(GRADE_SYMBOLS[grade]) + end end diff --git a/app/models/student_response_rate_calculator.rb b/app/models/student_response_rate_calculator.rb index 7ac543d7..c60a1160 100644 --- a/app/models/student_response_rate_calculator.rb +++ b/app/models/student_response_rate_calculator.rb @@ -40,11 +40,15 @@ class StudentResponseRateCalculator < ResponseRateCalculator def survey_items_with_sufficient_responses(grade:) @survey_items_with_sufficient_responses ||= Hash.new do |memo, grade| threshold = 10 - quarter_of_grade = enrollment_by_grade[grade] / 4 + quarter_of_grade = if enrollment_by_grade[grade].nil? + 10 + else + enrollment_by_grade[grade] / 4.0 + end threshold = threshold > quarter_of_grade ? quarter_of_grade : threshold si = SurveyItemResponse.student_survey_items_with_responses_by_grade(school:, - academic_year:) + academic_year:) si = si.reject do |_key, value| value < threshold end diff --git a/app/presenters/student_response_rate_presenter.rb b/app/presenters/student_response_rate_presenter.rb index 12c930ab..2e6fe781 100644 --- a/app/presenters/student_response_rate_presenter.rb +++ b/app/presenters/student_response_rate_presenter.rb @@ -18,14 +18,7 @@ class StudentResponseRatePresenter < ResponseRatePresenter .select(:response_id) .distinct .count - .reduce(0) do |largest, row| - count = row[1] - if count > largest - count - else - largest - end - end + .values.max || 0 non_early_ed_count + early_ed_count end