diff --git a/app/models/report/exports.rb b/app/models/report/exports.rb new file mode 100644 index 00000000..78997b3a --- /dev/null +++ b/app/models/report/exports.rb @@ -0,0 +1,97 @@ +module Report + class Exports + def self.create + academic_years = ::AcademicYear.all + districts = ::District.all + use_student_survey_items = ::SurveyItem.student_survey_items.map(&:id) + schools = ::School.all.includes(:district) + + reports = { + "Subcategory by School & District" => lambda { |schools, academic_years| + Report::Subcategory.to_csv(schools:, academic_years:, subcategories: ::Subcategory.all) + }, + "Measure by District only" => lambda { |schools, academic_years| + Report::MeasureSummary.to_csv(schools:, academic_years:, measures: ::Measure.all) + }, + "Measure by School & District" => lambda { |schools, academic_years| + Report::Measure.to_csv(schools:, academic_years:, + measures: ::Measure.all) + }, + "Survey Item by Item" => lambda { |schools, academic_years| + Report::SurveyItemByItem.to_csv(schools:, academic_years:) + }, + "Survey Item by Grade" => lambda { |schools, academic_years| + Report::SurveyItemByGrade.to_csv(schools:, academic_years:, + use_student_survey_items:) + }, + "Survey Entries by Measure" => lambda { |schools, academic_years| + Report::SurveyItemResponse.to_csv(schools:, academic_years:, use_student_survey_items:) + } + } + + reports.each do |report_name, runner| + report_name = report_name.gsub(" ", "_").downcase + + academic_years.each do |academic_year| + districts.each do |district| + # Each year + threads = [] + threads << Thread.new do + response_count = ::SurveyItemResponse.where(school: district.schools, academic_year: academic_year).count + if response_count > 100 + schools = district.schools + report_name = report_name.gsub(" ", "_").downcase + + ::FileUtils.mkdir_p ::Rails.root.join("tmp", "exports", report_name, academic_year.range, district.name) + + filename = "#{report_name}_#{academic_year.range}_#{district.name}.csv" + filepath = Rails.root.join("tmp", "exports", report_name, academic_year.range, district.name, filename) + csv = runner.call(schools, [academic_year]) + + File.write(filepath, csv) + end + end + threads.each(&:join) + GC.start + end + # All districts + response_count = ::SurveyItemResponse.where(school: districts.flat_map(&:schools), academic_year: academic_year).count + next unless response_count > 100 + + threads = [] + threads << Thread.new do + ::FileUtils.mkdir_p ::Rails.root.join("tmp", "exports", report_name, academic_year.range, "all_districts") + + filename = "#{report_name}_all_districts.csv" + filepath = Rails.root.join("tmp", "exports", report_name, academic_year.range, "all_districts", filename) + csv = runner.call(districts.flat_map(&:schools), [academic_year]) + + File.write(filepath, csv) + end + threads.each(&:join) + GC.start + end + + districts.each do |district| + # # All years for each district + threads = [] + threads << Thread.new do + response_count = ::SurveyItemResponse.where(school: district.schools, academic_year: academic_years).count + next unless response_count > 100 + + ::FileUtils.mkdir_p ::Rails.root.join("tmp", "exports", report_name, "all_years", district.name) + + filename = "#{report_name}_all_years_#{district.name}.csv" + filepath = Rails.root.join("tmp", "exports", report_name, "all_years", district.name, filename) + csv = runner.call(district.schools, academic_years) + + File.write(filepath, csv) + GC.start + end + threads.each(&:join) + GC.start + end + end + end + end +end diff --git a/app/models/report/measure_summary.rb b/app/models/report/measure_summary.rb index 9c1520b9..0fd691ae 100644 --- a/app/models/report/measure_summary.rb +++ b/app/models/report/measure_summary.rb @@ -22,32 +22,32 @@ module Report while measure = jobs.pop(true) all_grades = Respondent.grades_that_responded_to_survey(academic_year: academic_years, school: schools) grades = "#{all_grades.first}-#{all_grades.last}" - district = schools.first.district academic_years.each do |academic_year| - begin_date = ::SurveyItemResponse.where(school: schools, - academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date - end_date = ::SurveyItemResponse.where(school: schools, - academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date - date_range = "#{begin_date} - #{end_date}" - - row = [measure, district, academic_year] - - mutex.synchronize do - data << [measure.name, - measure.measure_id, - district.name, - academic_year.range, - date_range, - grades, - student_score(row:), - student_zone(row:), - teacher_score(row:), - teacher_zone(row:), - admin_score(row:), - admin_zone(row:), - all_data_score(row:), - all_data_zone(row:)] + schools.flat_map(&:district).uniq.each do |district| + selected_schools = district.schools + begin_date = ::SurveyItemResponse.where(school: selected_schools, + academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date + end_date = ::SurveyItemResponse.where(school: selected_schools, + academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date + date_range = "#{begin_date} - #{end_date}" + + mutex.synchronize do + data << [measure.name, + measure.measure_id, + district.name, + academic_year.range, + date_range, + grades, + student_score(measure:, schools: selected_schools, academic_year:), + student_zone(measure:, schools: selected_schools, academic_year:), + teacher_score(measure:, schools: selected_schools, academic_year:), + teacher_zone(measure:, schools: selected_schools, academic_year:), + admin_score(measure:, schools: selected_schools, academic_year:), + admin_zone(measure:, schools: selected_schools, academic_year:), + all_data_score(measure:, schools: selected_schools, academic_year:), + all_data_zone(measure:, schools: selected_schools, academic_year:)] + end end end end @@ -68,18 +68,18 @@ module Report File.write(filepath, csv) end - def self.all_data_score(row:) - row in [ measure, district, academic_year] - score = district.schools.map do |school| - score = measure.score(school:, academic_year:).average - end.remove_blanks.average - score || "N/A" + def self.all_data_score(measure:, schools:, academic_year:) + @all_data_score ||= Hash.new do |memo, (measure, schools, academic_year)| + score = schools.map do |school| + score = measure.score(school:, academic_year:).average + end.remove_blanks.average + memo[[measure, schools, academic_year]] = score || "N/A" + end + @all_data_score[[measure, schools, academic_year]] end - def self.all_data_zone(row:) - row in [ measure, district, academic_year] - - average = all_data_score(row:) + def self.all_data_zone(measure:, schools:, academic_year:) + average = all_data_score(measure:, schools:, academic_year:) score = Score.new(average:, meets_teacher_threshold: true, meets_student_threshold: true, meets_admin_data_threshold: true) student_zone = measure.zone_for_score(score:).type.to_s @@ -87,17 +87,18 @@ module Report student_zone || "N/A" end - def self.student_score(row:) - row in [ measure, district, academic_year] - student_score = district.schools.map do |school| - student_score = measure.student_score(school:, academic_year:).average - end.remove_blanks.average - student_score || "N/A" + def self.student_score(measure:, schools:, academic_year:) + @student_score ||= Hash.new do |memo, (measure, schools, academic_year)| + student_score = schools.map do |school| + student_score = measure.student_score(school:, academic_year:).average + end.remove_blanks.average + memo[[measure, schools, academic_year]] = student_score || "N/A" + end + @student_score[[measure, schools, academic_year]] end - def self.student_zone(row:) - row in [ measure, district, academic_year] - average = student_score(row:) + def self.student_zone(measure:, schools:, academic_year:) + average = student_score(measure:, schools:, academic_year:) score = Score.new(average:, meets_teacher_threshold: true, meets_student_threshold: true, meets_admin_data_threshold: true) student_zone = measure.zone_for_score(score:).type.to_s @@ -105,18 +106,19 @@ module Report student_zone || "N/A" end - def self.teacher_score(row:) - row in [ measure, district, academic_year] - teacher_score = district.schools.map do |school| - measure.teacher_score(school:, academic_year:).average - end.remove_blanks.average + def self.teacher_score(measure:, schools:, academic_year:) + @teacher_score ||= Hash.new do |memo, (measure, schools, academic_year)| + teacher_score = schools.map do |school| + measure.teacher_score(school:, academic_year:).average + end.remove_blanks.average - teacher_score || "N/A" + memo[[measure, schools, academic_year]] = teacher_score || "N/A" + end + @teacher_score[[measure, schools, academic_year]] end - def self.teacher_zone(row:) - row in [ measure, district, academic_year] - average = teacher_score(row:) + def self.teacher_zone(measure:, schools:, academic_year:) + average = teacher_score(measure:, schools:, academic_year:) score = Score.new(average:, meets_teacher_threshold: true, meets_student_threshold: true, meets_admin_data_threshold: true) teacher_zone = measure.zone_for_score(score:).type.to_s @@ -124,19 +126,20 @@ module Report teacher_zone || "N/A" end - def self.admin_score(row:) - row in [ measure, district, academic_year] - admin_score = district.schools.map do |school| - measure.admin_score(school:, academic_year:).average - end.remove_blanks.average + def self.admin_score(measure:, schools:, academic_year:) + @admin_score ||= Hash.new do |memo, (measure, schools, academic_year)| + admin_score = schools.map do |school| + measure.admin_score(school:, academic_year:).average + end.remove_blanks.average - admin_score = "N/A" unless admin_score.present? && admin_score >= 0 - admin_score + admin_score = "N/A" unless admin_score.present? && admin_score >= 0 + memo[[measure, schools, academic_year]] = admin_score + end + @admin_score[[measure, schools, academic_year]] end - def self.admin_zone(row:) - row in [ measure, district, academic_year] - average = admin_score(row:) + def self.admin_zone(measure:, schools:, academic_year:) + average = admin_score(measure:, schools:, academic_year:) score = Score.new(average:, meets_teacher_threshold: true, meets_student_threshold: true, meets_admin_data_threshold: true) diff --git a/app/models/report/subcategory.rb b/app/models/report/subcategory.rb index ac3fdc47..af301817 100644 --- a/app/models/report/subcategory.rb +++ b/app/models/report/subcategory.rb @@ -1,14 +1,14 @@ module Report class Subcategory def self.create_report(schools: School.all.includes(:district), academic_years: AcademicYear.all, subcategories: ::Subcategory.all, filename: "subcategories.csv") - csv = to_csv(schools:, academic_years:, subcategories:, filename:) + csv = to_csv(schools:, academic_years:, subcategories:) FileUtils.mkdir_p Rails.root.join("tmp", "reports") filepath = Rails.root.join("tmp", "reports", filename) write_csv(csv:, filepath:) csv end - def self.to_csv(schools: School.all.includes(:district), academic_years: AcademicYear.all, subcategories: ::Subcategory.all, filename: "subcategories.csv") + def self.to_csv(schools: School.all.includes(:district), academic_years: AcademicYear.all, subcategories: ::Subcategory.all) data = [] mutex = Thread::Mutex.new data << ["District", "School", "School Code", "Academic Year", "Recorded Date Range", "Grades", "Subcategory", "Student Score", "Student Zone", "Teacher Score", diff --git a/app/models/report/survey_item_by_grade.rb b/app/models/report/survey_item_by_grade.rb index c8136d65..27fdbab1 100644 --- a/app/models/report/survey_item_by_grade.rb +++ b/app/models/report/survey_item_by_grade.rb @@ -9,11 +9,12 @@ module Report csv end - def self.to_csv(schools:, academic_years:, use_student_survey_items:) + def self.to_csv(schools:, academic_years:, use_student_survey_items: ::SurveyItem.student_survey_items.pluck(:id)) # get list of survey items with sufficient responses survey_items = Set.new # also get a map of grade->survey_id sufficient_survey_items = {} + survey_items_by_id = ::SurveyItem.by_id_includes_all grades = Respondent.grades_that_responded_to_survey(academic_year: academic_years, school: schools) @@ -41,9 +42,9 @@ module Report "Grade", "Academic Year" ] - survey_items = survey_items.sort_by { |id| ::SurveyItem.find(id).prompt } + survey_items = survey_items.sort_by { |id| survey_items_by_id[id].prompt } survey_items.each do |survey_item_id| - headers << ::SurveyItem.find_by_id(survey_item_id).prompt + headers << survey_items_by_id[survey_item_id].prompt end data = [] data << headers @@ -65,7 +66,8 @@ module Report end row << academic_year.range survey_items.each do |survey_item_id| - survey_item = ::SurveyItem.find_by_id survey_item_id + survey_item = survey_items_by_id[survey_item_id] + byebug if survey_item.nil? if sufficient_survey_items[grade].include? survey_item_id row.append("#{survey_item.survey_item_responses.where(school:, academic_year:, grade:).average(:likert_score).to_f.round(2)}") @@ -84,11 +86,10 @@ module Report row.append("All School") row.append(academic_year.range) survey_items.each do |survey_item_id| - survey_item = ::SurveyItem.find_by_id survey_item_id + survey_item = survey_items_by_id[survey_item_id] + byebug if survey_item.nil? # filter out response rate at subcategory level <24.5% for school average - scale = Scale.find_by_id(survey_item.scale_id) - measure = ::Measure.find_by_id(scale.measure_id) - subcategory = ::Subcategory.find_by_id(measure.subcategory_id) + subcategory = survey_item.scale.measure.subcategory if ::StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).meets_student_threshold? row.append("#{survey_item.survey_item_responses.where( # We allow the nil (unknown) grades in the school survey item average diff --git a/app/models/report/survey_item_by_item.rb b/app/models/report/survey_item_by_item.rb index d5b640f3..1eb789bd 100644 --- a/app/models/report/survey_item_by_item.rb +++ b/app/models/report/survey_item_by_item.rb @@ -28,124 +28,100 @@ module Report # we do not display the grade avg if there are < 10 responses survey_ids_to_grades[key[1]].add(key[0]) if count >= 10 end - headers = [ - "Survey Item", - "Category", - "Subcategory", - "Survey Measure", - "Survey Scale", - "Survey Population", - "School Name", - "Academic Year" - ] grades = Respondent.grades_that_responded_to_survey(academic_year: academic_years, school: schools) - - grades.each do |value| - if value == 0 - headers.append("Kindergarten") - else - headers.append("Grade #{value}") - end - end - headers.append("All School") - headers.append("Teacher") # Then, add the headers to data data = [] - data << headers + data << generate_headers(schools:, academic_years:, grades:) survey_items_by_id = ::SurveyItem.by_id_includes_all - mutex = Thread::Mutex.new - pool_size = 2 - jobs = Queue.new - schools.each { |school| jobs << school } + # mutex = Thread::Mutex.new + # pool_size = 2 + # jobs = Queue.new + # schools.each { |school| jobs << school } academic_years.each do |academic_year| - workers = pool_size.times.map do - Thread.new do - while school = jobs.pop(true) - # for each survey item id - survey_ids_to_grades.sort_by do |id, _value| - survey_items_by_id[id].prompt - end.each do |id, school_grades| - school_grades = school_grades.reject(&:nil?) - row = [] - survey_item = survey_items_by_id[id] - row.concat(survey_item_info(survey_item:)) # fills prompt + categories - row.append("Students") - row.append(school.name) - row.append(academic_year.range) - - # add padding before grade average section - starting_grade = school_grades.sort.first - starting_grade = grades.index(starting_grade) || 0 - padding = Array.new(starting_grade) { "" } - row.concat(padding) - - school_grades.sort.each do |grade| - next if grade == -1 - - if school_grades.include?(grade) - # we already know grade has sufficient responses - score = survey_item.survey_item_responses.where(school:, academic_year:, - grade:).average(:likert_score).to_f.round(2) - score = "" if score.zero? - row.append("#{score}") - else - row.append("") - end - end - - # add padding after the grade average section - ending_grade = school_grades.sort.last - ending_grade = grades.index(ending_grade) + 1 || 0 - padding = Array.new(grades.length - ending_grade) { "" } - row.concat(padding) - - # filter out response rate at subcategory level <24.5% for school average - if response_rate(subcategory: survey_item.subcategory, school:, - academic_year:).meets_student_threshold? - all_student_score = survey_item.survey_item_responses.where( - # We allow the nil (unknown) grades in the school survey item average - # also filter less than 10 responses in the whole school - "school_id = ? and academic_year_id = ? and (grade IS NULL or grade IN (?))", school.id, academic_year.id, school.grades(academic_year:) - ).group("survey_item_id").having("count(*) >= 10").average(:likert_score).values[0].to_f.round(2) - - all_student_score = "" if all_student_score.zero? - row.append("#{all_student_score}") - else - row.append("") - end - data << row - end - # Next up is teacher data - # each key is a survey item id - ::SurveyItemResponse.teacher_survey_items_with_sufficient_responses(school:, - academic_year:).keys.sort_by do |id| - survey_items_by_id[id].prompt - end.each do |key| - row = [] - survey_item = survey_items_by_id[key] - row.concat(survey_item_info(survey_item:)) - row.append("Teacher") - row.append(school.name) - row.append(academic_year.range) - # we need to add padding to skip the grades columns and the 'all school' column - padding = Array.new(grades.length + 1) { "" } - row.concat(padding) - # we already know that the survey item we are looking at has sufficient responses - all_teacher_score = survey_item.survey_item_responses.where(school:, - academic_year:).average(:likert_score).to_f.round(2) - all_teacher_score = "" if all_teacher_score.zero? - row.append("#{all_teacher_score}") - data << row + next unless ::SurveyItemResponse.where(school: schools, academic_year: academic_year, survey_item_id: use_student_survey_items).count > 100 + + schools.each do |school| + # for each survey item id + survey_ids_to_grades.sort_by do |id, _value| + survey_items_by_id[id].prompt + end.each do |id, school_grades| + school_grades = school_grades.reject(&:nil?) + row = [] + survey_item = survey_items_by_id[id] + row.concat(survey_item_info(survey_item:)) # fills prompt + categories + row.append("Students") + row.append(school.name) + row.append(academic_year.range) + + # add padding before grade average section + starting_grade = school_grades.sort.first + starting_grade = grades.index(starting_grade) || 0 + padding = Array.new(starting_grade) { "" } + row.concat(padding) + + school_grades.sort.each do |grade| + next if grade == -1 + + if school_grades.include?(grade) + # we already know grade has sufficient responses + score = survey_item.survey_item_responses.where(school:, academic_year:, + grade:).average(:likert_score).to_f.round(2) + score = "" if score.zero? + row.append("#{score}") + else + row.append("") end end - rescue ThreadError + + # add padding after the grade average section + ending_grade = school_grades.sort.last + ending_grade = grades.index(ending_grade) + 1 || 0 + padding = Array.new(grades.length - ending_grade) { "" } + row.concat(padding) + + # filter out response rate at subcategory level <24.5% for school average + if response_rate(subcategory: survey_item.subcategory, school:, + academic_year:).meets_student_threshold? + all_student_score = survey_item.survey_item_responses.where( + # We allow the nil (unknown) grades in the school survey item average + # also filter less than 10 responses in the whole school + "school_id = ? and academic_year_id = ? and (grade IS NULL or grade IN (?))", school.id, academic_year.id, school.grades(academic_year:) + ).group("survey_item_id").having("count(*) >= 10").average(:likert_score).values[0].to_f.round(2) + + all_student_score = "" if all_student_score.zero? + row.append("#{all_student_score}") + else + row.append("") + end + data << row + end + # Next up is teacher data + # each key is a survey item id + ::SurveyItemResponse.teacher_survey_items_with_sufficient_responses(school:, + academic_year:).keys.sort_by do |id| + survey_items_by_id[id].prompt + end.each do |key| + row = [] + survey_item = survey_items_by_id[key] + row.concat(survey_item_info(survey_item:)) + row.append("Teacher") + row.append(school.name) + row.append(academic_year.range) + # we need to add padding to skip the grades columns and the 'all school' column + padding = Array.new(grades.length + 1) { "" } + row.concat(padding) + # we already know that the survey item we are looking at has sufficient responses + all_teacher_score = survey_item.survey_item_responses.where(school:, + academic_year:).average(:likert_score).to_f.round(2) + all_teacher_score = "" if all_teacher_score.zero? + row.append("#{all_teacher_score}") + data << row end end - workers.each(&:join) end CSV.generate do |csv| @@ -175,5 +151,29 @@ module Report survey_item.measure.name, survey_item.scale.scale_id] end + + def self.generate_headers(schools:, academic_years:, grades:) + headers = [ + "Survey Item", + "Category", + "Subcategory", + "Survey Measure", + "Survey Scale", + "Survey Population", + "School Name", + "Academic Year" + ] + + grades.each do |value| + if value == 0 + headers.append("Kindergarten") + else + headers.append("Grade #{value}") + end + end + headers.append("All School") + headers.append("Teacher") + headers + end end end diff --git a/app/models/report/survey_item_response.rb b/app/models/report/survey_item_response.rb index 0665109a..b953e365 100644 --- a/app/models/report/survey_item_response.rb +++ b/app/models/report/survey_item_response.rb @@ -9,7 +9,7 @@ module Report data end - def self.to_csv(schools:, academic_years:, use_student_survey_items:) + def self.to_csv(schools:, academic_years:, use_student_survey_items: ::SurveyItem.student_survey_items.map(&:id)) data = [] data << ["Response ID", "Race", "Gender", "Grade", "School ID", "District", "Academic Year", "Student Physical Safety", "Student Emotional Safety", "Student Sense of Belonging", "Student-Teacher Relationships", "Valuing of Learning", "Academic Challenge", "Content Specialists & Support Staff", "Cultural Responsiveness", "Engagement In School", "Appreciation For Diversity", "Civic Participation", "Perseverance & Determination", "Growth Mindset", "Participation In Creative & Performing Arts", "Valuing Creative & Performing Arts", "Social & Emotional Health"] diff --git a/lib/tasks/report.rake b/lib/tasks/report.rake index 6723fa82..b27ed687 100644 --- a/lib/tasks/report.rake +++ b/lib/tasks/report.rake @@ -194,4 +194,12 @@ namespace :report do Report::SurveyItemResponse.create(schools: district.schools, academic_years:, filename:) end end + + # Usage example + # bundle exec rake "report:exports:create" + namespace :exports do + task :create, %i[district academic_year] => :environment do |_, _args| + Report::Exports.create + end + end end