Add an exports page that allows a user to select which reports they want

to run

app/models/report/beyond_learning_loss.rb

@@ -1,6 +1,14 @@
 module Report
   class BeyondLearningLoss
     def self.create_report(schools: School.all.includes(:district), academic_years: AcademicYear.all, scales: ::Scale.all, filename: "bll_report.csv")
+      data = to_csv(schools:, academic_years:, scales:)
+      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
+      filepath = Rails.root.join("tmp", "reports", filename)
+      write_csv(data:, filepath:)
+      data
+    end
+
+    def self.to_csv(schools:, academic_years:, scales:)
       data = []
       mutex = Thread::Mutex.new
       data << ["District", "School", "School Code", "Academic Year", "Recorded Date Range", "Grades", "Measure", "Scale",
@@ -29,10 +37,10 @@ module Report
                           scale.score(school:, academic_year:)
                         end
 
-                begin_date = SurveyItemResponse.where(school:,
-                                                      academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date
-                end_date = SurveyItemResponse.where(school:,
-                                                    academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date
+                begin_date = ::SurveyItemResponse.where(school:,
+                                                        academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date
+                end_date = ::SurveyItemResponse.where(school:,
+                                                      academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date
                 date_range = "#{begin_date} - #{end_date}"
 
                 row = [response_rate, scale, school, academic_year]
@@ -58,18 +66,14 @@ module Report
       end
 
       workers.each(&:join)
-      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
-      filepath = Rails.root.join("tmp", "reports", filename)
-      write_csv(data:, filepath:)
-      data
-    end
-
-    def self.write_csv(data:, filepath:)
-      csv = CSV.generate do |csv|
+      CSV.generate do |csv|
         data.each do |row|
           csv << row
         end
       end
+    end
+
+    def self.write_csv(data:, filepath:)
       File.write(filepath, csv)
     end
   end

app/models/report/measure.rb

@@ -1,6 +1,14 @@
 module Report
   class Measure
     def self.create_report(schools: School.all.includes(:district), academic_years: AcademicYear.all, measures: ::Measure.all, filename: "measure_report.csv")
+      data = to_csv(schools, academic_years:, measures:)
+      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
+      filepath = Rails.root.join("tmp", "reports", filename)
+      write_csv(data:, filepath:)
+      data
+    end
+
+    def self.to_csv(schools:, academic_years:, measures:)
       data = []
       mutex = Thread::Mutex.new
       data << ["Measure Name", "Measure ID", "District", "School", "School Code", "Academic Year", "Recorded Date Range", "Grades", "Student Score", "Student Zone", "Teacher Score",
@@ -23,10 +31,10 @@ module Report
                 score = measure.score(school:, academic_year:)
                 zone = measure.zone(school:, academic_year:).type.to_s.capitalize
 
-                begin_date = SurveyItemResponse.where(school:,
-                                                      academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date
-                end_date = SurveyItemResponse.where(school:,
-                                                    academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date
+                begin_date = ::SurveyItemResponse.where(school:,
+                                                        academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date
+                end_date = ::SurveyItemResponse.where(school:,
+                                                      academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date
                 date_range = "#{begin_date} - #{end_date}"
 
                 row = [response_rate, measure, school, academic_year]
@@ -59,18 +67,15 @@ module Report
       end
 
       workers.each(&:join)
-      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
-      filepath = Rails.root.join("tmp", "reports", filename)
-      write_csv(data:, filepath:)
-      data
-    end
 
-    def self.write_csv(data:, filepath:)
       csv = CSV.generate do |csv|
         data.each do |row|
           csv << row
         end
       end
+    end
+
+    def self.write_csv(data:, filepath:)
       File.write(filepath, csv)
     end
 

app/models/report/measure_summary.rb

@@ -1,6 +1,14 @@
 module Report
   class MeasureSummary
-    def self.create_report(district:, academic_years: AcademicYear.all, measures: ::Measure.all.order(measure_id: :ASC), filename: "measure_summary.csv")
+    def self.create_report(schools:, academic_years: AcademicYear.all, measures: ::Measure.all.order(measure_id: :ASC), filename: "measure_summary.csv")
+      data = to_csv(schools:, academic_years:, measures:)
+      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
+      filepath = Rails.root.join("tmp", "reports", filename)
+      write_csv(data:, filepath:)
+      data
+    end
+
+    def self.to_csv(schools:, academic_years:, measures:)
       data = []
       mutex = Thread::Mutex.new
       data << ["Measure Name", "Measure ID", "District", "Academic Year", "Recorded Date Range", "Grades", "Student Score", "Student Zone", "Teacher Score",
@@ -15,7 +23,7 @@ module Report
             academic_years.each do |academic_year|
               all_grades = Set.new
 
-              respondents = Respondent.where(school: district.schools, academic_year:)
+              respondents = Respondent.where(school: schools, academic_year:)
               respondents.each do |respondent|
                 respondent.enrollment_by_grade.keys.each do |grade|
                   all_grades.add(grade)
@@ -24,12 +32,13 @@ module Report
               all_grades = all_grades.to_a
               grades = "#{all_grades.first}-#{all_grades.last}"
 
-              begin_date = SurveyItemResponse.where(school: district.schools,
-                                                    academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date
-              end_date = SurveyItemResponse.where(school: district.schools,
-                                                  academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date
+              begin_date = ::SurveyItemResponse.where(school: schools,
+                                                      academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date
+              end_date = ::SurveyItemResponse.where(school: schools,
+                                                    academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date
               date_range = "#{begin_date} - #{end_date}"
 
+              district = schools.first.district
               row = [measure, district, academic_year]
 
               mutex.synchronize do
@@ -55,18 +64,15 @@ module Report
       end
 
       workers.each(&:join)
-      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
-      filepath = Rails.root.join("tmp", "reports", filename)
-      write_csv(data:, filepath:)
-      data
-    end
 
-    def self.write_csv(data:, filepath:)
-      csv = CSV.generate do |csv|
+      CSV.generate do |csv|
         data.each do |row|
           csv << row
         end
       end
+    end
+
+    def self.write_csv(data:, filepath:)
       File.write(filepath, csv)
     end
 

app/models/report/subcategory.rb

@@ -1,6 +1,14 @@
 module Report
   class Subcategory
     def self.create_report(schools: School.all.includes(:district), academic_years: AcademicYear.all, subcategories: ::Subcategory.all, filename: "subcategories.csv")
+      data = to_csv(schools:, academic_years:, subcategories:, filename:)
+      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
+      filepath = Rails.root.join("tmp", "reports", filename)
+      write_csv(data:, filepath:)
+      data
+    end
+
+    def self.to_csv(schools: School.all.includes(:district), academic_years: AcademicYear.all, subcategories: ::Subcategory.all, filename: "subcategories.csv")
       data = []
       mutex = Thread::Mutex.new
       data << ["District", "School", "School Code", "Academic Year", "Recorded Date Range", "Grades", "Subcategory", "Student Score", "Student Zone", "Teacher Score",
@@ -24,9 +32,9 @@ module Report
                 zone = subcategory.zone(school:, academic_year:).type.to_s.capitalize
 
                 begin_date = ::SurveyItemResponse.where(school:,
-                                                      academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date
+                                                        academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date
                 end_date = ::SurveyItemResponse.where(school:,
-                                                    academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date
+                                                      academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date
                 date_range = "#{begin_date} - #{end_date}"
 
                 row = [response_rate, subcategory, school, academic_year]
@@ -58,18 +66,15 @@ module Report
       end
 
       workers.each(&:join)
-      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
-      filepath = Rails.root.join("tmp", "reports", filename)
-      write_csv(data:, filepath:)
-      data
-    end
 
-    def self.write_csv(data:, filepath:)
-      csv = CSV.generate do |csv|
+      CSV.generate do |csv|
         data.each do |row|
           csv << row
         end
       end
+    end
+
+    def self.write_csv(data:, filepath:)
       File.write(filepath, csv)
     end
 

app/models/report/survey_item_by_grade.rb

@@ -0,0 +1,115 @@
+module Report
+  class SurveyItemByGrade
+    def self.create_grade_report(schools:, academic_years:, filename:, use_student_survey_items: ::SurveyItem.student_survey_items.pluck(:id))
+      data = to_csv(schools:, academic_years:, use_student_survey_items:)
+      # write out file
+      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
+      filepath = Rails.root.join("tmp", "reports", filename)
+      write_csv(data:, filepath:)
+      data
+    end
+
+    def self.to_csv(schools:, academic_years:, use_student_survey_items:)
+      # get list of survey items with sufficient responses
+      survey_items = Set.new
+      # also get a map of grade->survey_id
+      sufficient_survey_items = {}
+      grades = ::SurveyItemResponse.where(school: schools,
+                                          academic_year: academic_years).where.not(grade: nil).pluck(:grade).uniq.sort
+      grades.each do |grade|
+        sufficient_survey_items[grade] ||= Set.new
+      end
+
+      ::SurveyItemResponse.student_survey_items_with_responses_by_grade(
+        school: schools,
+        academic_year: academic_years
+      ).select do |key, _value|
+        use_student_survey_items.include?(key[1])
+      end.each do |key, count|
+        # key[1] is survey item id
+        next if key[0].nil?
+
+        survey_items.add(key[1])
+        # we do not display the grade avg if there are < 10 responses
+        sufficient_survey_items[key[0]].add(key[1]) if count >= 10
+      end
+
+      # write headers
+      headers = [
+        "School Name",
+        "Grade",
+        "Academic Year"
+      ]
+      survey_items.each do |survey_item_id|
+        headers << ::SurveyItem.find_by_id(survey_item_id).prompt
+      end
+      data = []
+      data << headers
+
+      academic_years.each do |academic_year|
+        schools.each do |school|
+          # for each grade, iterate through all survey items in our list
+          # if it has sufficient responses, write out the avg score
+          # else, write 'N/A'
+          grades.each do |grade|
+            next if grade == -1 # skip pre-k
+
+            row = []
+            row << school.name
+            if grade == 0
+              row.append("Kindergarten")
+            else
+              row.append("Grade #{grade}")
+            end
+            row << academic_year.range
+            survey_items.each do |survey_item_id|
+              survey_item = ::SurveyItem.find_by_id survey_item_id
+              if sufficient_survey_items[grade].include? survey_item_id
+                row.append("#{survey_item.survey_item_responses.where(school:, academic_year:,
+                                                                      grade:).average(:likert_score).to_f.round(2)}")
+              else
+                row.append("N/A")
+              end
+            end
+            data << row
+          end
+
+          # now iterate through all survey items again
+          # if the whole school has sufficient responses, write the school avg
+          # else, N/A
+          row = []
+          row.append(school.name)
+          row.append("All School")
+          row.append(academic_year.range)
+          survey_items.each do |survey_item_id|
+            survey_item = ::SurveyItem.find_by_id survey_item_id
+            # filter out response rate at subcategory level <24.5% for school average
+            scale = Scale.find_by_id(survey_item.scale_id)
+            measure = ::Measure.find_by_id(scale.measure_id)
+            subcategory = ::Subcategory.find_by_id(measure.subcategory_id)
+            if ::StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).meets_student_threshold?
+              row.append("#{survey_item.survey_item_responses.where(
+                # We allow the nil (unknown) grades in the school survey item average
+                # also filter less than 10 responses in the whole school
+                'school_id = ? and academic_year_id = ? and (grade IS NULL or grade IN (?))', school.id, academic_year.id, school.grades(academic_year:)
+              ).group('survey_item_id').having('count(*) >= 10').average(:likert_score).values[0].to_f.round(2)}")
+            else
+              row.append("N/A")
+            end
+          end
+          data << row
+        end
+      end
+
+      CSV.generate do |csv|
+        data.each do |row|
+          csv << row
+        end
+      end
+    end
+
+    def self.write_csv(data:, filepath:)
+      File.write(filepath, csv)
+    end
+  end
+end

app/models/report/survey_item_by_item.rb

@@ -0,0 +1,172 @@
+module Report
+  class SurveyItemByItem
+    def self.create_item_report(schools:, academic_years:, filename:, use_student_survey_items: ::SurveyItem.student_survey_items.pluck(:id))
+      data = to_csv(schools:, academic_years:, use_student_survey_items:)
+      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
+      filepath = Rails.root.join("tmp", "reports", filename)
+      write_csv(data:, filepath:)
+      data
+    end
+
+    def self.to_csv(schools:, academic_years:, use_student_survey_items: ::SurveyItem.student_survey_items.pluck(:id))
+      # first, determine headers by finding the school and grades
+      # Q: Should a grade not be included if it has no sufficient responses?
+      # A: Include all grades, except preschool
+
+      # Convert they keys in this hash to a hash where the key is the grade
+      # and the value is a set of sufficient survey IDs
+      survey_ids_to_grades = {}
+      ::SurveyItemResponse.student_survey_items_with_responses_by_grade(
+        school: schools,
+        academic_year: academic_years
+      ).select do |key, _value|
+        use_student_survey_items.include?(key[1])
+      end.each do |key, count|
+        # key[1] is survey item ID
+        # key[0] is grade
+        survey_ids_to_grades[key[1]] ||= Set.new
+        # we do not display the grade avg if there are < 10 responses
+        survey_ids_to_grades[key[1]].add(key[0]) if count >= 10
+      end
+      headers = [
+        "Survey Item",
+        "Category",
+        "Subcategory",
+        "Survey Measure",
+        "Survey Scale",
+        "Survey Population",
+        "School Name",
+        "Academic Year"
+      ]
+
+      grades = ::SurveyItemResponse.where(school: schools,
+                                          academic_year: academic_years)
+                                   .where.not(grade: nil)
+                                   .pluck(:grade)
+                                   .reject { |grade| grade == -1 } # ignore preschool
+                                   .uniq
+                                   .sort
+      grades.each do |value|
+        if value == 0
+          headers.append("Kindergarten")
+        else
+          headers.append("Grade #{value}")
+        end
+      end
+      headers.append("All School")
+      headers.append("Teacher")
+      # Then, add the headers to data
+      data = []
+      data << headers
+
+      academic_years.each do |academic_year|
+        schools.each do |school|
+          # for each survey item id
+          survey_ids_to_grades.each do |id, school_grades|
+            row = []
+            survey_item = survey_item_for_id(id)
+            row.concat(survey_item_info(survey_item:)) # fills prompt + categories
+            row.append("Students")
+            row.append(school.name)
+            row.append(academic_year.range)
+
+            # add padding before grade average section
+            starting_grade = school_grades.sort.first
+            starting_grade = grades.index(starting_grade) || 0
+            padding = Array.new(starting_grade) { "" }
+            row.concat(padding)
+
+            school_grades.sort.each do |grade|
+              next if grade == -1
+
+              if school_grades.include?(grade)
+                # we already know grade has sufficient responses
+                row.append("#{survey_item.survey_item_responses.where(school:, academic_year:,
+                                                                      grade:).average(:likert_score).to_f.round(2)}")
+              else
+                row.append("N/A")
+              end
+            end
+
+            # add padding after the grade average section
+            ending_grade = school_grades.sort.last
+            ending_grade = grades.index(ending_grade) + 1 || 0
+            padding = Array.new(grades.length - ending_grade) { "" }
+            row.concat(padding)
+
+            # filter out response rate at subcategory level <24.5% for school average
+            subcategory = scale_for_id(survey_item.scale_id).measure.subcategory
+            # measure = ::Measure.find_by_id(scale.measure_id)
+            # subcategory = ::Subcategory.find_by_id(measure.subcategory_id)
+            if response_rate(subcategory:, school:, academic_year:).meets_student_threshold?
+              row.append("#{survey_item.survey_item_responses.where(
+                # We allow the nil (unknown) grades in the school survey item average
+                # also filter less than 10 responses in the whole school
+                'school_id = ? and academic_year_id = ? and (grade IS NULL or grade IN (?))', school.id, academic_year.id, school.grades(academic_year:)
+              ).group('survey_item_id').having('count(*) >= 10').average(:likert_score).values[0].to_f.round(2)}")
+
+            else
+              row.append("N/A")
+            end
+            data << row
+          end
+          # Next up is teacher data
+          ::SurveyItemResponse.teacher_survey_items_with_sufficient_responses(school:, academic_year:).keys.each do |key| # each key is a survey item id
+            row = []
+            survey_item = survey_item_for_id(key)
+            row.concat(survey_item_info(survey_item:))
+            row.append("Teacher")
+            row.append(school.name)
+            row.append(academic_year.range)
+            # we need to add padding to skip the grades columns and the 'all school' column
+            padding = Array.new(grades.length + 1) { "" }
+            row.concat(padding)
+            # we already know that the survey item we are looking at has sufficient responses
+            row.append("#{survey_item.survey_item_responses.where(school:,
+                                                                  academic_year:).average(:likert_score).to_f.round(2)}")
+            data << row
+          end
+        end
+      end
+
+      CSV.generate do |csv|
+        data.each do |row|
+          csv << row
+        end
+      end
+    end
+
+    def self.response_rate(subcategory:, school:, academic_year:)
+      @response_rate ||= Hash.new do |memo, subcategory, school, academic_year|
+        memo[[subcategory, school, academic_year]] =
+          ::StudentResponseRateCalculator.new(subcategory:, school:, academic_year:)
+      end
+      @response_rate[[subcategory, school, academic_year]]
+    end
+
+    def self.survey_item_for_id(survey_item_id)
+      @survey_items ||= ::SurveyItem.all.map do |survey_item|
+        [survey_item.id, survey_item]
+      end.to_h
+      @survey_items[survey_item_id]
+    end
+
+    def self.scale_for_id(scale_id)
+      @scales ||= Scale.includes([measure: :subcategory]).all.map { |scale| [scale.id, scale] }.to_h
+      @scales[scale_id]
+    end
+
+    def self.write_csv(data:, filepath:)
+      File.write(filepath, csv)
+    end
+
+    def self.survey_item_info(survey_item:)
+      prompt = survey_item.prompt
+      scale = Scale.find_by_id(survey_item.scale_id)
+      measure = ::Measure.find_by_id(scale.measure_id)
+      subcategory = ::Subcategory.find_by_id(measure.subcategory_id)
+      category = Category.find_by_id(subcategory.category_id)
+      [prompt, category.name, subcategory.name, measure.name, scale.scale_id]
+    end
+  end
+end

app/models/report/survey_item_response.rb

@@ -2,6 +2,14 @@
 module Report
   class SurveyItemResponse
     def self.create(schools:, academic_years:, filename:)
+      data = to_csv(schools:, academic_years:)
+      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
+      filepath = Rails.root.join("tmp", "reports", filename)
+      write_csv(data:, filepath:)
+      data
+    end
+
+    def self.to_csv(schools:, academic_years:)
       data = []
       data << ["Response ID", "Race", "Gender", "Grade", "School ID", "District", "Academic Year", "Student Physical Safety",
                "Student Emotional Safety", "Student Sense of Belonging", "Student-Teacher Relationships", "Valuing of Learning", "Academic Challenge", "Content Specialists & Support Staff", "Cultural Responsiveness", "Engagement In School", "Appreciation For Diversity", "Civic Participation", "Perseverance & Determination", "Growth Mindset", "Participation In Creative & Performing Arts", "Valuing Creative & Performing Arts", "Social & Emotional Health"]
@@ -69,10 +77,12 @@ module Report
       end
 
       workers.each(&:join)
-      FileUtils.mkdir_p Rails.root.join("tmp", "reports")
-      filepath = Rails.root.join("tmp", "reports", filename)
-      write_csv(data:, filepath:)
-      data
+
+      CSV.generate do |csv|
+        data.each do |row|
+          csv << row
+        end
+      end
     end
 
     def self.average_for_measure(measure_id, responses)
@@ -94,11 +104,6 @@ module Report
     end
 
     def self.write_csv(data:, filepath:)
-      csv = CSV.generate do |csv|
-        data.each do |row|
-          csv << row
-        end
-      end
       File.write(filepath, csv)
     end