From 6c88ed18912a8d2c240b9814f191ed6b2de79343 Mon Sep 17 00:00:00 2001
From: Nelson Jovel <memoryman51@hotmail.com>
Date: Thu, 15 Aug 2024 09:25:58 -0700
Subject: [PATCH] fix: swap over to using respondents table to calculate which
 grades need to be included.   This fixes a problem where errant responses
 were polluting the set of grades for a school.   Also order the survey items
 by prompt(alphabetically)

---
 app/models/report/survey_item_by_grade.rb | 24 ++++++++++------
 app/models/report/survey_item_by_item.rb  | 35 +++++++++++++----------
 2 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/app/models/report/survey_item_by_grade.rb b/app/models/report/survey_item_by_grade.rb
index 1358b592..3bcfe7c5 100644
--- a/app/models/report/survey_item_by_grade.rb
+++ b/app/models/report/survey_item_by_grade.rb
@@ -14,19 +14,26 @@ module Report
       survey_items = Set.new
       # also get a map of grade->survey_id
       sufficient_survey_items = {}
-      grades = ::SurveyItemResponse.where(school: schools,
-                                          academic_year: academic_years).where.not(grade: nil).pluck(:grade).uniq.sort
+
+      grades = []
+      schools.each do |school|
+        academic_years.each do |academic_year|
+          grades.concat(school.grades(academic_year:))
+        end
+      end
+      grades = grades.uniq.reject { |grade| grade == -1 }.reject(&:nil?) # remove preschool and nil grades
+
       grades.each do |grade|
         sufficient_survey_items[grade] ||= Set.new
       end
 
-      ::SurveyItemResponse.student_survey_items_with_responses_by_grade(
-        school: schools,
-        academic_year: academic_years
-      ).select do |key, _value|
-        use_student_survey_items.include?(key[1])
-      end.each do |key, count|
+      sufficient_responses_by_grade_and_survey_item_id = ::SurveyItemResponse.where(school: schools, academic_year: academic_years, survey_item_id: use_student_survey_items).where.not(grade: nil).having("count(*) >= ?", 10).group(
+        :grade, :survey_item_id
+      ).count
+
+      sufficient_responses_by_grade_and_survey_item_id.each do |key, count|
         # key[1] is survey item id
+        # key[0] is grade
         next if key[0].nil?
 
         survey_items.add(key[1])
@@ -40,6 +47,7 @@ module Report
         "Grade",
         "Academic Year"
       ]
+      survey_items = survey_items.sort_by { |id| ::SurveyItem.find(id).prompt }
       survey_items.each do |survey_item_id|
         headers << ::SurveyItem.find_by_id(survey_item_id).prompt
       end
diff --git a/app/models/report/survey_item_by_item.rb b/app/models/report/survey_item_by_item.rb
index 875239dc..e6fea760 100644
--- a/app/models/report/survey_item_by_item.rb
+++ b/app/models/report/survey_item_by_item.rb
@@ -16,12 +16,12 @@ module Report
       # Convert they keys in this hash to a hash where the key is the grade
       # and the value is a set of sufficient survey IDs
       survey_ids_to_grades = {}
-      ::SurveyItemResponse.student_survey_items_with_responses_by_grade(
-        school: schools,
-        academic_year: academic_years
-      ).select do |key, _value|
-        use_student_survey_items.include?(key[1])
-      end.each do |key, count|
+
+      sufficient_responses_by_grade_and_survey_item_id = ::SurveyItemResponse.where(school: schools, academic_year: academic_years, survey_item_id: use_student_survey_items).where.not(grade: nil).having("count(*) >= ?", 10).group(
+        :grade, :survey_item_id
+      ).count
+
+      sufficient_responses_by_grade_and_survey_item_id.each do |key, count|
         # key[1] is survey item ID
         # key[0] is grade
         survey_ids_to_grades[key[1]] ||= Set.new
@@ -39,13 +39,14 @@ module Report
         "Academic Year"
       ]
 
-      grades = ::SurveyItemResponse.where(school: schools,
-                                          academic_year: academic_years)
-                                   .where.not(grade: nil)
-                                   .pluck(:grade)
-                                   .reject { |grade| grade == -1 } # ignore preschool
-                                   .uniq
-                                   .sort
+      grades = []
+      schools.each do |school|
+        academic_years.each do |academic_year|
+          grades.concat(school.grades(academic_year:))
+        end
+      end
+      grades = grades.uniq.reject { |grade| grade == -1 }.reject(&:nil?) # remove preschool and nil grades
+
       grades.each do |value|
         if value == 0
           headers.append("Kindergarten")
@@ -62,7 +63,7 @@ module Report
       academic_years.each do |academic_year|
         schools.each do |school|
           # for each survey item id
-          survey_ids_to_grades.each do |id, school_grades|
+          survey_ids_to_grades.sort_by { |id, _value| ::SurveyItem.find(id).prompt }.each do |id, school_grades|
             school_grades = school_grades.reject(&:nil?)
             row = []
             survey_item = survey_item_for_id(id)
@@ -112,7 +113,11 @@ module Report
             data << row
           end
           # Next up is teacher data
-          ::SurveyItemResponse.teacher_survey_items_with_sufficient_responses(school:, academic_year:).keys.each do |key| # each key is a survey item id
+          # each key is a survey item id
+          ::SurveyItemResponse.teacher_survey_items_with_sufficient_responses(school:,
+                                                                              academic_year:).keys.sort_by do |id|
+            ::SurveyItem.find(id).prompt
+          end.each do |key|
             row = []
             survey_item = survey_item_for_id(key)
             row.concat(survey_item_info(survey_item:))