Create ability to run exports locally so they get created in

tmp/exports.
Fix measure summary for 'all years' output.

Slight speedup to survey item by grade.
Fix Survey item by item for the 'all years' output

app/models/report/exports.rb

@@ -0,0 +1,97 @@
+module Report
+  class Exports
+    def self.create
+      academic_years = ::AcademicYear.all
+      districts = ::District.all
+      use_student_survey_items = ::SurveyItem.student_survey_items.map(&:id)
+      schools = ::School.all.includes(:district)
+
+      reports = {
+        "Subcategory by School & District" => lambda { |schools, academic_years|
+          Report::Subcategory.to_csv(schools:, academic_years:, subcategories: ::Subcategory.all)
+        },
+        "Measure by District only" => lambda { |schools, academic_years|
+          Report::MeasureSummary.to_csv(schools:, academic_years:, measures: ::Measure.all)
+        },
+        "Measure by School & District" => lambda { |schools, academic_years|
+          Report::Measure.to_csv(schools:, academic_years:,
+                                 measures: ::Measure.all)
+        },
+        "Survey Item by Item" => lambda { |schools, academic_years|
+          Report::SurveyItemByItem.to_csv(schools:, academic_years:)
+        },
+        "Survey Item by Grade" => lambda { |schools, academic_years|
+          Report::SurveyItemByGrade.to_csv(schools:, academic_years:,
+                                           use_student_survey_items:)
+        },
+        "Survey Entries by Measure" => lambda { |schools, academic_years|
+          Report::SurveyItemResponse.to_csv(schools:, academic_years:, use_student_survey_items:)
+        }
+      }
+
+      reports.each do |report_name, runner|
+        report_name = report_name.gsub(" ", "_").downcase
+
+        academic_years.each do |academic_year|
+          districts.each do |district|
+            # Each year
+            threads = []
+            threads << Thread.new do
+              response_count = ::SurveyItemResponse.where(school: district.schools, academic_year: academic_year).count
+              if response_count > 100
+                schools = district.schools
+                report_name = report_name.gsub(" ", "_").downcase
+
+                ::FileUtils.mkdir_p ::Rails.root.join("tmp", "exports", report_name, academic_year.range, district.name)
+
+                filename = "#{report_name}_#{academic_year.range}_#{district.name}.csv"
+                filepath = Rails.root.join("tmp", "exports", report_name, academic_year.range, district.name, filename)
+                csv = runner.call(schools, [academic_year])
+
+                File.write(filepath, csv)
+              end
+            end
+            threads.each(&:join)
+            GC.start
+          end
+          # All districts
+          response_count = ::SurveyItemResponse.where(school: districts.flat_map(&:schools), academic_year: academic_year).count
+          next unless response_count > 100
+
+          threads = []
+          threads << Thread.new do
+            ::FileUtils.mkdir_p ::Rails.root.join("tmp", "exports", report_name, academic_year.range, "all_districts")
+
+            filename = "#{report_name}_all_districts.csv"
+            filepath = Rails.root.join("tmp", "exports", report_name, academic_year.range, "all_districts", filename)
+            csv = runner.call(districts.flat_map(&:schools), [academic_year])
+
+            File.write(filepath, csv)
+          end
+          threads.each(&:join)
+          GC.start
+        end
+
+        districts.each do |district|
+          #   # All years for each district
+          threads = []
+          threads << Thread.new do
+            response_count = ::SurveyItemResponse.where(school: district.schools, academic_year: academic_years).count
+            next unless response_count > 100
+
+            ::FileUtils.mkdir_p ::Rails.root.join("tmp", "exports", report_name, "all_years", district.name)
+
+            filename = "#{report_name}_all_years_#{district.name}.csv"
+            filepath = Rails.root.join("tmp", "exports", report_name, "all_years", district.name, filename)
+            csv = runner.call(district.schools, academic_years)
+
+            File.write(filepath, csv)
+            GC.start
+          end
+          threads.each(&:join)
+          GC.start
+        end
+      end
+    end
+  end
+end

app/models/report/measure_summary.rb

@@ -22,32 +22,32 @@ module Report
           while measure = jobs.pop(true)
             all_grades = Respondent.grades_that_responded_to_survey(academic_year: academic_years, school: schools)
             grades = "#{all_grades.first}-#{all_grades.last}"
-            district = schools.first.district
 
             academic_years.each do |academic_year|
-              begin_date = ::SurveyItemResponse.where(school: schools,
-                                                      academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date
-              end_date = ::SurveyItemResponse.where(school: schools,
-                                                    academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date
-              date_range = "#{begin_date} - #{end_date}"
-
-              row = [measure, district, academic_year]
-
-              mutex.synchronize do
-                data << [measure.name,
-                         measure.measure_id,
-                         district.name,
-                         academic_year.range,
-                         date_range,
-                         grades,
-                         student_score(row:),
-                         student_zone(row:),
-                         teacher_score(row:),
-                         teacher_zone(row:),
-                         admin_score(row:),
-                         admin_zone(row:),
-                         all_data_score(row:),
-                         all_data_zone(row:)]
+              schools.flat_map(&:district).uniq.each do |district|
+                selected_schools = district.schools
+                begin_date = ::SurveyItemResponse.where(school: selected_schools,
+                                                        academic_year:).where.not(recorded_date: nil).order(:recorded_date).first&.recorded_date&.to_date
+                end_date = ::SurveyItemResponse.where(school: selected_schools,
+                                                      academic_year:).where.not(recorded_date: nil).order(:recorded_date).last&.recorded_date&.to_date
+                date_range = "#{begin_date} - #{end_date}"
+
+                mutex.synchronize do
+                  data << [measure.name,
+                           measure.measure_id,
+                           district.name,
+                           academic_year.range,
+                           date_range,
+                           grades,
+                           student_score(measure:,  schools: selected_schools, academic_year:),
+                           student_zone(measure:,   schools: selected_schools, academic_year:),
+                           teacher_score(measure:,  schools: selected_schools, academic_year:),
+                           teacher_zone(measure:,   schools: selected_schools, academic_year:),
+                           admin_score(measure:,    schools: selected_schools, academic_year:),
+                           admin_zone(measure:,     schools: selected_schools, academic_year:),
+                           all_data_score(measure:, schools: selected_schools, academic_year:),
+                           all_data_zone(measure:,  schools: selected_schools, academic_year:)]
+                end
               end
             end
           end
@@ -68,18 +68,18 @@ module Report
       File.write(filepath, csv)
     end
 
-    def self.all_data_score(row:)
-      row in [ measure, district, academic_year]
-      score = district.schools.map do |school|
-        score = measure.score(school:, academic_year:).average
-      end.remove_blanks.average
-      score || "N/A"
+    def self.all_data_score(measure:, schools:, academic_year:)
+      @all_data_score ||= Hash.new do |memo, (measure, schools, academic_year)|
+        score = schools.map do |school|
+          score = measure.score(school:, academic_year:).average
+        end.remove_blanks.average
+        memo[[measure, schools, academic_year]] = score || "N/A"
+      end
+      @all_data_score[[measure, schools, academic_year]]
     end
 
-    def self.all_data_zone(row:)
-      row in [ measure, district, academic_year]
-
-      average = all_data_score(row:)
+    def self.all_data_zone(measure:, schools:, academic_year:)
+      average = all_data_score(measure:, schools:, academic_year:)
       score = Score.new(average:, meets_teacher_threshold: true, meets_student_threshold: true,
                         meets_admin_data_threshold: true)
       student_zone = measure.zone_for_score(score:).type.to_s
@@ -87,17 +87,18 @@ module Report
       student_zone || "N/A"
     end
 
-    def self.student_score(row:)
-      row in [ measure, district, academic_year]
-      student_score = district.schools.map do |school|
-        student_score = measure.student_score(school:, academic_year:).average
-      end.remove_blanks.average
-      student_score || "N/A"
+    def self.student_score(measure:, schools:, academic_year:)
+      @student_score ||= Hash.new do |memo, (measure, schools, academic_year)|
+        student_score = schools.map do |school|
+          student_score = measure.student_score(school:, academic_year:).average
+        end.remove_blanks.average
+        memo[[measure, schools, academic_year]] = student_score || "N/A"
+      end
+      @student_score[[measure, schools, academic_year]]
     end
 
-    def self.student_zone(row:)
-      row in [ measure, district, academic_year]
-      average = student_score(row:)
+    def self.student_zone(measure:, schools:, academic_year:)
+      average = student_score(measure:, schools:, academic_year:)
       score = Score.new(average:, meets_teacher_threshold: true, meets_student_threshold: true,
                         meets_admin_data_threshold: true)
       student_zone = measure.zone_for_score(score:).type.to_s
@@ -105,18 +106,19 @@ module Report
       student_zone || "N/A"
     end
 
-    def self.teacher_score(row:)
-      row in [ measure, district, academic_year]
-      teacher_score = district.schools.map do |school|
-        measure.teacher_score(school:, academic_year:).average
-      end.remove_blanks.average
+    def self.teacher_score(measure:, schools:, academic_year:)
+      @teacher_score ||= Hash.new do |memo, (measure, schools, academic_year)|
+        teacher_score = schools.map do |school|
+          measure.teacher_score(school:, academic_year:).average
+        end.remove_blanks.average
 
-      teacher_score || "N/A"
+        memo[[measure, schools, academic_year]] = teacher_score || "N/A"
+      end
+      @teacher_score[[measure, schools, academic_year]]
     end
 
-    def self.teacher_zone(row:)
-      row in [ measure, district, academic_year]
-      average = teacher_score(row:)
+    def self.teacher_zone(measure:, schools:, academic_year:)
+      average = teacher_score(measure:, schools:, academic_year:)
       score = Score.new(average:, meets_teacher_threshold: true, meets_student_threshold: true,
                         meets_admin_data_threshold: true)
       teacher_zone = measure.zone_for_score(score:).type.to_s
@@ -124,19 +126,20 @@ module Report
       teacher_zone || "N/A"
     end
 
-    def self.admin_score(row:)
-      row in [ measure, district, academic_year]
-      admin_score = district.schools.map do |school|
-        measure.admin_score(school:, academic_year:).average
-      end.remove_blanks.average
+    def self.admin_score(measure:, schools:, academic_year:)
+      @admin_score ||= Hash.new do |memo, (measure, schools, academic_year)|
+        admin_score = schools.map do |school|
+          measure.admin_score(school:, academic_year:).average
+        end.remove_blanks.average
 
-      admin_score = "N/A" unless admin_score.present? && admin_score >= 0
-      admin_score
+        admin_score = "N/A" unless admin_score.present? && admin_score >= 0
+        memo[[measure, schools, academic_year]] = admin_score
+      end
+      @admin_score[[measure, schools, academic_year]]
     end
 
-    def self.admin_zone(row:)
-      row in [ measure, district, academic_year]
-      average = admin_score(row:)
+    def self.admin_zone(measure:, schools:, academic_year:)
+      average = admin_score(measure:, schools:, academic_year:)
       score = Score.new(average:, meets_teacher_threshold: true, meets_student_threshold: true,
                         meets_admin_data_threshold: true)
 

app/models/report/subcategory.rb

@@ -8,7 +8,7 @@ module Report
       csv
     end
 
-    def self.to_csv(schools: School.all.includes(:district), academic_years: AcademicYear.all, subcategories: ::Subcategory.all, filename: "subcategories.csv")
+    def self.to_csv(schools: School.all.includes(:district), academic_years: AcademicYear.all, subcategories: ::Subcategory.all)
       data = []
       mutex = Thread::Mutex.new
       data << ["District", "School", "School Code", "Academic Year", "Recorded Date Range", "Grades", "Subcategory", "Student Score", "Student Zone", "Teacher Score",

app/models/report/survey_item_by_grade.rb

@@ -9,11 +9,12 @@ module Report
       csv
     end
 
-    def self.to_csv(schools:, academic_years:, use_student_survey_items:)
+    def self.to_csv(schools:, academic_years:, use_student_survey_items: ::SurveyItem.student_survey_items.pluck(:id))
       # get list of survey items with sufficient responses
       survey_items = Set.new
       # also get a map of grade->survey_id
       sufficient_survey_items = {}
+      survey_items_by_id = ::SurveyItem.by_id_includes_all
 
       grades = Respondent.grades_that_responded_to_survey(academic_year: academic_years, school: schools)
 
@@ -41,9 +42,9 @@ module Report
         "Grade",
         "Academic Year"
       ]
-      survey_items = survey_items.sort_by { |id| ::SurveyItem.find(id).prompt }
+      survey_items = survey_items.sort_by { |id| survey_items_by_id[id].prompt }
       survey_items.each do |survey_item_id|
-        headers << ::SurveyItem.find_by_id(survey_item_id).prompt
+        headers << survey_items_by_id[survey_item_id].prompt
       end
       data = []
       data << headers
@@ -65,7 +66,8 @@ module Report
             end
             row << academic_year.range
             survey_items.each do |survey_item_id|
-              survey_item = ::SurveyItem.find_by_id survey_item_id
+              survey_item = survey_items_by_id[survey_item_id]
+              byebug if survey_item.nil?
               if sufficient_survey_items[grade].include? survey_item_id
                 row.append("#{survey_item.survey_item_responses.where(school:, academic_year:,
                                                                       grade:).average(:likert_score).to_f.round(2)}")
@@ -84,11 +86,10 @@ module Report
           row.append("All School")
           row.append(academic_year.range)
           survey_items.each do |survey_item_id|
-            survey_item = ::SurveyItem.find_by_id survey_item_id
+            survey_item = survey_items_by_id[survey_item_id]
+            byebug if survey_item.nil?
             # filter out response rate at subcategory level <24.5% for school average
-            scale = Scale.find_by_id(survey_item.scale_id)
-            measure = ::Measure.find_by_id(scale.measure_id)
-            subcategory = ::Subcategory.find_by_id(measure.subcategory_id)
+            subcategory = survey_item.scale.measure.subcategory
             if ::StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).meets_student_threshold?
               row.append("#{survey_item.survey_item_responses.where(
                 # We allow the nil (unknown) grades in the school survey item average

app/models/report/survey_item_by_item.rb

@@ -28,124 +28,100 @@ module Report
         # we do not display the grade avg if there are < 10 responses
         survey_ids_to_grades[key[1]].add(key[0]) if count >= 10
       end
-      headers = [
-        "Survey Item",
-        "Category",
-        "Subcategory",
-        "Survey Measure",
-        "Survey Scale",
-        "Survey Population",
-        "School Name",
-        "Academic Year"
-      ]
 
       grades = Respondent.grades_that_responded_to_survey(academic_year: academic_years, school: schools)
-
-      grades.each do |value|
-        if value == 0
-          headers.append("Kindergarten")
-        else
-          headers.append("Grade #{value}")
-        end
-      end
-      headers.append("All School")
-      headers.append("Teacher")
       # Then, add the headers to data
       data = []
-      data << headers
+      data << generate_headers(schools:, academic_years:, grades:)
 
       survey_items_by_id = ::SurveyItem.by_id_includes_all
 
-      mutex = Thread::Mutex.new
-      pool_size = 2
-      jobs = Queue.new
-      schools.each { |school| jobs << school }
+      # mutex = Thread::Mutex.new
+      # pool_size = 2
+      # jobs = Queue.new
+      # schools.each { |school| jobs << school }
 
       academic_years.each do |academic_year|
-        workers = pool_size.times.map do
-          Thread.new do
-            while school = jobs.pop(true)
-              # for each survey item id
-              survey_ids_to_grades.sort_by do |id, _value|
-                survey_items_by_id[id].prompt
-              end.each do |id, school_grades|
-                school_grades = school_grades.reject(&:nil?)
-                row = []
-                survey_item = survey_items_by_id[id]
-                row.concat(survey_item_info(survey_item:)) # fills prompt + categories
-                row.append("Students")
-                row.append(school.name)
-                row.append(academic_year.range)
-
-                # add padding before grade average section
-                starting_grade = school_grades.sort.first
-                starting_grade = grades.index(starting_grade) || 0
-                padding = Array.new(starting_grade) { "" }
-                row.concat(padding)
-
-                school_grades.sort.each do |grade|
-                  next if grade == -1
-
-                  if school_grades.include?(grade)
-                    # we already know grade has sufficient responses
-                    score = survey_item.survey_item_responses.where(school:, academic_year:,
-                                                                    grade:).average(:likert_score).to_f.round(2)
-                    score = "" if score.zero?
-                    row.append("#{score}")
-                  else
-                    row.append("")
-                  end
-                end
-
-                # add padding after the grade average section
-                ending_grade = school_grades.sort.last
-                ending_grade = grades.index(ending_grade) + 1 || 0
-                padding = Array.new(grades.length - ending_grade) { "" }
-                row.concat(padding)
-
-                # filter out response rate at subcategory level <24.5% for school average
-                if response_rate(subcategory: survey_item.subcategory, school:,
-                                 academic_year:).meets_student_threshold?
-                  all_student_score = survey_item.survey_item_responses.where(
-                    # We allow the nil (unknown) grades in the school survey item average
-                    # also filter less than 10 responses in the whole school
-                    "school_id = ? and academic_year_id = ? and (grade IS NULL or grade IN (?))", school.id, academic_year.id, school.grades(academic_year:)
-                  ).group("survey_item_id").having("count(*) >= 10").average(:likert_score).values[0].to_f.round(2)
-
-                  all_student_score = "" if all_student_score.zero?
-                  row.append("#{all_student_score}")
-                else
-                  row.append("")
-                end
-                data << row
-              end
-              # Next up is teacher data
-              # each key is a survey item id
-              ::SurveyItemResponse.teacher_survey_items_with_sufficient_responses(school:,
-                                                                                  academic_year:).keys.sort_by do |id|
-                survey_items_by_id[id].prompt
-              end.each do |key|
-                row = []
-                survey_item = survey_items_by_id[key]
-                row.concat(survey_item_info(survey_item:))
-                row.append("Teacher")
-                row.append(school.name)
-                row.append(academic_year.range)
-                # we need to add padding to skip the grades columns and the 'all school' column
-                padding = Array.new(grades.length + 1) { "" }
-                row.concat(padding)
-                # we already know that the survey item we are looking at has sufficient responses
-                all_teacher_score = survey_item.survey_item_responses.where(school:,
-                                                                            academic_year:).average(:likert_score).to_f.round(2)
-                all_teacher_score = "" if all_teacher_score.zero?
-                row.append("#{all_teacher_score}")
-                data << row
+        next unless ::SurveyItemResponse.where(school: schools, academic_year: academic_year, survey_item_id: use_student_survey_items).count > 100
+
+        schools.each do |school|
+          # for each survey item id
+          survey_ids_to_grades.sort_by do |id, _value|
+            survey_items_by_id[id].prompt
+          end.each do |id, school_grades|
+            school_grades = school_grades.reject(&:nil?)
+            row = []
+            survey_item = survey_items_by_id[id]
+            row.concat(survey_item_info(survey_item:)) # fills prompt + categories
+            row.append("Students")
+            row.append(school.name)
+            row.append(academic_year.range)
+
+            # add padding before grade average section
+            starting_grade = school_grades.sort.first
+            starting_grade = grades.index(starting_grade) || 0
+            padding = Array.new(starting_grade) { "" }
+            row.concat(padding)
+
+            school_grades.sort.each do |grade|
+              next if grade == -1
+
+              if school_grades.include?(grade)
+                # we already know grade has sufficient responses
+                score = survey_item.survey_item_responses.where(school:, academic_year:,
+                                                                grade:).average(:likert_score).to_f.round(2)
+                score = "" if score.zero?
+                row.append("#{score}")
+              else
+                row.append("")
               end
             end
-          rescue ThreadError
+
+            # add padding after the grade average section
+            ending_grade = school_grades.sort.last
+            ending_grade = grades.index(ending_grade) + 1 || 0
+            padding = Array.new(grades.length - ending_grade) { "" }
+            row.concat(padding)
+
+            # filter out response rate at subcategory level <24.5% for school average
+            if response_rate(subcategory: survey_item.subcategory, school:,
+                             academic_year:).meets_student_threshold?
+              all_student_score = survey_item.survey_item_responses.where(
+                # We allow the nil (unknown) grades in the school survey item average
+                # also filter less than 10 responses in the whole school
+                "school_id = ? and academic_year_id = ? and (grade IS NULL or grade IN (?))", school.id, academic_year.id, school.grades(academic_year:)
+              ).group("survey_item_id").having("count(*) >= 10").average(:likert_score).values[0].to_f.round(2)
+
+              all_student_score = "" if all_student_score.zero?
+              row.append("#{all_student_score}")
+            else
+              row.append("")
+            end
+            data << row
+          end
+          # Next up is teacher data
+          # each key is a survey item id
+          ::SurveyItemResponse.teacher_survey_items_with_sufficient_responses(school:,
+                                                                              academic_year:).keys.sort_by do |id|
+            survey_items_by_id[id].prompt
+          end.each do |key|
+            row = []
+            survey_item = survey_items_by_id[key]
+            row.concat(survey_item_info(survey_item:))
+            row.append("Teacher")
+            row.append(school.name)
+            row.append(academic_year.range)
+            # we need to add padding to skip the grades columns and the 'all school' column
+            padding = Array.new(grades.length + 1) { "" }
+            row.concat(padding)
+            # we already know that the survey item we are looking at has sufficient responses
+            all_teacher_score = survey_item.survey_item_responses.where(school:,
+                                                                        academic_year:).average(:likert_score).to_f.round(2)
+            all_teacher_score = "" if all_teacher_score.zero?
+            row.append("#{all_teacher_score}")
+            data << row
           end
         end
-        workers.each(&:join)
       end
 
       CSV.generate do |csv|
@@ -175,5 +151,29 @@ module Report
        survey_item.measure.name,
        survey_item.scale.scale_id]
     end
+
+    def self.generate_headers(schools:, academic_years:, grades:)
+      headers = [
+        "Survey Item",
+        "Category",
+        "Subcategory",
+        "Survey Measure",
+        "Survey Scale",
+        "Survey Population",
+        "School Name",
+        "Academic Year"
+      ]
+
+      grades.each do |value|
+        if value == 0
+          headers.append("Kindergarten")
+        else
+          headers.append("Grade #{value}")
+        end
+      end
+      headers.append("All School")
+      headers.append("Teacher")
+      headers
+    end
   end
 end

app/models/report/survey_item_response.rb

@@ -9,7 +9,7 @@ module Report
       data
     end
 
-    def self.to_csv(schools:, academic_years:, use_student_survey_items:)
+    def self.to_csv(schools:, academic_years:, use_student_survey_items: ::SurveyItem.student_survey_items.map(&:id))
       data = []
       data << ["Response ID", "Race", "Gender", "Grade", "School ID", "District", "Academic Year", "Student Physical Safety",
                "Student Emotional Safety", "Student Sense of Belonging", "Student-Teacher Relationships", "Valuing of Learning", "Academic Challenge", "Content Specialists & Support Staff", "Cultural Responsiveness", "Engagement In School", "Appreciation For Diversity", "Civic Participation", "Perseverance & Determination", "Growth Mindset", "Participation In Creative & Performing Arts", "Valuing Creative & Performing Arts", "Social & Emotional Health"]

lib/tasks/report.rake

@@ -194,4 +194,12 @@ namespace :report do
       Report::SurveyItemResponse.create(schools: district.schools, academic_years:, filename:)
     end
   end
+
+  # Usage example
+  # bundle exec rake "report:exports:create"
+  namespace :exports do
+    task :create, %i[district academic_year] => :environment do |_, _args|
+      Report::Exports.create
+    end
+  end
 end