alter logic for insufficiency so that a count of 0 survey item responses is enough to trigger insufficiency

Implement large speed improvements to score calculations.  Add page caching to all pages.  Small speed improvements to response rate by filtering out
survey items without responses with `none?` `method vs count == 0`.

app/models/measure.rb

@@ -22,6 +22,13 @@ class Measure < ActiveRecord::Base
     @student_survey_items ||= survey_items.student_survey_items
   end
 
+  def student_survey_items_by_survey_type(school:, academic_year:)
+    survey = Survey.where(school:, academic_year:).first
+    return survey_items.student_survey_items.short_form_items if survey.form == 'short'
+
+    survey_items.student_survey_items
+  end
+
   def teacher_scales
     @teacher_scales ||= scales.teacher_scales
   end
@@ -31,11 +38,11 @@ class Measure < ActiveRecord::Base
   end
 
   def includes_teacher_survey_items?
-    @includes_teacher_survey_items ||= teacher_survey_items.any?
+    teacher_survey_items.any?
   end
 
   def includes_student_survey_items?
-    @includes_student_survey_items ||= student_survey_items.any?
+    student_survey_items.any?
   end
 
   def includes_admin_data_items?
@@ -61,8 +68,14 @@ class Measure < ActiveRecord::Base
       next Score.new(nil, false, false, false) if incalculable_score
 
       scores = []
-      scores << teacher_score(school:, academic_year:).average if meets_teacher_threshold
+      if meets_teacher_threshold
-      scores << student_score(school:, academic_year:).average if meets_student_threshold
+        scores << collect_survey_item_average(survey_items: teacher_survey_items, school:,
+                                              academic_year:)
+      end
+      if meets_student_threshold
+        scores << collect_survey_item_average(survey_items: student_survey_items_by_survey_type(school:, academic_year:), school:,
+                                              academic_year:)
+      end
       scores << collect_admin_scale_average(admin_data_items, school, academic_year) if includes_admin_data_items?
 
       average = scores.flatten.compact.remove_zeros.average
@@ -81,7 +94,10 @@ class Measure < ActiveRecord::Base
       meets_student_threshold = sufficient_student_data?(school:, academic_year:)
       meets_teacher_threshold = sufficient_teacher_data?(school:, academic_year:)
       meets_admin_data_threshold = all_admin_data_collected?(school:, academic_year:)
-      average = collect_survey_scale_average(student_scales, school, academic_year) if meets_student_threshold
+      if meets_student_threshold
+        average = collect_survey_item_average(survey_items: student_survey_items_by_survey_type(school:, academic_year:), school:,
+                                              academic_year:)
+      end
       memo[[school, academic_year]] =
         Score.new(average, meets_teacher_threshold, meets_student_threshold, meets_admin_data_threshold)
     end
@@ -94,7 +110,10 @@ class Measure < ActiveRecord::Base
       meets_student_threshold = sufficient_student_data?(school:, academic_year:)
       meets_teacher_threshold = sufficient_teacher_data?(school:, academic_year:)
       meets_admin_data_threshold = all_admin_data_collected?(school:, academic_year:)
-      average = collect_survey_scale_average(teacher_scales, school, academic_year) if meets_teacher_threshold
+      if meets_teacher_threshold
+        average = collect_survey_item_average(survey_items: teacher_survey_items, school:,
+                                              academic_year:)
+      end
       memo[[school, academic_year]] =
         Score.new(average, meets_teacher_threshold, meets_student_threshold, meets_admin_data_threshold)
     end
@@ -124,13 +143,14 @@ class Measure < ActiveRecord::Base
 
   def sufficient_student_data?(school:, academic_year:)
     return false unless includes_student_survey_items?
+    return false if student_scales.all? { |scale| scale.survey_item_responses.where(school:, academic_year:).none? }
 
-    @sufficient_student_data ||= subcategory.student_response_rate(school:,
+    @sufficient_student_data ||= subcategory.student_response_rate(school:, academic_year:).meets_student_threshold?
-                                                                   academic_year:).meets_student_threshold?
   end
 
   def sufficient_teacher_data?(school:, academic_year:)
     return false unless includes_teacher_survey_items?
+    return false if teacher_scales.all? { |scale| scale.survey_item_responses.where(school:, academic_year:).none? }
 
     @sufficient_teacher_data ||=  subcategory.teacher_response_rate(school:, academic_year:).meets_teacher_threshold?
   end
@@ -146,16 +166,20 @@ class Measure < ActiveRecord::Base
   end
 
   def sufficient_survey_responses?(school:, academic_year:)
-    @sufficient_survey_responses ||= sufficient_student_data?(school:,
+    @sufficient_survey_responses = Hash.new do |memo, (school, academic_year)|
-                                                              academic_year:) || sufficient_teacher_data?(
+      memo[[school, academic_year]] =
-                                                                school:, academic_year:
+        sufficient_student_data?(school:, academic_year:) || sufficient_teacher_data?(school:, academic_year:)
-                                                              )
+    end
+    @sufficient_survey_responses[[school, academic_year]]
   end
 
   private
 
-  def collect_survey_scale_average(scales, school, academic_year)
+  def collect_survey_item_average(survey_items:, school:, academic_year:)
-    scales.map { |scale| scale.score(school:, academic_year:) }.average
+    averages = survey_items.map do |survey_item|
+      grouped_responses(school:, academic_year:)[survey_item] || 0
+    end.remove_zeros
+    averages.any? ? averages.average : 0
   end
 
   def collect_admin_scale_average(scales, school, academic_year)
@@ -165,6 +189,14 @@ class Measure < ActiveRecord::Base
     end
   end
 
+  def grouped_responses(school:, academic_year:)
+    @grouped_responses ||= Hash.new do |memo, (school, academic_year)|
+      memo[[school, academic_year]] =
+        SurveyItemResponse.where(school:, academic_year:).group(:survey_item).average(:likert_score)
+    end
+    @grouped_responses[[school, academic_year]]
+  end
+
   def benchmark(name)
     averages = []
     averages << student_survey_items.first.send(name) if includes_student_survey_items?

app/models/student_response_rate.rb

@@ -10,7 +10,7 @@ class StudentResponseRate
                                             measure]).student_survey_items.where("scale.measure": @subcategory.measures)
       survey_items = survey_items.where(on_short_form: true) if survey.form == 'short'
       survey_items = survey_items.reject do |survey_item|
-        survey_item.survey_item_responses.where(school: @school, academic_year: @academic_year).count == 0
+        survey_item.survey_item_responses.where(school: @school, academic_year: @academic_year).none?
       end
       survey_items.count
     end
@@ -34,3 +34,25 @@ class StudentResponseRate
     end
   end
 end
+
+# survey = Survey.where(school:, academic_year:).first
+# total_possible_student_responses = Respondent.where(school:, academic_year:).first
+
+# student_survey_items = Subcategory.all.map do |subcategory|
+#   subcategory.measures.map do |measure|
+#     measure.student_scales.map do |scale|
+#       scale.survey_items.count
+#     end.sum
+#   end.sum
+# end
+# student_response_counts = Subcategory.all.map do |subcategory|
+#   subcategory.measures.map do |measure|
+#     measure.student_survey_items.map do |survey_item|
+#       survey_item.survey_item_responses.where(school:, academic_year:).exclude_boston.count
+#     end.sum
+#   end.sum
+# end
+
+# student_response_counts.each_with_index.map do |value, index|
+#   value.to_f / student_survey_items[index] / total_possible_student_responses * 100
+# end

app/models/teacher_response_rate.rb

@@ -4,7 +4,7 @@ class TeacherResponseRate
   def survey_item_count
     @survey_item_count ||= @subcategory.measures.map do |measure|
       measure.teacher_survey_items.reject do |survey_item|
-        survey_item.survey_item_responses.where(school: @school, academic_year: @academic_year).count == 0
+        survey_item.survey_item_responses.where(school: @school, academic_year: @academic_year).none?
       end.count
     end.sum
   end

app/presenters/analyze_bar_presenter.rb

@@ -61,6 +61,6 @@ class AnalyzeBarPresenter
   def average
     return 0 if score.average.nil?
 
-    score.average.round(2)
+    score.average.round(6)
   end
 end

app/views/analyze/_grouped_bar_column.html.erb

@@ -1,9 +1,10 @@
 <g class="grouped-bar-column" data-for-measure-id="<%= presenter.measure.measure_id  %>">
+<% score_label_y = [5, 10, 15, 5, 10, 15 ]   %>
   <% presenter.bars.each_with_index do |bar, index|  %>
     <rect data-for-academic-year="<%= bar.academic_year.range  %>" x="<%= bar.x_position %>%" y="<%= bar.y_offset %>%" width="<%= presenter.bar_width  %>%" height="<%= bar.bar_height_percentage %>%" fill="<%= bar.color  %>" />
 
     <% if ENV["SCORES"].present?  && ENV["SCORES"].upcase == "SHOW" %>
-      <text x="<%= bar.x_position + 1 %>%" y="5%" text-anchor="middle" dominant-baseline="middle" >
+      <text x="<%= bar.x_position + 3  %>%" y="<%= score_label_y[index]  %>%" text-anchor="middle" dominant-baseline="middle" >
         <%= bar.average %>
       </text>
     <% end %>

app/views/analyze/index.html.erb

@@ -2,12 +2,12 @@
   <h1 class="sub-header-2 color-white m-0"> Analysis of <%= @school.name %> </h1>
 <% end %>
 
-  <div class="graph-content">
+<div class="graph-content">
   <div class="breadcrumbs sub-header-4">
     <%= @category.category_id %>:<%= @category.name %> > <%= @subcategory.subcategory_id  %>:<%= @subcategory.name %>
   </div>
   <hr/>
-  </div>
+</div>
 
 <div class="d-flex flex-row pt-5 row">
   <div class="d-flex flex-column flex-grow-6 bg-color-white col-3 px-5" data-controller="analyze">
@@ -49,6 +49,7 @@
     <% end %>
   </div>
 
+<% cache [@subcategory, @school, @selected_academic_years]  do %>
   <div class="bg-color-white flex-grow-1 col-9">
     <% @measures.each do  |measure|%>
     <section class="mb-6" >
@@ -60,4 +61,6 @@
     </section>
     <% end %>
   </div>
+<% end  %>
+
 </div>

app/views/categories/show.html.erb

@@ -16,6 +16,8 @@
     <% end %>
   </select>
 <% end %>
+
+<% cache [@category, @school, @academic_year] do %>
 <p class="construct-id">Category <%= @category.id %></p>
 <h1 class="sub-header font-bitter color-red"><%= @category.name %></h1>
 <p class="col-8 body-large"><%= @category.description %></p>
@@ -23,3 +25,5 @@
 <% @category.subcategories(academic_year: @academic_year, school: @school).each do |subcategory| %>
   <%= render partial: "subcategory_section", locals: {subcategory: subcategory} %>
 <% end %>
+
+<% end %>

app/views/overview/index.html.erb

@@ -99,8 +99,10 @@
 </div>
 <% end %>
 
-<% if @district == District.find_by_name("Boston")  %>
+<% cache [@district, @school, @academic_year] do %>
+  <% if @district == District.find_by_name("Boston")  %>
     <%= render partial: 'layouts/boston_modal' %>
-<% elsif @has_empty_dataset %>
+  <% elsif @has_empty_dataset %>
     <%= render partial: 'layouts/empty_dataset_modal' %>
+  <% end %>
 <% end  %>

spec/models/measure_spec.rb

@@ -164,13 +164,13 @@ RSpec.describe Measure, type: :model do
                                          growth_low_benchmark: admin_growth_low_benchmark,
                                          approval_low_benchmark: admin_approval_low_benchmark,
                                          ideal_low_benchmark: admin_ideal_low_benchmark)
-        create_list(:student_survey_item, 3, scale:,
+        create_list(:student_survey_item, 3, scale: student_scale,
                                              watch_low_benchmark: student_watch_low_benchmark,
                                              growth_low_benchmark: student_growth_low_benchmark,
                                              approval_low_benchmark: student_approval_low_benchmark,
                                              ideal_low_benchmark: student_ideal_low_benchmark)
 
-        create_list(:teacher_survey_item, 3, scale:,
+        create_list(:teacher_survey_item, 3, scale: teacher_scale,
                                              watch_low_benchmark: teacher_watch_low_benchmark,
                                              growth_low_benchmark: teacher_growth_low_benchmark,
                                              approval_low_benchmark: teacher_approval_low_benchmark,

spec/presenters/grouped_bar_column_presenter_spec.rb

@@ -79,6 +79,7 @@ describe GroupedBarColumnPresenter do
   before do
     create(:respondent, school:, academic_year:, total_students: 1, total_teachers: 1)
     create(:survey, form: :normal, school:, academic_year:)
+    create(:survey, form: :normal, school:, academic_year: another_academic_year)
   end
 
   shared_examples_for 'measure_name' do

spec/system/sqm_application_spec.rb

@@ -6,7 +6,7 @@ describe 'SQM Application' do
   let(:academic_year) { create(:academic_year) }
   let(:category) { create(:category) }
   let(:measure) { create(:measure) }
-  let(:scale) { create(:scale, measure:) }
+  let(:scale) { create(:teacher_scale, measure:) }
 
   before :each do
     driven_by :rack_test