Update logic for calculating student response rate. Remove references

to survey table.  We no longer check or keep track of the survey type.
Instead we look in the database to see if a survey item has at least 10
responses.  If it does, that survey item was presented to the respondent
and we count it, and all responses when calculating the response rate.

Remove response rate timestamp from caching logic because we no longer
add the response rate to the database. All response rates are calculated
on the fly

Update three_b_two scraper to use teacher only numbers

swap over to using https://profiles.doe.mass.edu/statereport/gradesubjectstaffing.aspx as the source of staffing information

app/controllers/analyze_controller.rb

@@ -2,7 +2,7 @@
 
 class AnalyzeController < SqmApplicationController
   before_action :assign_categories, :assign_subcategories, :assign_measures, :assign_academic_years,
-                :response_rate_timestamp, :races, :selected_races, :graph, :graphs, :background, :race_score_timestamp,
+                :races, :selected_races, :graph, :graphs, :background, :race_score_timestamp,
                 :source, :sources, :group, :groups, :selected_grades, :grades, :slice, :selected_genders, :genders, only: [:index]
   def index; end
 
@@ -35,18 +35,6 @@ class AnalyzeController < SqmApplicationController
     end
   end
 
-  def response_rate_timestamp
-    @response_rate_timestamp = begin
-      academic_year = @selected_academic_years.last
-      academic_year ||= @academic_year
-      rate = ResponseRate.where(school: @school,
-                                academic_year:).order(updated_at: :DESC).first || Today.new
-
-      rate.updated_at
-    end
-    @response_rate_timestamp
-  end
-
   def races
     @races ||= Race.all.order(designation: :ASC)
   end

app/controllers/categories_controller.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class CategoriesController < SqmApplicationController
-  before_action :response_rate_timestamp, only: [:index]
   helper GaugeHelper
 
   def show
@@ -9,16 +8,4 @@ class CategoriesController < SqmApplicationController
 
     @category = CategoryPresenter.new(category: Category.find_by_slug(params[:id]))
   end
-
-  private
-
-  def response_rate_timestamp
-    @response_rate_timestamp = begin
-      rate = ResponseRate.where(school: @school,
-                                academic_year: @academic_year).order(updated_at: :DESC).first || Today.new
-
-      rate.updated_at
-    end
-    @response_rate_timestamp
-  end
 end

app/controllers/overview_controller.rb

@@ -2,7 +2,6 @@
 
 class OverviewController < SqmApplicationController
   before_action :check_empty_dataset, only: [:index]
-  before_action :response_rate_timestamp, only: [:index]
   helper VarianceHelper
 
   def index
@@ -32,14 +31,4 @@ class OverviewController < SqmApplicationController
   def subcategories
     @subcategories ||= Subcategory.all
   end
-
-  def response_rate_timestamp
-    @response_rate_timestamp = begin
-      rate = ResponseRate.where(school: @school,
-                                academic_year: @academic_year).order(updated_at: :DESC).first || Today.new
-
-      rate.updated_at
-    end
-    @response_rate_timestamp
-  end
 end

app/controllers/sqm_application_controller.rb

@@ -3,7 +3,6 @@
 class SqmApplicationController < ApplicationController
   protect_from_forgery with: :exception, prepend: true
   before_action :set_schools_and_districts
-  before_action :response_rate_timestamp
   helper HeaderHelper
 
   private

app/models/measure.rb

@@ -51,16 +51,6 @@ class Measure < ActiveRecord::Base
     @includes_admin_data_items ||= admin_data_items.any?
   end
 
-  # def sources
-  #   @sources ||= begin
-  #     sources = []
-  #     sources << Source.new(name: :admin_data, collection: admin_data_items) if includes_admin_data_items?
-  #     sources << Source.new(name: :student_surveys, collection: student_survey_items) if includes_student_survey_items?
-  #     sources << Source.new(name: :teacher_surveys, collection: teacher_survey_items) if includes_teacher_survey_items?
-  #     sources
-  #   end
-  # end
-
   def score(school:, academic_year:)
     @score ||= Hash.new do |memo, (school, academic_year)|
       next Score::NIL_SCORE if incalculable_score(school:, academic_year:)
@@ -72,7 +62,6 @@ class Measure < ActiveRecord::Base
 
       memo[[school, academic_year]] = scorify(average:, school:, academic_year:)
     end
-
     @score[[school, academic_year]]
   end
 
@@ -212,18 +201,16 @@ class Measure < ActiveRecord::Base
 
   def no_student_responses_exist?(school:, academic_year:)
     @no_student_responses_exist ||= Hash.new do |memo, (school, academic_year)|
-      memo[[school, academic_year]] = student_survey_items_by_survey_type(school:, academic_year:).all? do |survey_item|
+      memo[[school, academic_year]] =
-        survey_item.survey_item_responses.where(school:, academic_year:).none?
+        SurveyItemResponse.where(school:, academic_year:, survey_item: survey_items.student_survey_items).count.zero?
-      end
     end
     @no_student_responses_exist[[school, academic_year]]
   end
 
   def no_teacher_responses_exist?(school:, academic_year:)
     @no_teacher_responses_exist ||= Hash.new do |memo, (school, academic_year)|
-      memo[[school, academic_year]] = teacher_survey_items.all? do |survey_item|
+      memo[[school, academic_year]] =
-        survey_item.survey_item_responses.where(school:, academic_year:).none?
+        SurveyItemResponse.where(school:, academic_year:, survey_item: survey_items.teacher_survey_items).count.zero?
-      end
     end
     @no_teacher_responses_exist[[school, academic_year]]
   end

app/models/respondent.rb

@@ -5,4 +5,15 @@ class Respondent < ApplicationRecord
   belongs_to :academic_year
 
   validates :school, uniqueness: { scope: :academic_year }
+
+  def counts_by_grade
+    @counts_by_grade ||= {}.tap do |row|
+      attributes = %i[pk k one two three four five six seven eight nine ten eleven twelve]
+      grades = [-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
+      attributes.zip(grades).each do |attribute, grade|
+        count = send(attribute) if send(attribute).present?
+        row[grade] = count unless count.nil? || count.zero?
+      end
+    end
+  end
 end

app/models/response_rate_calculator.rb

@@ -14,7 +14,7 @@ class ResponseRateCalculator
   def rate
     return 100 if population_data_unavailable?
 
-    return 0 unless survey_item_count.positive?
+    return 0 unless survey_items_have_sufficient_responses?
 
     return 0 unless total_possible_responses.positive?
 
@@ -35,10 +35,6 @@ class ResponseRateCalculator
     response_rate > 100 ? 100 : response_rate
   end
 
-  def survey
-    Survey.find_by(school:, academic_year:)
-  end
-
   def average_responses_per_survey_item
     response_count / survey_item_count.to_f
   end

app/models/student_response_rate_calculator.rb

@@ -1,69 +1,54 @@
 # frozen_string_literal: true
 
 class StudentResponseRateCalculator < ResponseRateCalculator
-  private
-
   def raw_response_rate
-    # def rate
+    rates_by_grade.length.positive? ? rates_by_grade.average : 0
-    # check to see if enrollment data is available
+  end
-    #   if not, run the dese loader to get the data
-    #   then upload the enrollment data into the db
-    #
-    # if you still don't see enrollment for the school, raise an error and return 100 from this method
-    #
-    #  Get the enrollment information from the db
-    #  Get the list of all grades
-    #  For each grade, get the survey items with data
-    #
-    #
-    #  All methods below will need to specify a grade
 
-    grades_with_sufficient_responses.map do |grade|
+  def rates_by_grade
-      puts "Grade: #{grade}"
+    @rates_by_grade ||= counts_by_grade.map do |grade, num_of_students_in_grade|
+      sufficient_survey_items = survey_items_with_sufficient_responses(grade:).keys
+      actual_response_count_for_grade = SurveyItemResponse.where(school:, academic_year:, grade:,
+                                                                 survey_item: sufficient_survey_items).count.to_f
+      count_of_survey_items_with_sufficient_responses = survey_item_count(grade:)
+      if count_of_survey_items_with_sufficient_responses.nil? || count_of_survey_items_with_sufficient_responses.zero? || num_of_students_in_grade.nil? || num_of_students_in_grade.zero?
+        next nil
       end
-    (average_responses_per_survey_item / total_possible_responses.to_f * 100).round
+
+      actual_response_count_for_grade / count_of_survey_items_with_sufficient_responses / num_of_students_in_grade * 100
+    end.compact
   end
 
-  def survey_item_count
+  def counts_by_grade
-    @survey_item_count ||= begin
+    @counts_by_grade ||= respondents.counts_by_grade
-      survey_items = SurveyItem.includes(%i[scale
-                                            measure]).student_survey_items.where("scale.measure": @subcategory.measures)
-      survey_items = survey_items.where(on_short_form: true) if survey.form == 'short'
-      survey_items = survey_items.reject do |survey_item|
-        survey_item.survey_item_responses.where(school:, academic_year:).none?
   end
-      survey_items.count
+
+  def survey_items_have_sufficient_responses?
+    rates_by_grade.length.positive?
   end
+
+  def survey_items_with_sufficient_responses(grade:)
+    SurveyItem.joins('inner join survey_item_responses on  survey_item_responses.survey_item_id = survey_items.id')
+              .student_survey_items
+              .where("survey_item_responses.school": school, "survey_item_responses.academic_year": academic_year, "survey_item_responses.grade": grade, "survey_item_responses.survey_item_id": subcategory.survey_items.student_survey_items)
+              .group('survey_items.id')
+              .having('count(*) >= 10')
+              .count
   end
 
-  def response_count
+  def survey_item_count(grade:)
-    @response_count ||= @subcategory.measures.map do |measure|
+    survey_items_with_sufficient_responses(grade:).count
-      measure.student_survey_items.map do |survey_item|
+  end
-        next 0 if survey.form == 'short' && survey_item.on_short_form == false
 
-        survey_item.survey_item_responses.where(school:,
+  def respondents
-                                                academic_year:).exclude_boston.count
+    @respondents ||= Respondent.find_by(school:, academic_year:)
-      end.sum
-    end.sum
   end
 
   def total_possible_responses
     @total_possible_responses ||= begin
-      total_responses = Respondent.find_by(school:, academic_year:)
+      return 0 unless respondents.present?
-      return 0 unless total_responses.present?
 
-      total_responses.total_students
+      respondents.total_students
     end
   end
-
-  def grades_with_sufficient_responses
-    SurveyItemResponse.where(school:, academic_year:,
-                             survey_item: subcategory.survey_items.student_survey_items).where.not(grade: nil)
-                      .group(:grade)
-                      .select(:response_id)
-                      .distinct(:response_id)
-                      .count.reject do |_key, value|
-      value < 10
-    end.keys
-  end
 end

app/models/subcategory.rb

@@ -16,19 +16,13 @@ class Subcategory < ActiveRecord::Base
 
   def response_rate(school:, academic_year:)
     @response_rate ||= Hash.new do |memo, (school, academic_year)|
-      memo[[school, academic_year]] = ResponseRate.find_by(subcategory: self, school:, academic_year:)
-    end
-
-    @response_rate[[school, academic_year]] || create_response_rate(school:, academic_year:)
-  end
-
-  private
-
-  def create_response_rate(school:, academic_year:)
       student = StudentResponseRateCalculator.new(subcategory: self, school:, academic_year:)
       teacher = TeacherResponseRateCalculator.new(subcategory: self, school:, academic_year:)
-    ResponseRate.create(school:, academic_year:, subcategory: self, student_response_rate: student.rate,
+      memo[[school, academic_year]] = ResponseRate.new(school:, academic_year:, subcategory: self, student_response_rate: student.rate,
                                                        teacher_response_rate: teacher.rate, meets_student_threshold: student.meets_student_threshold?,
                                                        meets_teacher_threshold: teacher.meets_teacher_threshold?)
     end
+
+    @response_rate[[school, academic_year]]
+  end
 end

app/models/survey_item.rb

@@ -16,16 +16,16 @@ class SurveyItem < ActiveRecord::Base
   end
 
   scope :student_survey_items, lambda {
-    where("survey_item_id LIKE 's-%'")
+    where("survey_items.survey_item_id LIKE 's-%'")
   }
   scope :teacher_survey_items, lambda {
-    where("survey_item_id LIKE 't-%'")
+    where("survey_items.survey_item_id LIKE 't-%'")
   }
   scope :short_form_items, lambda {
     where(on_short_form: true)
   }
   scope :early_education_surveys, lambda {
-    where("survey_item_id LIKE '%-%-es%'")
+    where("survey_items.survey_item_id LIKE '%-%-es%'")
   }
 
   scope :survey_items_for_grade, lambda { |school, academic_year, grade|

app/models/survey_item_response.rb

@@ -2,7 +2,7 @@
 
 class SurveyItemResponse < ActiveRecord::Base
   TEACHER_RESPONSE_THRESHOLD = 2
-  STUDENT_RESPONSE_THRESHOLD = 2
+  STUDENT_RESPONSE_THRESHOLD = 10
 
   belongs_to :academic_year
   belongs_to :school

app/models/teacher_response_rate_calculator.rb

@@ -9,6 +9,10 @@ class TeacherResponseRateCalculator < ResponseRateCalculator
     end.sum
   end
 
+  def survey_items_have_sufficient_responses?
+    survey_item_count.positive?
+  end
+
   def response_count
     @response_count ||= @subcategory.measures.map do |measure|
       measure.teacher_survey_items.map do |survey_item|

app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-# TODO: resize bars so they never extend beyond the bounds of the column
 module Analyze
   module Graph
     module Column

app/presenters/subcategory_presenter.rb

@@ -27,13 +27,13 @@ class SubcategoryPresenter
   def student_response_rate
     return 'N / A' if Respondent.where(school: @school, academic_year: @academic_year).count.zero?
 
-    "#{@subcategory.response_rate(school: @school, academic_year: @academic_year).student_response_rate.to_i}%"
+    "#{@subcategory.response_rate(school: @school, academic_year: @academic_year).student_response_rate.round}%"
   end
 
   def teacher_response_rate
     return 'N / A' if Respondent.where(school: @school, academic_year: @academic_year).count.zero?
 
-    "#{@subcategory.response_rate(school: @school, academic_year: @academic_year).teacher_response_rate.to_i}%"
+    "#{@subcategory.response_rate(school: @school, academic_year: @academic_year).teacher_response_rate.round}%"
   end
 
   def admin_collection_rate

app/services/dese/scraper.rb

@@ -35,7 +35,7 @@ module Dese
       browser.goto(url)
 
       selectors.each do |key, value|
-        return unless browser.option(text: value).present?
+        next unless browser.option(text: value).present?
 
         browser.select(id: key).select(text: value)
       end

app/services/dese/staffing.rb

@@ -0,0 +1,35 @@
+require 'watir'
+
+module Dese
+  class Staffing
+    include Dese::Scraper
+    attr_reader :filepath
+
+    def initialize(filepath: Rails.root.join('data', 'staffing', 'staffing.csv'))
+      @filepath = filepath
+    end
+
+    def run_all
+      scrape_staffing(filepath:)
+    end
+
+    def scrape_staffing(filepath:)
+      headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year',
+                 'School Name', 'DESE ID',
+                 'PK-2 (#)', '3-5 (#)', '6-8 (#)', '9-12 (#)', 'Multiple Grades (#)',
+                 'All Grades (#)', 'FTE Count']
+      write_headers(filepath:, headers:)
+      run do |academic_year|
+        admin_data_item_id = 'NA'
+        url = 'https://profiles.doe.mass.edu/statereport/gradesubjectstaffing.aspx'
+        range = academic_year.range
+        selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
+                      'ctl00_ContentPlaceHolder1_ddYear' => range,
+                      'ctl00_ContentPlaceHolder1_ddDisplay' => 'Full-time Equivalents' }
+        submit_id = 'btnViewReport'
+        calculation = ->(_headers, _items) { 'NA' }
+        Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
+      end
+    end
+  end
+end

app/services/dese/three_b_two.rb

@@ -42,7 +42,8 @@ module Dese
         range = academic_year.range
         selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
                       'ctl00_ContentPlaceHolder1_ddYear' => range,
-                      'ctl00_ContentPlaceHolder1_ddDisplay' => 'Percentages' }
+                      'ctl00_ContentPlaceHolder1_ddDisplay' => 'Percentages',
+                      'ctl00_ContentPlaceHolder1_ddClassification' => 'Teacher' }
         submit_id = 'ctl00_ContentPlaceHolder1_btnViewReport'
         calculation = lambda { |headers, items|
           african_american_index = headers['African American (%)']

app/services/response_rate_loader.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-class ResponseRateLoader
-  def self.reset(schools: School.all, academic_years: AcademicYear.all, subcategories: Subcategory.all)
-    subcategories.each do |subcategory|
-      schools.each do |school|
-        academic_years.each do |academic_year|
-          process_response_rate(subcategory:, school:, academic_year:)
-        end
-      end
-    end
-  end
-
-  private
-
-  def self.process_response_rate(subcategory:, school:, academic_year:)
-    student = StudentResponseRateCalculator.new(subcategory:, school:, academic_year:)
-    teacher = TeacherResponseRateCalculator.new(subcategory:, school:, academic_year:)
-
-    response_rate = ResponseRate.find_or_create_by!(subcategory:, school:, academic_year:)
-
-    response_rate.update!(student_response_rate: student.rate,
-                          teacher_response_rate: teacher.rate,
-                          meets_student_threshold: student.meets_student_threshold?,
-                          meets_teacher_threshold: teacher.meets_teacher_threshold?)
-  end
-
-  private_class_method :process_response_rate
-end

app/services/staffing_loader.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-# TODO
 require 'csv'
 
 class StaffingLoader

app/views/analyze/index.html.erb

@@ -1,34 +1,27 @@
 <% content_for :title do %>
   <h1 class="sub-header-2 color-white m-0"> Analysis of <%= @school.name %> </h1>
 <% end %>
-
 <div class="graph-content">
   <div class="breadcrumbs sub-header-4">
     <%= @category.category_id %>:<%= @category.name %> > <%= @subcategory.subcategory_id %>:<%= @subcategory.name %>
   </div>
   <hr>
 </div>
-
 <div class="d-flex flex-row pt-5 row">
   <div class="d-flex flex-column flex-grow-6 bg-color-white col-3 px-5" data-controller="analyze">
     <%= render partial: "focus_area", locals: {categories: @categories, district: @district, school: @school, academic_year: @academic_year, category: @category,  subcategories: @subcategories} %>
-
     <%= render partial: "school_years", locals: {available_academic_years: @available_academic_years, selected_academic_years: @selected_academic_years, district: @district, school: @school, academic_year: @academic_year, category: @category, subcategory: @subcategory, measures: @measures} %>
-
     <%= render partial: "data_filters", locals: {district: @district, school: @school, academic_year: @academic_year, category: @category, subcategory: @subcategory} %>
   </div>
-
+  <% cache [@subcategory, @school, @selected_academic_years,  @graph, @selected_races, @race_score_timestamp, @selected_grades, @grades, @selected_genders, @genders]  do %>
-  <% cache [@subcategory, @school, @selected_academic_years, @response_rate_timestamp, @graph, @selected_races, @race_score_timestamp, @selected_grades, @grades, @selected_genders, @genders]  do %>
     <div class="bg-color-white flex-grow-1 col-9">
       <% @measures.each do  |measure| %>
         <section class="mb-6">
           <p class="construct-id">Measure <%= measure.measure_id %></p>
           <h2> <%= measure.name %> </h2>
-
           <%= render partial: "grouped_bar_chart" , locals: { measure: measure} %>
         </section>
       <% end %>
     </div>
   <% end %>
-
 </div>

app/views/categories/show.html.erb

@@ -9,21 +9,17 @@
       </div>
     <% end %>
   </nav>
-
   <select id="select-academic-year" class="form-select" name="academic-year">
     <% @academic_years.each do |year| %>
       <option value="<%= url_for [@district, @school, @category , {year: year.range} ]%>" <%= @academic_year == year ? "selected" : nil %>><%= year.formatted_range %></option>
       <% end %>
     </select>
   <% end %>
-
   <% cache [@category, @school, @academic_year] do %>
     <p class="construct-id">Category <%= @category.id %></p>
     <h1 class="sub-header font-bitter color-red"><%= @category.name %></h1>
     <p class="col-8 body-large"><%= @category.description %></p>
-
     <% @category.subcategories(academic_year: @academic_year, school: @school).each do |subcategory| %>
       <%= render partial: "subcategory_section", locals: {subcategory: subcategory} %>
     <% end %>
-
   <% end %>

app/views/overview/index.html.erb

@@ -1,16 +1,13 @@
 <% content_for :navigation do %>
   <h2 class="sub-header-2 color-white m-0">Areas Of Interest</h2>
-
   <select id="select-academic-year" class="form-select" name="academic-year">
     <%  @academic_years.each do |year| %>
       <option value="<%= district_school_overview_index_path(@district, @school,  {year: year.range}) %>" <%= @academic_year == year ? "selected" : nil %>><%= year.formatted_range %></option>
       <% end %>
     </select>
   <% end  %>
-
   <% cache do   %>
     <svg class="d-none">
-
       <symbol viewBox="0 0 24 24" id="warning-harvey-ball">
         <circle cx="12" cy="12" r="11.5" fill="white" stroke="none" />
         <path d="
@@ -22,7 +19,6 @@
         />
         <circle cx="12" cy="12" r="11.5" fill="none" />
       </symbol>
-
       <symbol viewBox="0 0 24 24" id="watch-harvey-ball">
         <circle cx="12" cy="12" r="11.5" fill="white" stroke="none" />
         <path d="
@@ -34,7 +30,6 @@
         />
         <circle cx="12" cy="12" r="11.5" fill="none" />
       </symbol>
-
       <symbol viewBox="0 0 24 24" id="growth-harvey-ball">
         <circle cx="12" cy="12" r="11.5" fill="white" stroke="none" />
         <path d="
@@ -46,30 +41,25 @@
         />
         <circle cx="12" cy="12" r="11.5" fill="none" />
       </symbol>
-
       <symbol viewBox="0 0 24 24" id="approval-harvey-ball">
         <circle cx="12" cy="12" r="11.5" />
         <path d="M19 8C19 8.28125 18.875 8.53125 18.6875 8.71875L10.6875 16.7188C10.5 16.9062 10.25 17 10 17C9.71875 17 9.46875 16.9062 9.28125 16.7188L5.28125 12.7188C5.09375 12.5312 5 12.2812 5 12C5 11.4375 5.4375 11 6 11C6.25 11 6.5 11.125 6.6875 11.3125L10 14.5938L17.2812 7.3125C17.4688 7.125 17.7188 7 18 7C18.5312 7 19 7.4375 19 8Z"
             stroke-width=".5" stroke="white" fill="white" />
       </symbol>
-
       <symbol viewBox="0 0 24 24" id="ideal-harvey-ball">
         <circle cx="12" cy="12" r="11.5" />
         <path d="M9.28125 11.7188C9.46875 11.9062 9.71875 12 10 12C10.25 12 10.5 11.9062 10.6875 11.7188L15.6875 6.71875C15.875 6.53125 16 6.28125 16 6C16 5.4375 15.5312 5 15 5C14.7188 5 14.4688 5.125 14.2812 5.3125L10 9.59375L8.1875 7.8125C8 7.625 7.75 7.5 7.5 7.5C6.9375 7.5 6.5 7.9375 6.5 8.5C6.5 8.78125 6.59375 9.03125 6.78125 9.21875L9.28125 11.7188ZM19 10C19 9.4375 18.5312 9 18 9C17.7188 9 17.4688 9.125 17.2812 9.3125L10 16.5938L6.6875 13.3125C6.5 13.125 6.25 13 6 13C5.4375 13 5 13.4375 5 14C5 14.2812 5.09375 14.5312 5.28125 14.7188L9.28125 18.7188C9.46875 18.9062 9.71875 19 10 19C10.25 19 10.5 18.9062 10.6875 18.7188L18.6875 10.7188C18.875 10.5312 19 10.2812 19 10Z"
             stroke-width=".5" stroke="white" fill="white" />
       </symbol>
-
       <symbol viewBox="0 0 24 24" id="insufficient_data-harvey-ball">
         <circle cx="12" cy="12" r="11.5" />
       </symbol>
     </svg>
   <% end  %>
-
+  <% cache [@school, @academic_year] do %>
-<% cache [@school, @academic_year, @response_rate_timestamp] do %>
     <div class="card">
       <div class="d-flex justify-content-between align-items-center">
         <h2 class="sub-header-2">School Quality Framework Indicators</h2>
-
         <div class="harvey-ball-legend">
           <div class="font-size-14">Warning</div>
           <svg class="ms-3 me-1" width="16" height="16" xmlns="http://www.w3.org/2000/svg">
@@ -90,16 +80,12 @@
           <div class="font-size-14">Ideal</div>
         </div>
       </div>
-
       <%= render partial: "quality_framework_indicators", locals: { category_presenters: @category_presenters }, cached: true %>
     </div>
-
     <div class="card">
       <h2 class="sub-header-2 mb-4">Distance From Benchmark</h2>
-
       <%= render partial: "variance_chart", locals: { presenters: @variance_chart_row_presenters } , cached: true %>
     </div>
-
     <% if @district == District.find_by_name("Boston")  %>
       <%= render partial: 'layouts/boston_modal' %>
     <% elsif @has_empty_dataset %>

lib/tasks/scrape.rake

@@ -16,6 +16,6 @@ namespace :scrape do
 
   desc 'scrape dese site for student staffing information'
   task staffing: :environment do
-    Dese::OneAThree.new(filepaths: ['not used', Rails.root.join('data', 'staffing', 'staffing.csv')]).run_a_pcom_i3
+    Dese::Staffing.new.run_all
   end
 end

spec/factories.rb

@@ -160,6 +160,7 @@ FactoryBot.define do
   factory :survey_item_response do
     likert_score { 3 }
     response_id { rand.to_s }
+    grade { 1 }
     academic_year
     school
     survey_item factory: :teacher_survey_item
@@ -174,6 +175,8 @@ FactoryBot.define do
   factory :respondent do
     school
     academic_year
+    one { 40 }
+
     total_students { SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD * 4 }
     total_teachers { SurveyItemResponse::TEACHER_RESPONSE_THRESHOLD * 4 }
   end

spec/models/measure_spec.rb

@@ -26,7 +26,7 @@ RSpec.describe Measure, type: :model do
   let(:teacher_ideal_low_benchmark) { 4.2 }
 
   before do
-    create(:respondent, school:, academic_year:)
+    create(:respondent, school:, academic_year:, one: 40)
     create(:survey, school:, academic_year:)
     create(:respondent, school: short_form_school, academic_year:)
     create(:survey, school: short_form_school, academic_year:, form: 'short')
@@ -369,27 +369,6 @@ RSpec.describe Measure, type: :model do
           expect(measure.score(school:, academic_year:).meets_student_threshold?).to be false
         end
       end
-
-      context 'and the school is a short form school' do
-        before :each do
-          create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD,
-                      survey_item: student_survey_item_1, academic_year:, school: short_form_school, likert_score: 1)
-          create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD,
-                      survey_item: student_survey_item_2, academic_year:, school: short_form_school, likert_score: 1)
-          create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD,
-                      survey_item: student_survey_item_3, academic_year:, school: short_form_school, likert_score: 1)
-          create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD,
-                      survey_item: short_form_student_survey_item_1, academic_year:, school: short_form_school, likert_score: 3)
-          create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD,
-                      survey_item: short_form_student_survey_item_2, academic_year:, school: short_form_school, likert_score: 4)
-          create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD,
-                      survey_item: short_form_student_survey_item_3, academic_year:, school: short_form_school, likert_score: 5)
-        end
-
-        it 'ignores any responses not on the short form and gives the average of short form survey items' do
-          expect(measure.score(school: short_form_school, academic_year:).average).to eq 4
-        end
-      end
     end
 
     context 'when the measure includes both teacher and student data' do
@@ -475,7 +454,7 @@ RSpec.describe Measure, type: :model do
           create_list(:survey_item_response, SurveyItemResponse::TEACHER_RESPONSE_THRESHOLD - 1,
                       survey_item: teacher_survey_item_1, academic_year:, school:, likert_score: 1)
           create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD,
-                      survey_item: student_survey_item_1, academic_year:, school:, likert_score: 5)
+                      survey_item: student_survey_item_1, academic_year:, school:, likert_score: 5, grade: 1)
         end
 
         it 'returns the average of the likert scores of the student survey items' do

spec/models/pillar_spec.rb

@@ -42,9 +42,9 @@ RSpec.describe Report::Pillar, type: :model do
 
   context '.score' do
     before do
-      create(:survey_item_response, survey_item: survey_item_1, school:, academic_year: academic_year_1,
+      create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: survey_item_1, school:, academic_year: academic_year_1,
                                                                                          likert_score: 3)
-      create(:survey_item_response, survey_item: survey_item_1, school:, academic_year: academic_year_1,
+      create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: survey_item_1, school:, academic_year: academic_year_1,
                                                                                          likert_score: 5)
     end
     it 'returns the average score for all the measures in the pillar' do
@@ -56,13 +56,13 @@ RSpec.describe Report::Pillar, type: :model do
 
   context '.zone' do
     before do
-      create(:survey_item_response, survey_item: survey_item_1, school:, academic_year: academic_year_1,
+      create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: survey_item_1, school:, academic_year: academic_year_1,
                                                                                          likert_score: 4)
-      create(:survey_item_response, survey_item: survey_item_1, school:, academic_year: academic_year_1,
+      create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: survey_item_1, school:, academic_year: academic_year_1,
                                                                                          likert_score: 5)
-      create(:survey_item_response, survey_item: survey_item_2, school:, academic_year: academic_year_1,
+      create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: survey_item_2, school:, academic_year: academic_year_1,
                                                                                          likert_score: 4)
-      create(:survey_item_response, survey_item: survey_item_2, school:, academic_year: academic_year_1,
+      create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: survey_item_2, school:, academic_year: academic_year_1,
                                                                                          likert_score: 5)
     end
 

spec/models/respondent_spec.rb

@@ -0,0 +1,17 @@
+require 'rails_helper'
+
+RSpec.describe Respondent, type: :model do
+  describe 'grade_counts' do
+    let(:single_grade_of_respondents) { create(:respondent, one: 10) }
+    let(:two_grades_of_respondents) { create(:respondent, pk: 10, k: 5, one: 0, two: 0, three: 0) }
+    let(:three_grades_of_respondents) { create(:respondent, one: 10, two: 5, twelve: 6, eleven: 0) }
+    context 'when the student respondents include one or more counts for the number of respondents' do
+      it 'returns a hash with only the grades that have a non-zero count of students' do
+        expect(single_grade_of_respondents.one).to eq(10)
+        expect(single_grade_of_respondents.counts_by_grade).to eq({ 1 => 10 })
+        expect(two_grades_of_respondents.counts_by_grade).to eq({ -1 => 10, 0 => 5 })
+        expect(three_grades_of_respondents.counts_by_grade).to eq({ 1 => 10, 2 => 5, 12 => 6 })
+      end
+    end
+  end
+end

spec/models/response_rate_calculator_spec.rb

@@ -4,11 +4,10 @@ describe ResponseRateCalculator, type: :model do
   let(:school) { create(:school) }
   let(:academic_year) { create(:academic_year) }
   let(:survey) { create(:survey, school:, academic_year:) }
-  let(:short_form_survey) { create(:survey, form: :short, school:, academic_year:) }
-  let(:respondent) { create(:respondent, school:, academic_year:) }
 
   describe StudentResponseRateCalculator do
     let(:subcategory) { create(:subcategory) }
+    let(:second_subcategory) { create(:subcategory_with_measures) }
     let(:sufficient_measure_1) { create(:measure, subcategory:) }
     let(:sufficient_scale_1) { create(:scale, measure: sufficient_measure_1) }
     let(:sufficient_measure_2) { create(:measure, subcategory:) }
@@ -17,123 +16,177 @@ describe ResponseRateCalculator, type: :model do
     let(:sufficient_student_survey_item_1) { create(:student_survey_item, scale: sufficient_scale_1) }
     let(:insufficient_student_survey_item_1) { create(:student_survey_item, scale: sufficient_scale_1) }
     let(:sufficient_student_survey_item_2) { create(:student_survey_item, scale: sufficient_scale_2) }
+    let(:sufficient_student_survey_item_3) { create(:student_survey_item, scale: sufficient_scale_2) }
 
-    context '.grades_with_sufficient_responses' do
+    context '.raw_response_rate' do
-      pending 'implement this'
+      context 'when no survey item responses exist' do
-      before :each do
+        before do
+          create(:respondent, school:, academic_year:, pk: 20)
         end
+        it 'returns an average of the response rates for all grades' do
+          expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 0
         end
-    context 'when a students take a regular survey' do
+
-      context 'when the average number of student responses per question in a subcategory is equal to the student response threshold' do
+        context 'or when the count of survey items does not meet the minimum threshold' do
-        before :each do
+          before do
-          create_list(:survey_item_response, SurveyItemResponse::TEACHER_RESPONSE_THRESHOLD, survey_item: sufficient_teacher_survey_item,
+            create_list(:survey_item_response, 9, survey_item: sufficient_student_survey_item_1, academic_year:,
-                                                                                             academic_year:, school:, likert_score: 1)
+                                                  school:, grade: 1)
-          create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: sufficient_student_survey_item_1,
+          end
-                                                                                             academic_year:, school:, likert_score: 4)
+          it 'returns an average of the response rates for all grades' do
-          create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: sufficient_student_survey_item_2,
+            expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 0
-                                                                                             academic_year:, school:, likert_score: 4)
+          end
-          respondent
+        end
-          survey
+      end
+
+      context 'when at least one survey item has sufficient responses' do
+        before do
+          create(:respondent, school:, academic_year:, total_students: 20, one: 20)
+          create_list(:survey_item_response, 10, survey_item: sufficient_student_survey_item_1, academic_year:,
+                                                 school:, grade: 1)
         end
 
-        it 'returns a response rate equal to the response threshold' do
+        context 'and half of students responded' do
+          it 'reports a response rate of fifty percent' do
             expect(StudentResponseRateCalculator.new(subcategory:, school:,
-                                                   academic_year:).rate).to eq 25
+                                                     academic_year:).rate).to eq 50
           end
         end
 
-      context 'when the average number of student responses per question is below the student threshold' do
+        context 'and another unrelated subcategory has responses' do
-        before :each do
+          before do
-          create_list(:survey_item_response, 1, survey_item: sufficient_student_survey_item_1,
+            create_list(:survey_item_response, 10,
-                                                academic_year:, school:, likert_score: 4)
+                        survey_item: second_subcategory.measures.first.scales.first.survey_items.first, academic_year:, school:, grade: 1)
-          create_list(:survey_item_response, 1, survey_item: sufficient_student_survey_item_2,
-                                                academic_year:, school:, likert_score: 4)
-          respondent
-          survey
           end
 
-        it 'reports insufficient student responses' do
+          it 'does not count the responses for the unrelated subcategory' do
-          expect(StudentResponseRateCalculator.new(subcategory:, school:,
-                                                   academic_year:).rate).to eq 13
             expect(StudentResponseRateCalculator.new(subcategory:, school:,
-                                                   academic_year:).meets_student_threshold?).to eq false
+                                                     academic_year:).rate).to eq 50
+          end
+        end
+
+        context 'there are responses for another survey item but not enough to meet the minimum threshold' do
+          before do
+            create_list(:survey_item_response, 9, survey_item: insufficient_student_survey_item_1, academic_year:,
+                                                  school:, grade: 1)
+          end
+          it 'returns an average of the response rates for all grades' do
+            expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 50
           end
         end
       end
 
-    context 'when students take the short form survey' do
+      context 'when two survey items have sufficient responses' do
-      before :each do
+        before do
-        create_list(:survey_item_response, SurveyItemResponse::TEACHER_RESPONSE_THRESHOLD, survey_item: sufficient_teacher_survey_item,
+          create(:respondent, school:, academic_year:, total_students: 20, one: 20)
-                                                                                           academic_year:, school:, likert_score: 1)
+          create_list(:survey_item_response, 10, survey_item: sufficient_student_survey_item_1, academic_year:,
-        create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: sufficient_student_survey_item_1,
+                                                 school:, grade: 1)
-                                                                                           academic_year:, school:, likert_score: 4)
+          create_list(:survey_item_response, 20, survey_item: sufficient_student_survey_item_2, academic_year:,
-        create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: sufficient_student_survey_item_2,
+                                                 school:, grade: 1)
-                                                                                           academic_year:, school:, likert_score: 4)
-        respondent
-        short_form_survey
         end
 
-      context 'when the average number of student responses per question in a subcategory is equal to the student response threshold' do
+        context 'one one question got half the students to respond and the other got all the students to respond' do
-        before :each do
+          it 'reports a response rate that averages fifty and 100' do
-          sufficient_student_survey_item_1.update! on_short_form: true
+            expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 75
-          sufficient_student_survey_item_2.update! on_short_form: true
+          end
         end
 
-        it 'takes into account the responses from both survey items' do
+        context 'and another unrelated subcategory has responses' do
-          expect(StudentResponseRateCalculator.new(subcategory:, school:,
+          before do
-                                                   academic_year:).rate).to eq 25
+            create_list(:survey_item_response, 10,
+                        survey_item: second_subcategory.measures.first.scales.first.survey_items.first, academic_year:, school:, grade: 1)
           end
 
-        context 'and only one of the survey items is on the short form' do
+          it 'does not count the responses for the unrelated subcategory' do
+            expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 75
+          end
+        end
+      end
+
+      context 'when there survey items between two scales' do
         before do
-            sufficient_student_survey_item_2.update! on_short_form: false
+          create(:respondent, school:, academic_year:, total_students: 20, one: 20)
+          create_list(:survey_item_response, 20, survey_item: sufficient_student_survey_item_1, academic_year:,
+                                                 school:, grade: 1)
+          create_list(:survey_item_response, 15, survey_item: sufficient_student_survey_item_2, academic_year:,
+                                                 school:, grade: 1)
+          create_list(:survey_item_response, 10, survey_item: sufficient_student_survey_item_3, academic_year:,
+                                                 school:, grade: 1)
         end
 
-          it 'the response rate ignores the responses in the non-short form item' do
+        context 'one scale got all students to respond and another scale got an average response rate of fifty percent' do
-            expect(StudentResponseRateCalculator.new(subcategory:, school:,
+          it 'computes the response rate by dividing the actual responses over possible responses' do
-                                                     academic_year:).rate).to eq 25
+            # (20 + 15 + 10) / (20 + 20 + 20) * 100 = 75%
+            expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 75
           end
         end
+
+        context 'and another unrelated subcategory has responses' do
+          before do
+            create_list(:survey_item_response, 10,
+                        survey_item: second_subcategory.measures.first.scales.first.survey_items.first, academic_year:, school:, grade: 1)
+          end
+
+          it 'does not count the responses for the unrelated subcategory' do
+            expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 75
+          end
+        end
+      end
+      context 'when two grades have sufficient responses' do
+        context 'and half of one grade responded and all of the other grade responded' do
+          before do
+            create(:respondent, school:, academic_year:, total_students: 20, one: 20, two: 20)
+            create_list(:survey_item_response, 10, survey_item: sufficient_student_survey_item_1, academic_year:,
+                                                   school:, grade: 1)
+            create_list(:survey_item_response, 20, survey_item: sufficient_student_survey_item_1, academic_year:,
+                                                   school:, grade: 2)
+          end
+          it 'reports a response rate that averages fifty and 100' do
+            expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 75
           end
         end
 
-    context 'when the average number of teacher responses is greater than the total possible responses' do
+        context 'and two grades responded to different questions at different rates' do
           before do
-        respondent
+            create(:respondent, school:, academic_year:, total_students: 20, one: 20, two: 20)
-        survey
+            create_list(:survey_item_response, 10, survey_item: sufficient_student_survey_item_1, academic_year:,
-        create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD * 11, survey_item: sufficient_student_survey_item_2,
+                                                   school:, grade: 1)
-                                                                                                academic_year:, school:, likert_score: 1)
+            create_list(:survey_item_response, 20, survey_item: sufficient_student_survey_item_2, academic_year:,
+                                                   school:, grade: 2)
+          end
+          it 'reports a response rate that averages fifty and 100' do
+            expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 75
           end
-      it 'returns 100 percent' do
-        expect(StudentResponseRateCalculator.new(subcategory:, school:,
-                                                 academic_year:).rate).to eq 100
         end
       end
 
-    context 'when no survey information exists for that school or year' do
+      context 'when one grade gets surveyed but another does not, the grade that does not get surveyed is not counted' do
-      it 'returns 100 percent' do
+        before do
-        expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 100
+          create(:respondent, school:, academic_year:, total_students: 20, one: 20, two: 20)
+          create_list(:survey_item_response, 10, survey_item: sufficient_student_survey_item_1, academic_year:,
+                                                 school:, grade: 1)
+        end
+        it 'reports a response rate that averages fifty and 100' do
+          expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 50
+        end
       end
     end
 
-    context 'when there is an imbalance in the response rate of the student items' do
+    context 'when the average number of student responses is greater than the total possible responses' do
-      context 'and one of the student items has no associated survey item responses' do
       before do
-          create_list(:survey_item_response, SurveyItemResponse::TEACHER_RESPONSE_THRESHOLD, survey_item: sufficient_teacher_survey_item,
+        create(:respondent, school:, academic_year:, total_students: 20, one: 20, two: 20)
-                                                                                             academic_year:, school:, likert_score: 1)
+        create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD * 11, survey_item: sufficient_student_survey_item_2,
-          create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: sufficient_student_survey_item_1,
+                                                                                                academic_year:, school:, likert_score: 1, grade: 1)
-                                                                                             academic_year:, school:, likert_score: 4)
-          create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: sufficient_student_survey_item_2,
-                                                                                             academic_year:, school:, likert_score: 4)
-          create(:respondent, school:, academic_year:)
-          create(:survey, school:, academic_year:)
-          insufficient_student_survey_item_1
       end
-        it 'ignores the empty survey item and returns only the average response rate of student survey items with responses' do
+      it 'returns 100 percent' do
         expect(StudentResponseRateCalculator.new(subcategory:, school:,
-                                                   academic_year:).rate).to eq 25
+                                                 academic_year:).rate).to eq 100
       end
     end
+
+    context 'when no survey information exists for that school or year' do
+      it 'returns 100 percent' do
+        expect(StudentResponseRateCalculator.new(subcategory:, school:, academic_year:).rate).to eq 100
+      end
     end
   end
 
@@ -148,6 +201,7 @@ describe ResponseRateCalculator, type: :model do
     let(:sufficient_teacher_survey_item_3) { create(:teacher_survey_item, scale: sufficient_scale_1) }
     let(:insufficient_teacher_survey_item_4) { create(:teacher_survey_item, scale: sufficient_scale_1) }
     let(:sufficient_student_survey_item_1) { create(:student_survey_item, scale: sufficient_scale_1) }
+    let(:respondent) { create(:respondent, school:, academic_year:, total_teachers: 8) }
 
     before :each do
       create_list(:survey_item_response, SurveyItemResponse::TEACHER_RESPONSE_THRESHOLD, survey_item: sufficient_teacher_survey_item_1,

spec/presenters/grouped_bar_column_presenter_spec.rb

@@ -110,12 +110,12 @@ describe GroupedBarColumnPresenter do
   context 'for a grouped column presenter with both student and teacher responses' do
     context 'with a single year'
     before do
-      create(:survey_item_response,
+      create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD,
                   survey_item: student_survey_item_for_composite_measure,
                   school:,
                   academic_year:,
                   likert_score: 4)
-      create(:survey_item_response,
+      create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD,
                   survey_item: student_survey_item_for_composite_measure, school:,
                   academic_year:,
                   likert_score: 5)
@@ -165,9 +165,9 @@ describe GroupedBarColumnPresenter do
 
     context 'when the score is in the Ideal zone' do
       before do
-        create(:survey_item_response,  survey_item: student_survey_item, school:,
+        create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: student_survey_item, school:,
                                                                                            academic_year:, likert_score: 5)
-        create(:survey_item_response,  survey_item: student_survey_item, school:,
+        create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: student_survey_item, school:,
                                                                                            academic_year:, likert_score: 4)
       end
 
@@ -219,7 +219,7 @@ describe GroupedBarColumnPresenter do
 
     context 'when the score is in the Approval zone' do
       before do
-        create(:survey_item_response,  survey_item: student_survey_item, school:,
+        create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: student_survey_item, school:,
                                                                                            academic_year:, likert_score: 4)
       end
 
@@ -240,7 +240,7 @@ describe GroupedBarColumnPresenter do
     end
     context 'when the score is in the Growth zone' do
       before do
-        create(:survey_item_response,  survey_item: student_survey_item, school:,
+        create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: student_survey_item, school:,
                                                                                            academic_year:, likert_score: 3)
       end
 
@@ -255,7 +255,7 @@ describe GroupedBarColumnPresenter do
 
       context 'when the score is less than 5 percent away from the approval low benchmark line' do
         before do
-          create_list(:survey_item_response, 40, survey_item: student_survey_item, school:,
+          create_list(:survey_item_response, 80, survey_item: student_survey_item, school:,
                                                  academic_year:, likert_score: 4)
         end
 
@@ -267,7 +267,7 @@ describe GroupedBarColumnPresenter do
 
     context 'when the score is in the Watch zone' do
       before do
-        create(:survey_item_response,  survey_item: student_survey_item, school:,
+        create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: student_survey_item, school:,
                                                                                            academic_year:, likert_score: 2)
       end
 
@@ -282,7 +282,7 @@ describe GroupedBarColumnPresenter do
     end
     context 'when the score is in the Warning zone' do
       before do
-        create(:survey_item_response,  survey_item: student_survey_item, school:,
+        create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD, survey_item: student_survey_item, school:,
                                                                                            academic_year:, likert_score: 1)
       end
 

spec/presenters/subcategory_presenter_spec.rb

@@ -39,10 +39,8 @@ describe SubcategoryPresenter do
                                                                                        academic_year:, school:, likert_score: 1)
     create_list(:survey_item_response, SurveyItemResponse::TEACHER_RESPONSE_THRESHOLD, survey_item: survey_item3,
                                                                                        academic_year:, school:, likert_score: 5)
-    create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD / 2, survey_item: survey_item4,
+    create_list(:survey_item_response, 10, survey_item: survey_item4,
-                                                                                           academic_year:, school:, likert_score: 3)
+                                           academic_year:, school:, likert_score: 3, grade: 1)
-    create_list(:survey_item_response, SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD / 2, survey_item: survey_item4,
-                                                                                           academic_year:, school:, likert_score: 3)
 
     # Adding responses corresponding to different years and schools should not pollute the score calculations
     create_survey_item_responses_for_different_years_and_schools(survey_item1)
@@ -51,7 +49,7 @@ describe SubcategoryPresenter do
   end
 
   before do
-    create(:respondent, school:, academic_year:)
+    create(:respondent, school:, academic_year:, one: 40)
     create(:survey, school:, academic_year:)
   end
 

spec/services/response_rate_loader_spec.rb

@@ -1,140 +0,0 @@
-require 'rails_helper'
-
-describe ResponseRateLoader do
-  let(:school) { create(:school, name: 'milford-high-school') }
-  let(:academic_year) { create(:academic_year, range: '2020-21') }
-  let(:respondent) do
-    respondent = create(:respondent, school:, academic_year:)
-    respondent.total_students = 10
-    respondent.total_teachers = 10
-    respondent.save
-  end
-
-  let(:short_form_survey) do
-    survey = create(:survey, school:, academic_year:)
-    survey.form = :short
-    survey.save
-    survey
-  end
-
-  let(:subcategory) { create(:subcategory, subcategory_id: '5D', name: 'Health') }
-  let(:measure) { create(:measure, measure_id: '5D-ii', subcategory:) }
-
-  let(:s_acst_q1) { create(:survey_item, survey_item_id: 's-acst-q1', scale: s_acst) }
-  let(:s_acst_q2) { create(:survey_item, survey_item_id: 's-acst-q2', scale: s_acst, on_short_form: true) } # short form
-  let(:s_acst_q3) { create(:survey_item, survey_item_id: 's-acst-q3', scale: s_acst) }
-  let(:s_poaf_q1) { create(:survey_item, survey_item_id: 's-poaf-q1', scale: s_poaf) }
-  let(:s_poaf_q2) { create(:survey_item, survey_item_id: 's-poaf-q2', scale: s_poaf) }
-  let(:s_poaf_q3) { create(:survey_item, survey_item_id: 's-poaf-q3', scale: s_poaf, on_short_form: true) } # short form
-  let(:s_poaf_q4) { create(:survey_item, survey_item_id: 's-poaf-q4', scale: s_poaf) }
-  let(:t_phya_q2) { create(:survey_item, survey_item_id: 't-phya-q2', scale: t_phya) }
-  let(:t_phya_q3) { create(:survey_item, survey_item_id: 't-phya-q3', scale: t_phya) }
-
-  let(:s_acst) { create(:scale, scale_id: 's-acst', measure:) }
-  let(:s_poaf) { create(:scale, scale_id: 's-poaf', measure:) }
-  let(:t_phya) { create(:scale, scale_id: 't-phya', measure:) }
-  let(:response_rate) { ResponseRate.find_by(school:, academic_year:) }
-  before do
-    short_form_survey
-    respondent
-  end
-
-  after do
-    DatabaseCleaner.clean
-  end
-
-  describe 'self.reset' do
-    context 'When resetting response rates' do
-      context 'and half the students responded to each question' do
-        before :each do
-          create_list(:survey_item_response, 5, survey_item: s_acst_q1, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 5, survey_item: s_acst_q2, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 5, survey_item: s_acst_q3, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 5, survey_item: s_poaf_q1, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 5, survey_item: s_poaf_q2, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 5, survey_item: s_poaf_q3, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 5, survey_item: s_poaf_q4, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 5, survey_item: t_phya_q2, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 5, survey_item: t_phya_q3, likert_score: 3, school:, academic_year:)
-
-          ResponseRateLoader.reset(schools: [school], academic_years: [academic_year])
-        end
-
-        it 'populates the database with response rates' do
-          expect(s_acst_q1.survey_item_id).to eq 's-acst-q1'
-          expect(subcategory.subcategory_id).to eq '5D'
-          expect(subcategory.name).to eq 'Health'
-          expect(s_acst.score(school:, academic_year:)).to eq 3
-          expect(s_poaf.score(school:, academic_year:)).to eq 3
-          expect(t_phya.score(school:, academic_year:)).to eq 3
-          expect(response_rate.student_response_rate).to eq 50
-          expect(response_rate.teacher_response_rate).to eq 50
-          expect(response_rate.meets_student_threshold).to be true
-          expect(response_rate.meets_teacher_threshold).to be true
-        end
-        context 'when running the loader a second time' do
-          it 'is idempotent' do
-            response_count = ResponseRate.count
-            ResponseRateLoader.reset(schools: [school], academic_years: [academic_year])
-            second_count = ResponseRate.count
-
-            expect(response_count).to eq second_count
-          end
-        end
-      end
-
-      context 'and only the first question was asked; e.g. its on a short form and this is marked as a short form school' do
-        before do
-          create_list(:survey_item_response, 5, survey_item: s_acst_q1, likert_score: 3, school:, academic_year:)
-          s_acst_q1.update(on_short_form: true)
-          create_list(:survey_item_response, 5, survey_item: s_poaf_q1, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 5, survey_item: t_phya_q2, likert_score: 3, school:, academic_year:)
-
-          ResponseRateLoader.reset(schools: [school], academic_years: [academic_year])
-        end
-
-        it 'only takes into account the first question and ignores the other questions in the scale' do
-          expect(response_rate.student_response_rate).to eq 50
-          expect(response_rate.teacher_response_rate).to eq 50
-        end
-      end
-
-      context 'and no respondent entry exists for the school and year' do
-        before do
-          Respondent.delete_all
-          create_list(:survey_item_response, 5, survey_item: s_acst_q1, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 5, survey_item: s_poaf_q1, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 5, survey_item: t_phya_q2, likert_score: 3, school:, academic_year:)
-
-          ResponseRateLoader.reset(schools: [school], academic_years: [academic_year])
-        end
-
-        it 'since no score can be calculated, it returns a default of 100' do
-          expect(response_rate.student_response_rate).to eq 100
-          expect(response_rate.teacher_response_rate).to eq 100
-        end
-      end
-
-      context 'and the school took the short form student survey' do
-        before do
-          create_list(:survey_item_response, 1, survey_item: s_acst_q1, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 6, survey_item: s_acst_q2, likert_score: 3, school:, academic_year:) # short form
-          create_list(:survey_item_response, 1, survey_item: s_acst_q3, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 1, survey_item: s_poaf_q1, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 1, survey_item: s_poaf_q2, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 6, survey_item: s_poaf_q3, likert_score: 3, school:, academic_year:) # short form
-          create_list(:survey_item_response, 1, survey_item: s_poaf_q4, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 1, survey_item: t_phya_q2, likert_score: 3, school:, academic_year:)
-          create_list(:survey_item_response, 1, survey_item: t_phya_q3, likert_score: 3, school:, academic_year:)
-          short_form_survey
-
-          ResponseRateLoader.reset(schools: [school], academic_years: [academic_year])
-        end
-
-        it 'only counts responses from survey items on the short form' do
-          expect(response_rate.student_response_rate).to eq 60
-        end
-      end
-    end
-  end
-end

spec/system/journey_spec.rb

@@ -46,11 +46,13 @@ describe 'District Admin', js: true do
     respondent = Respondent.find_or_initialize_by(school:, academic_year: ay_2021_22)
     respondent.total_students = 8
     respondent.total_teachers = 8
+    respondent.one = 20
     respondent.save
 
     respondent = Respondent.find_or_initialize_by(school:, academic_year: ay_2019_20)
     respondent.total_students = 8
     respondent.total_teachers = 8
+    respondent.one = 20
     respondent.save
   end
 
@@ -71,28 +73,28 @@ describe 'District Admin', js: true do
     survey_items_for_measure_2A_i.each do |survey_item|
       SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD.times do
         survey_item_responses << SurveyItemResponse.new(response_id: rand.to_s, academic_year: ay_2021_22,
-                                                        school:, survey_item:, likert_score: 5)
+                                                        school:, survey_item:, likert_score: 5, grade: 1)
       end
     end
 
     survey_items_for_measure_2A_ii.each do |survey_item|
       SurveyItemResponse::STUDENT_RESPONSE_THRESHOLD.times do
         survey_item_responses << SurveyItemResponse.new(response_id: rand.to_s, academic_year: ay_2021_22,
-                                                        school:, survey_item:, likert_score: 5)
+                                                        school:, survey_item:, likert_score: 5, grade: 1)
       end
     end
 
     survey_items_for_measure_4C_i.each do |survey_item|
       SurveyItemResponse::TEACHER_RESPONSE_THRESHOLD.times do
         survey_item_responses << SurveyItemResponse.new(response_id: rand.to_s, academic_year: ay_2021_22,
-                                                        school:, survey_item:, likert_score: 1)
+                                                        school:, survey_item:, likert_score: 1, grade: 1)
       end
     end
 
     survey_items_for_subcategory.each do |survey_item|
       2.times do
         survey_item_responses << SurveyItemResponse.new(response_id: rand.to_s, academic_year: ay_2021_22,
-                                                        school:, survey_item:, likert_score: 4)
+                                                        school:, survey_item:, likert_score: 4, grade: 1)
       end
     end
 

spec/views/analyze/index.html.erb_spec.rb

@@ -171,7 +171,6 @@ describe 'analyze/index' do
     end
 
     it 'displays disabled checkboxes for years that dont have data' do
-      ResponseRateLoader.reset
       year_checkbox = subject.css("##{academic_year.range}").first
       expect(year_checkbox.name).to eq 'input'
       expect(academic_year.range).to eq '2050-51'