chore: remove precalculated race scores. Calculate race scores on every reload

app/models/race_score.rb

@@ -1,6 +0,0 @@
-class RaceScore < ApplicationRecord
-  belongs_to :measure
-  belongs_to :school
-  belongs_to :academic_year
-  belongs_to :race
-end

app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb

@@ -20,6 +20,10 @@ module Analyze
           @number_of_columns = number_of_columns
         end
 
+        def academic_year_for_year_index(year_index)
+          academic_years[year_index]
+        end
+
         def score(year_index)
           measure.score(school:, academic_year: academic_years[year_index]) || 0
         end

app/presenters/analyze/graph/column/score_for_race.rb

@@ -3,16 +3,64 @@ module Analyze
     module Column
       module ScoreForRace
         def score(year_index)
-          s = ::RaceScore.find_by(measure:, school:, academic_year: academic_years[year_index], race:)
-          average = s.average.round(2) unless s.nil?
-          average ||= 0
-          meets_student_threshold = s.meets_student_threshold? unless s.nil?
-          meets_student_threshold ||= false
-          Score.new(average:,
-                    meets_teacher_threshold: false,
-                    meets_student_threshold:,
+          academic_year = academic_year_for_year_index(year_index)
+          rate = response_rate(school:, academic_year:, measure:)
+          return Score::NIL_SCORE unless rate.meets_student_threshold
+
+          survey_items = measure.student_survey_items
+
+          averages = grouped_responses(school:, academic_year:, survey_items:, race:)
+          meets_student_threshold = sufficient_responses(school:, academic_year:, race:)
+          scorify(responses: averages, meets_student_threshold:, measure:)
+        end
+
+        def grouped_responses(school:, academic_year:, survey_items:, race:)
+          @grouped_responses ||= Hash.new do |memo, (school, academic_year, survey_items, race)|
+            memo[[school, academic_year, survey_items, race]] =
+              SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where(
+                school:, academic_year:, grade: school.grades(academic_year:)
+              ).where("student_races.race_id": race.id).group(:survey_item_id).having("count(*) >= 10").average(:likert_score)
+          end
+
+          @grouped_responses[[school, academic_year, survey_items, race]]
+        end
+
+        def response_rate(school:, academic_year:, measure:)
+          subcategory = measure.subcategory
+          @response_rate ||= Hash.new do |memo, (school, academic_year, subcategory)|
+            memo[[school, academic_year, subcategory]] = subcategory.response_rate(school:, academic_year:)
+          end
+
+          @response_rate[[school, academic_year, subcategory]]
+        end
+
+        def scorify(responses:, meets_student_threshold:, measure:)
+          averages = bubble_up_averages(responses:, measure:)
+          average = averages.average.round(2)
+
+          average = 0 unless meets_student_threshold
+
+          Score.new(average:, meets_teacher_threshold: false, meets_student_threshold:,
                     meets_admin_data_threshold: false)
         end
+
+        def sufficient_responses(school:, academic_year:, race:)
+          @sufficient_responses ||= Hash.new do |memo, (school, academic_year, race)|
+            number_of_students_for_a_racial_group = SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where(
+              school:, academic_year:
+            ).where("student_races.race_id": race.id).distinct.pluck(:student_id).count
+            memo[[school, academic_year, race]] = number_of_students_for_a_racial_group >= 10
+          end
+          @sufficient_responses[[school, academic_year, race]]
+        end
+
+        def bubble_up_averages(responses:, measure:)
+          measure.student_scales.map do |scale|
+            scale.survey_items.map do |survey_item|
+              responses[survey_item.id]
+            end.remove_blanks.average
+          end.remove_blanks
+        end
       end
     end
   end

app/presenters/analyze/presenter.rb

@@ -137,12 +137,6 @@ module Analyze
       end.keys
     end
 
-    def race_score_timestamp
-      score = RaceScore.where(school: @school,
-                              academic_year: @academic_year).order(updated_at: :DESC).first || Today.new
-      score.updated_at
-    end
-
     def incomes
       @incomes ||= Income.all
     end

app/services/race_score_loader.rb

@@ -1,114 +0,0 @@
-class RaceScoreLoader
-  def self.reset(schools: School.all, academic_years: AcademicYear.all, measures: Measure.all, races: Race.all, fast_processing: true)
-    RaceScore.where(school: schools, academic_year: academic_years, measure: measures, race: races).delete_all
-    measures.each do |measure|
-      if fast_processing
-        large_memory_use(measure:, schools:, academic_years:, races:)
-      else
-        slow_loading_time(measure:, schools:, academic_years:, races:)
-      end
-    end
-  end
-
-  private
-
-  def self.large_memory_use(measure:, schools:, academic_years:, races:)
-    loadable_race_scores = schools.map do |school|
-      academic_years.map do |academic_year|
-        races.map do |race|
-          process_score(measure:, school:, academic_year:, race:)
-        end
-      end
-    end
-    RaceScore.import(loadable_race_scores.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
-  end
-
-  def self.slow_loading_time(measure:, schools:, academic_years:, races:)
-    schools.each do |school|
-      loadable_race_scores = academic_years.map do |academic_year|
-        races.map do |race|
-          process_score(measure:, school:, academic_year:, race:)
-        end
-      end
-      RaceScore.import(loadable_race_scores.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
-      @grouped_responses = nil
-      @response_rate = nil
-      @sufficient_responses = nil
-    end
-  end
-
-  def self.process_score(measure:, school:, academic_year:, race:)
-    score = race_score(measure:, school:, academic_year:, race:)
-    { measure_id: measure.id, school_id: school.id, academic_year_id: academic_year.id, race_id: race.id, average: score.average,
-      meets_student_threshold: score.meets_student_threshold? }
-  end
-
-  def self.race_score(measure:, school:, academic_year:, race:)
-    rate = response_rate(school:, academic_year:, measure:)
-    unless rate.meets_student_threshold
-      return Score.new(average: 0, meets_teacher_threshold: false, meets_student_threshold: false,
-                       meets_admin_data_threshold: false)
-    end
-
-    survey_items = measure.student_survey_items
-
-    averages = grouped_responses(school:, academic_year:, survey_items:, race:)
-    meets_student_threshold = sufficient_responses(school:, academic_year:, race:)
-    scorify(responses: averages, meets_student_threshold:, measure:)
-  end
-
-  def self.grouped_responses(school:, academic_year:, survey_items:, race:)
-    @grouped_responses ||= Hash.new do |memo, (school, academic_year, survey_items, race)|
-      memo[[school, academic_year, survey_items, race]] =
-        SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where(
-          school:, academic_year:, grade: school.grades(academic_year:)
-        ).where("student_races.race_id": race.id).group(:survey_item_id).having("count(*) >= 10").average(:likert_score)
-    end
-
-    @grouped_responses[[school, academic_year, survey_items, race]]
-  end
-
-  def self.response_rate(school:, academic_year:, measure:)
-    subcategory = measure.subcategory
-    @response_rate ||= Hash.new do |memo, (school, academic_year, subcategory)|
-      memo[[school, academic_year, subcategory]] = subcategory.response_rate(school:, academic_year:)
-    end
-
-    @response_rate[[school, academic_year, subcategory]]
-  end
-
-  def self.scorify(responses:, meets_student_threshold:, measure:)
-    averages = bubble_up_averages(responses:, measure:)
-    average = averages.average
-
-    average = 0 unless meets_student_threshold
-
-    Score.new(average:, meets_teacher_threshold: false, meets_student_threshold:, meets_admin_data_threshold: false)
-  end
-
-  def self.sufficient_responses(school:, academic_year:, race:)
-    @sufficient_responses ||= Hash.new do |memo, (school, academic_year, race)|
-      number_of_students_for_a_racial_group = SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where(
-        school:, academic_year:
-      ).where("student_races.race_id": race.id).distinct.pluck(:student_id).count
-      memo[[school, academic_year, race]] = number_of_students_for_a_racial_group >= 10
-    end
-    @sufficient_responses[[school, academic_year, race]]
-  end
-
-  def self.bubble_up_averages(responses:, measure:)
-    measure.student_scales.map do |scale|
-      scale.survey_items.map do |survey_item|
-        responses[survey_item.id]
-      end.remove_blanks.average
-    end.remove_blanks
-  end
-
-  private_class_method :process_score
-  private_class_method :race_score
-  private_class_method :grouped_responses
-  private_class_method :response_rate
-  private_class_method :scorify
-  private_class_method :sufficient_responses
-  private_class_method :bubble_up_averages
-end

app/views/analyze/index.html.erb

@@ -13,7 +13,7 @@
     <%= render partial: "school_years", locals: {available_academic_years: @presenter.academic_years, selected_academic_years: @presenter.selected_academic_years, district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory, measures: @presenter.measures, graph: @presenter.graph} %>
     <%= render partial: "data_filters", locals: {district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory} %>
   </div>
-  <% cache [@presenter.subcategory, @school, @presenter.selected_academic_years,  @presenter.graph, @presenter.selected_races, @presenter.race_score_timestamp, @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders]  do %>
+  <% cache [@presenter.subcategory, @school, @presenter.selected_academic_years,  @presenter.graph, @presenter.selected_races,  @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders]  do %>
     <div class="bg-color-white flex-grow-1 col-9">
       <% @presenter.measures.each do  |measure| %>
         <section class="mb-6">

db/migrate/20230807222503_drop_race_scores.rb

@@ -0,0 +1,18 @@
+class DropRaceScores < ActiveRecord::Migration[7.0]
+  def up
+    drop_table :race_scores
+  end
+
+  def down
+    create_table :race_scores do |t|
+      t.references :measure, null: false, foreign_key: true
+      t.references :school, null: false, foreign_key: true
+      t.references :academic_year, null: false, foreign_key: true
+      t.references :race, null: false, foreign_key: true
+      t.float :average
+      t.boolean :meets_student_threshold
+
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_06_30_215110) do
+ActiveRecord::Schema[7.0].define(version: 2023_08_07_222503) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -304,21 +304,6 @@ ActiveRecord::Schema[7.0].define(version: 2023_06_30_215110) do
     t.index ["subcategory_id"], name: "index_measures_on_subcategory_id"
   end
 
-  create_table "race_scores", force: :cascade do |t|
-    t.bigint "measure_id", null: false
-    t.bigint "school_id", null: false
-    t.bigint "academic_year_id", null: false
-    t.bigint "race_id", null: false
-    t.float "average"
-    t.boolean "meets_student_threshold"
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.index ["academic_year_id"], name: "index_race_scores_on_academic_year_id"
-    t.index ["measure_id"], name: "index_race_scores_on_measure_id"
-    t.index ["race_id"], name: "index_race_scores_on_race_id"
-    t.index ["school_id"], name: "index_race_scores_on_school_id"
-  end
-
   create_table "races", force: :cascade do |t|
     t.string "designation"
     t.integer "qualtrics_code"
@@ -491,10 +476,6 @@ ActiveRecord::Schema[7.0].define(version: 2023_06_30_215110) do
   add_foreign_key "legacy_school_categories", "legacy_categories", column: "category_id"
   add_foreign_key "legacy_school_categories", "legacy_schools", column: "school_id"
   add_foreign_key "measures", "subcategories"
-  add_foreign_key "race_scores", "academic_years"
-  add_foreign_key "race_scores", "measures"
-  add_foreign_key "race_scores", "races"
-  add_foreign_key "race_scores", "schools"
   add_foreign_key "respondents", "academic_years"
   add_foreign_key "respondents", "schools"
   add_foreign_key "response_rates", "academic_years"

spec/services/race_score_loader_spec.rb

@@ -1,77 +0,0 @@
-require 'rails_helper'
-
-describe RaceScoreLoader do
-  let(:measure) { create(:measure, :with_student_survey_items) }
-  let(:school) { create(:school) }
-  let(:academic_year) { create(:academic_year) }
-  let(:race) { create(:race) }
-  let(:students) do
-    [].tap do |arr|
-      10.times do
-        s = create(:student)
-        s.races << race
-        s.save
-        arr << s
-      end
-    end
-  end
-  let(:survey_item_1) { measure.survey_items[0] }
-  let(:survey_item_2) { measure.survey_items[1] }
-  let(:survey_item_3) { measure.survey_items[2] }
-  let(:response_rate) do
-    create(:response_rate, school:, academic_year:, subcategory: measure.subcategory, meets_student_threshold: true)
-  end
-
-  # I'm not sure how to securely make available the key_derivation_salt for github actions.  Disabling the tests
-  context 'when survey item responses exist' do
-    before :each do
-      response_rate
-      students.each do |student|
-        create(:survey_item_response, school:, academic_year:, likert_score: 2, survey_item: survey_item_1, student:)
-      end
-      students.each do |student|
-        create(:survey_item_response, school:, academic_year:, likert_score: 3, survey_item: survey_item_2, student:)
-      end
-
-      RaceScoreLoader.reset
-    end
-    it 'returns a list of averages' do
-      expect(measure.student_survey_items.count).to eq 2
-      american_indian_score = RaceScore.find_by(measure:, school:, academic_year:, race:)
-      expect(american_indian_score.average).to eq 2.5
-      expect(american_indian_score.meets_student_threshold).to eq true
-    end
-
-    it 'is idempotent' do
-      original_count = RaceScore.count
-      RaceScoreLoader.reset
-      new_count = RaceScore.count
-      expect(original_count).to eq new_count
-    end
-  end
-
-  context 'when there NOT sufficient survey item responses' do
-    before :each do
-      response_rate
-      9.times do |index|
-        create(:survey_item_response, school:, academic_year:, likert_score: 2, survey_item: survey_item_1,
-                                      student: students[index])
-      end
-
-      9.times do |index|
-        create(:survey_item_response, school:, academic_year:, likert_score: 3, survey_item: survey_item_2,
-                                      student: students[index])
-      end
-
-      RaceScoreLoader.reset
-    end
-    it 'returns a list of averages' do
-      expect(measure.student_survey_items.count).to eq 2
-
-      expect(SurveyItemResponse.count).to eq 18
-      rs = RaceScore.find_by(measure:, school:, academic_year:, race:)
-      expect(rs.average).to eq 0
-      expect(rs.meets_student_threshold).to eq false
-    end
-  end
-end