diff --git a/app/models/race_score.rb b/app/models/race_score.rb deleted file mode 100644 index a10e34fb..00000000 --- a/app/models/race_score.rb +++ /dev/null @@ -1,6 +0,0 @@ -class RaceScore < ApplicationRecord - belongs_to :measure - belongs_to :school - belongs_to :academic_year - belongs_to :race -end diff --git a/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb b/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb index 97fc65ac..e43c677f 100644 --- a/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb +++ b/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb @@ -20,6 +20,10 @@ module Analyze @number_of_columns = number_of_columns end + def academic_year_for_year_index(year_index) + academic_years[year_index] + end + def score(year_index) measure.score(school:, academic_year: academic_years[year_index]) || 0 end diff --git a/app/presenters/analyze/graph/column/score_for_race.rb b/app/presenters/analyze/graph/column/score_for_race.rb index 207e198f..36bee8a5 100644 --- a/app/presenters/analyze/graph/column/score_for_race.rb +++ b/app/presenters/analyze/graph/column/score_for_race.rb @@ -3,16 +3,64 @@ module Analyze module Column module ScoreForRace def score(year_index) - s = ::RaceScore.find_by(measure:, school:, academic_year: academic_years[year_index], race:) - average = s.average.round(2) unless s.nil? - average ||= 0 - meets_student_threshold = s.meets_student_threshold? unless s.nil? - meets_student_threshold ||= false - Score.new(average:, - meets_teacher_threshold: false, - meets_student_threshold:, + academic_year = academic_year_for_year_index(year_index) + rate = response_rate(school:, academic_year:, measure:) + return Score::NIL_SCORE unless rate.meets_student_threshold + + survey_items = measure.student_survey_items + + averages = grouped_responses(school:, academic_year:, survey_items:, race:) + meets_student_threshold = sufficient_responses(school:, academic_year:, race:) + scorify(responses: averages, meets_student_threshold:, measure:) + end + + def grouped_responses(school:, academic_year:, survey_items:, race:) + @grouped_responses ||= Hash.new do |memo, (school, academic_year, survey_items, race)| + memo[[school, academic_year, survey_items, race]] = + SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where( + school:, academic_year:, grade: school.grades(academic_year:) + ).where("student_races.race_id": race.id).group(:survey_item_id).having("count(*) >= 10").average(:likert_score) + end + + @grouped_responses[[school, academic_year, survey_items, race]] + end + + def response_rate(school:, academic_year:, measure:) + subcategory = measure.subcategory + @response_rate ||= Hash.new do |memo, (school, academic_year, subcategory)| + memo[[school, academic_year, subcategory]] = subcategory.response_rate(school:, academic_year:) + end + + @response_rate[[school, academic_year, subcategory]] + end + + def scorify(responses:, meets_student_threshold:, measure:) + averages = bubble_up_averages(responses:, measure:) + average = averages.average.round(2) + + average = 0 unless meets_student_threshold + + Score.new(average:, meets_teacher_threshold: false, meets_student_threshold:, meets_admin_data_threshold: false) end + + def sufficient_responses(school:, academic_year:, race:) + @sufficient_responses ||= Hash.new do |memo, (school, academic_year, race)| + number_of_students_for_a_racial_group = SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where( + school:, academic_year: + ).where("student_races.race_id": race.id).distinct.pluck(:student_id).count + memo[[school, academic_year, race]] = number_of_students_for_a_racial_group >= 10 + end + @sufficient_responses[[school, academic_year, race]] + end + + def bubble_up_averages(responses:, measure:) + measure.student_scales.map do |scale| + scale.survey_items.map do |survey_item| + responses[survey_item.id] + end.remove_blanks.average + end.remove_blanks + end end end end diff --git a/app/presenters/analyze/presenter.rb b/app/presenters/analyze/presenter.rb index 9e5492d0..391b69a1 100644 --- a/app/presenters/analyze/presenter.rb +++ b/app/presenters/analyze/presenter.rb @@ -137,12 +137,6 @@ module Analyze end.keys end - def race_score_timestamp - score = RaceScore.where(school: @school, - academic_year: @academic_year).order(updated_at: :DESC).first || Today.new - score.updated_at - end - def incomes @incomes ||= Income.all end diff --git a/app/services/race_score_loader.rb b/app/services/race_score_loader.rb deleted file mode 100644 index 7f23b382..00000000 --- a/app/services/race_score_loader.rb +++ /dev/null @@ -1,114 +0,0 @@ -class RaceScoreLoader - def self.reset(schools: School.all, academic_years: AcademicYear.all, measures: Measure.all, races: Race.all, fast_processing: true) - RaceScore.where(school: schools, academic_year: academic_years, measure: measures, race: races).delete_all - measures.each do |measure| - if fast_processing - large_memory_use(measure:, schools:, academic_years:, races:) - else - slow_loading_time(measure:, schools:, academic_years:, races:) - end - end - end - - private - - def self.large_memory_use(measure:, schools:, academic_years:, races:) - loadable_race_scores = schools.map do |school| - academic_years.map do |academic_year| - races.map do |race| - process_score(measure:, school:, academic_year:, race:) - end - end - end - RaceScore.import(loadable_race_scores.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all) - end - - def self.slow_loading_time(measure:, schools:, academic_years:, races:) - schools.each do |school| - loadable_race_scores = academic_years.map do |academic_year| - races.map do |race| - process_score(measure:, school:, academic_year:, race:) - end - end - RaceScore.import(loadable_race_scores.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all) - @grouped_responses = nil - @response_rate = nil - @sufficient_responses = nil - end - end - - def self.process_score(measure:, school:, academic_year:, race:) - score = race_score(measure:, school:, academic_year:, race:) - { measure_id: measure.id, school_id: school.id, academic_year_id: academic_year.id, race_id: race.id, average: score.average, - meets_student_threshold: score.meets_student_threshold? } - end - - def self.race_score(measure:, school:, academic_year:, race:) - rate = response_rate(school:, academic_year:, measure:) - unless rate.meets_student_threshold - return Score.new(average: 0, meets_teacher_threshold: false, meets_student_threshold: false, - meets_admin_data_threshold: false) - end - - survey_items = measure.student_survey_items - - averages = grouped_responses(school:, academic_year:, survey_items:, race:) - meets_student_threshold = sufficient_responses(school:, academic_year:, race:) - scorify(responses: averages, meets_student_threshold:, measure:) - end - - def self.grouped_responses(school:, academic_year:, survey_items:, race:) - @grouped_responses ||= Hash.new do |memo, (school, academic_year, survey_items, race)| - memo[[school, academic_year, survey_items, race]] = - SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where( - school:, academic_year:, grade: school.grades(academic_year:) - ).where("student_races.race_id": race.id).group(:survey_item_id).having("count(*) >= 10").average(:likert_score) - end - - @grouped_responses[[school, academic_year, survey_items, race]] - end - - def self.response_rate(school:, academic_year:, measure:) - subcategory = measure.subcategory - @response_rate ||= Hash.new do |memo, (school, academic_year, subcategory)| - memo[[school, academic_year, subcategory]] = subcategory.response_rate(school:, academic_year:) - end - - @response_rate[[school, academic_year, subcategory]] - end - - def self.scorify(responses:, meets_student_threshold:, measure:) - averages = bubble_up_averages(responses:, measure:) - average = averages.average - - average = 0 unless meets_student_threshold - - Score.new(average:, meets_teacher_threshold: false, meets_student_threshold:, meets_admin_data_threshold: false) - end - - def self.sufficient_responses(school:, academic_year:, race:) - @sufficient_responses ||= Hash.new do |memo, (school, academic_year, race)| - number_of_students_for_a_racial_group = SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where( - school:, academic_year: - ).where("student_races.race_id": race.id).distinct.pluck(:student_id).count - memo[[school, academic_year, race]] = number_of_students_for_a_racial_group >= 10 - end - @sufficient_responses[[school, academic_year, race]] - end - - def self.bubble_up_averages(responses:, measure:) - measure.student_scales.map do |scale| - scale.survey_items.map do |survey_item| - responses[survey_item.id] - end.remove_blanks.average - end.remove_blanks - end - - private_class_method :process_score - private_class_method :race_score - private_class_method :grouped_responses - private_class_method :response_rate - private_class_method :scorify - private_class_method :sufficient_responses - private_class_method :bubble_up_averages -end diff --git a/app/views/analyze/index.html.erb b/app/views/analyze/index.html.erb index 63ec8a1f..d7704a5b 100644 --- a/app/views/analyze/index.html.erb +++ b/app/views/analyze/index.html.erb @@ -13,7 +13,7 @@ <%= render partial: "school_years", locals: {available_academic_years: @presenter.academic_years, selected_academic_years: @presenter.selected_academic_years, district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory, measures: @presenter.measures, graph: @presenter.graph} %> <%= render partial: "data_filters", locals: {district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory} %> - <% cache [@presenter.subcategory, @school, @presenter.selected_academic_years, @presenter.graph, @presenter.selected_races, @presenter.race_score_timestamp, @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders] do %> + <% cache [@presenter.subcategory, @school, @presenter.selected_academic_years, @presenter.graph, @presenter.selected_races, @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders] do %>
<% @presenter.measures.each do |measure| %>
diff --git a/db/migrate/20230807222503_drop_race_scores.rb b/db/migrate/20230807222503_drop_race_scores.rb new file mode 100644 index 00000000..50ba33d2 --- /dev/null +++ b/db/migrate/20230807222503_drop_race_scores.rb @@ -0,0 +1,18 @@ +class DropRaceScores < ActiveRecord::Migration[7.0] + def up + drop_table :race_scores + end + + def down + create_table :race_scores do |t| + t.references :measure, null: false, foreign_key: true + t.references :school, null: false, foreign_key: true + t.references :academic_year, null: false, foreign_key: true + t.references :race, null: false, foreign_key: true + t.float :average + t.boolean :meets_student_threshold + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 1f172a7f..e22679e8 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_06_30_215110) do +ActiveRecord::Schema[7.0].define(version: 2023_08_07_222503) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -304,21 +304,6 @@ ActiveRecord::Schema[7.0].define(version: 2023_06_30_215110) do t.index ["subcategory_id"], name: "index_measures_on_subcategory_id" end - create_table "race_scores", force: :cascade do |t| - t.bigint "measure_id", null: false - t.bigint "school_id", null: false - t.bigint "academic_year_id", null: false - t.bigint "race_id", null: false - t.float "average" - t.boolean "meets_student_threshold" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["academic_year_id"], name: "index_race_scores_on_academic_year_id" - t.index ["measure_id"], name: "index_race_scores_on_measure_id" - t.index ["race_id"], name: "index_race_scores_on_race_id" - t.index ["school_id"], name: "index_race_scores_on_school_id" - end - create_table "races", force: :cascade do |t| t.string "designation" t.integer "qualtrics_code" @@ -491,10 +476,6 @@ ActiveRecord::Schema[7.0].define(version: 2023_06_30_215110) do add_foreign_key "legacy_school_categories", "legacy_categories", column: "category_id" add_foreign_key "legacy_school_categories", "legacy_schools", column: "school_id" add_foreign_key "measures", "subcategories" - add_foreign_key "race_scores", "academic_years" - add_foreign_key "race_scores", "measures" - add_foreign_key "race_scores", "races" - add_foreign_key "race_scores", "schools" add_foreign_key "respondents", "academic_years" add_foreign_key "respondents", "schools" add_foreign_key "response_rates", "academic_years" diff --git a/spec/services/race_score_loader_spec.rb b/spec/services/race_score_loader_spec.rb deleted file mode 100644 index 929bf978..00000000 --- a/spec/services/race_score_loader_spec.rb +++ /dev/null @@ -1,77 +0,0 @@ -require 'rails_helper' - -describe RaceScoreLoader do - let(:measure) { create(:measure, :with_student_survey_items) } - let(:school) { create(:school) } - let(:academic_year) { create(:academic_year) } - let(:race) { create(:race) } - let(:students) do - [].tap do |arr| - 10.times do - s = create(:student) - s.races << race - s.save - arr << s - end - end - end - let(:survey_item_1) { measure.survey_items[0] } - let(:survey_item_2) { measure.survey_items[1] } - let(:survey_item_3) { measure.survey_items[2] } - let(:response_rate) do - create(:response_rate, school:, academic_year:, subcategory: measure.subcategory, meets_student_threshold: true) - end - - # I'm not sure how to securely make available the key_derivation_salt for github actions. Disabling the tests - context 'when survey item responses exist' do - before :each do - response_rate - students.each do |student| - create(:survey_item_response, school:, academic_year:, likert_score: 2, survey_item: survey_item_1, student:) - end - students.each do |student| - create(:survey_item_response, school:, academic_year:, likert_score: 3, survey_item: survey_item_2, student:) - end - - RaceScoreLoader.reset - end - it 'returns a list of averages' do - expect(measure.student_survey_items.count).to eq 2 - american_indian_score = RaceScore.find_by(measure:, school:, academic_year:, race:) - expect(american_indian_score.average).to eq 2.5 - expect(american_indian_score.meets_student_threshold).to eq true - end - - it 'is idempotent' do - original_count = RaceScore.count - RaceScoreLoader.reset - new_count = RaceScore.count - expect(original_count).to eq new_count - end - end - - context 'when there NOT sufficient survey item responses' do - before :each do - response_rate - 9.times do |index| - create(:survey_item_response, school:, academic_year:, likert_score: 2, survey_item: survey_item_1, - student: students[index]) - end - - 9.times do |index| - create(:survey_item_response, school:, academic_year:, likert_score: 3, survey_item: survey_item_2, - student: students[index]) - end - - RaceScoreLoader.reset - end - it 'returns a list of averages' do - expect(measure.student_survey_items.count).to eq 2 - - expect(SurveyItemResponse.count).to eq 18 - rs = RaceScore.find_by(measure:, school:, academic_year:, race:) - expect(rs.average).to eq 0 - expect(rs.meets_student_threshold).to eq false - end - end -end