feat: add special education disaggregation

app/javascript/controllers/analyze_controller.js

@@ -30,7 +30,9 @@ export default class extends Controller {
       "&grades=" +
       this.selected_items("grade").join(",") +
       "&ells=" +
-      this.selected_items("ell").join(",");
+      this.selected_items("ell").join(",") +
+      "&speds=" +
+      this.selected_items("sped").join(",");
 
     this.go_to(url);
   }
@@ -124,19 +126,11 @@ export default class extends Controller {
         return item.id;
       })[0];
 
-    const groups = new Map([
-      ['gender', 'students-by-gender'],
-      ['grade', 'students-by-grade'],
-      ['income', 'students-by-income'],
-      ['race', 'students-by-race'],
-      ['ell', 'students-by-ell'],
-    ])
-
     if (target.name === 'slice' || target.name === 'group') {
       if (selected_slice === 'students-and-teachers') {
         return 'students-and-teachers';
       }
-      return groups.get(this.selected_group());
+      return `students-by-${this.selected_group()}`;
     }
 
     return window.graph;

app/models/sped.rb

@@ -0,0 +1,7 @@
+class Sped < ApplicationRecord
+  scope :by_designation, -> { all.map { |sped| [sped.designation, sped] }.to_h }
+
+  include FriendlyId
+
+  friendly_id :designation, use: [:slugged]
+end

app/models/survey_item_response.rb

@@ -11,6 +11,7 @@ class SurveyItemResponse < ActiveRecord::Base
   belongs_to :gender
   belongs_to :income
   belongs_to :ell
+  belongs_to :sped
 
   has_one :measure, through: :survey_item
 
@@ -39,6 +40,11 @@ class SurveyItemResponse < ActiveRecord::Base
                              academic_year:, ell:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score)
   }
 
+  scope :averages_for_sped, lambda { |survey_items, school, academic_year, sped|
+    SurveyItemResponse.where(survey_item: survey_items, school:,
+                             academic_year:, sped:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score)
+  }
+
   scope :averages_for_race, lambda { |school, academic_year, race|
     SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where(
       school:, academic_year:, grade: school.grades(academic_year:)

app/presenters/analyze/graph/column/ell_column/not_ell.rb

@@ -8,7 +8,7 @@ module Analyze
           include Analyze::Graph::Column::EllColumn::ScoreForEll
           include Analyze::Graph::Column::EllColumn::EllCount
           def label
-            %w[Not-ELL]
+            ["Not ELL"]
           end
 
           def basis

app/presenters/analyze/graph/column/sped_column/not_sped.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module SpedColumn
+        class NotSped < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::SpedColumn::ScoreForSped
+          include Analyze::Graph::Column::SpedColumn::SpedCount
+
+          def label
+            ["Not Special", "Education"]
+          end
+
+          def basis
+            "student"
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def sped
+            ::Sped.find_by_slug "not-special-education"
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/sped_column/score_for_sped.rb

@@ -0,0 +1,42 @@
+module Analyze
+  module Graph
+    module Column
+      module SpedColumn
+        module ScoreForSped
+          def score(year_index)
+            academic_year = academic_years[year_index]
+            meets_student_threshold = sufficient_student_responses?(academic_year:)
+            return Score::NIL_SCORE unless meets_student_threshold
+
+            averages = SurveyItemResponse.averages_for_sped(measure.student_survey_items, school, academic_year,
+                                                            sped)
+            average = bubble_up_averages(averages:).round(2)
+
+            Score.new(average:,
+                      meets_teacher_threshold: false,
+                      meets_student_threshold:,
+                      meets_admin_data_threshold: false)
+          end
+
+          def bubble_up_averages(averages:)
+            measure.student_scales.map do |scale|
+              scale.survey_items.map do |survey_item|
+                averages[survey_item]
+              end.remove_blanks.average
+            end.remove_blanks.average
+          end
+
+          def sufficient_student_responses?(academic_year:)
+            return false unless measure.subcategory.response_rate(school:, academic_year:).meets_student_threshold?
+
+            yearly_counts = SurveyItemResponse.where(school:, academic_year:,
+                                                     sped:, survey_item: measure.student_survey_items).group(:sped).select(:response_id).distinct(:response_id).count
+            yearly_counts.any? do |count|
+              count[1] >= 10
+            end
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/sped_column/sped.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module SpedColumn
+        class Sped < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::SpedColumn::ScoreForSped
+          include Analyze::Graph::Column::SpedColumn::SpedCount
+
+          def label
+            %w[Special Education]
+          end
+
+          def basis
+            "student"
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def sped
+            ::Sped.find_by_slug "special-education"
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/sped_column/sped_count.rb

@@ -0,0 +1,18 @@
+module Analyze
+  module Graph
+    module Column
+      module SpedColumn
+        module SpedCount
+          def type
+            :student
+          end
+
+          def n_size(year_index)
+            SurveyItemResponse.where(sped:, survey_item: measure.student_survey_items, school:, grade: grades(year_index),
+                                     academic_year: academic_years[year_index]).select(:response_id).distinct.count
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/sped_column/unknown.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module SpedColumn
+        class Unknown < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::SpedColumn::ScoreForSped
+          include Analyze::Graph::Column::SpedColumn::SpedCount
+
+          def label
+            %w[Unknown]
+          end
+
+          def basis
+            "student"
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def sped
+            ::Sped.find_by_slug "unknown"
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/students_by_ell.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
-#
+
 module Analyze
   module Graph
     class StudentsByEll
-      include Analyze::Graph::Column::GenderColumn
+      include Analyze::Graph::Column::EllColumn
       attr_reader :ells
 
       def initialize(ells:)

app/presenters/analyze/graph/students_by_sped.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    class StudentsBySped
+      include Analyze::Graph::Column::SpedColumn
+      attr_reader :speds
+
+      def initialize(speds:)
+        @speds = speds
+      end
+
+      def to_s
+        "Students by SpEd"
+      end
+
+      def slug
+        "students-by-sped"
+      end
+
+      def columns
+        [].tap do |array|
+          speds.each do |sped|
+            array << column_for_sped_code(code: sped.slug)
+          end
+          array << Analyze::Graph::Column::AllStudent
+        end
+      end
+
+      private
+
+      def column_for_sped_code(code:)
+        CFR[code]
+      end
+
+      CFR = {
+        "special-education" => Analyze::Graph::Column::SpedColumn::Sped,
+        "not-special-education" => Analyze::Graph::Column::SpedColumn::NotSped,
+        "unknown" => Analyze::Graph::Column::SpedColumn::Unknown
+      }.freeze
+    end
+  end
+end

app/presenters/analyze/group/sped.rb

@@ -0,0 +1,13 @@
+module Analyze
+  module Group
+    class Sped
+      def name
+        "SpEd"
+      end
+
+      def slug
+        "sped"
+      end
+    end
+  end
+end

app/presenters/analyze/presenter.rb

@@ -67,6 +67,19 @@ module Analyze
       end
     end
 
+    def speds
+      @speds ||= Sped.all.order(id: :ASC)
+    end
+
+    def selected_speds
+      @selected_speds ||= begin
+        sped_params = params[:speds]
+        return speds unless sped_params
+
+        sped_params.split(",").map { |sped| Sped.find_by_slug sped }.compact
+      end
+    end
+
     def graphs
       @graphs ||= [Analyze::Graph::AllData.new,
                    Analyze::Graph::StudentsAndTeachers.new,
@@ -74,7 +87,8 @@ module Analyze
                    Analyze::Graph::StudentsByGrade.new(grades: selected_grades),
                    Analyze::Graph::StudentsByGender.new(genders: selected_genders),
                    Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes),
-                   Analyze::Graph::StudentsByEll.new(ells: selected_ells)]
+                   Analyze::Graph::StudentsByEll.new(ells: selected_ells),
+                   Analyze::Graph::StudentsBySped.new(speds: selected_speds)]
     end
 
     def graph
@@ -107,7 +121,7 @@ module Analyze
 
     def groups
       @groups = [Analyze::Group::Ell.new, Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new,
-                 Analyze::Group::Race.new]
+                 Analyze::Group::Race.new, Analyze::Group::Sped.new]
     end
 
     def group

app/services/demographic_loader.rb

@@ -7,8 +7,9 @@ class DemographicLoader
     CSV.parse(File.read(filepath), headers: true) do |row|
       process_race(row:)
       process_gender(row:)
-      process_income(row:)
-      process_ell(row:)
+      create_from_column(column: "Income", row:, model: Income)
+      create_from_column(column: "ELL", row:, model: Ell)
+      create_from_column(column: "Special Ed Status", row:, model: Sped)
     end
   end
 
@@ -33,18 +34,11 @@ class DemographicLoader
     gender.save
   end
 
-  def self.process_income(row:)
-    designation = row["Income"]
+  def self.create_from_column(column:, row:, model:)
+    designation = row[column]
     return unless designation
 
-    Income.find_or_create_by!(designation:)
-  end
-
-  def self.process_ell(row:)
-    designation = row["ELL"]
-    return unless designation
-
-    Ell.find_or_create_by!(designation:)
+    model.find_or_create_by!(designation:)
   end
 end
 

app/services/survey_responses_data_loader.rb

@@ -62,6 +62,10 @@ class SurveyResponsesDataLoader
     @ells ||= Ell.by_designation
   end
 
+  def speds
+    @speds ||= Sped.by_designation
+  end
+
   def process_row(row:, rules:)
     return unless row.dese_id?
     return unless row.school.present?
@@ -91,12 +95,14 @@ class SurveyResponsesDataLoader
     grade = row.grade
     income = incomes[row.income.parameterize]
     ell = ells[row.ell]
+    sped = speds[row.sped]
     if survey_item_response.present?
-      survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:)
+      survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:,
+                                   sped:)
       []
     else
       SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:,
-                             likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:)
+                             likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:)
     end
   end
 

app/views/analyze/_group_selectors.html.erb

@@ -25,3 +25,7 @@
 <% @presenter.ells.each do |ell| %>
   <%= render(partial: "checkboxes", locals: {id: "ell-#{ell.slug}", item: ell, selected_items: @presenter.selected_ells, name: "ell", label_text: ell.designation})  %>
 <% end %>
+
+<% @presenter.speds.each do |sped| %>
+  <%= render(partial: "checkboxes", locals: {id: "sped-#{sped.slug}", item: sped, selected_items: @presenter.selected_speds, name: "sped", label_text: sped.designation})  %>
+<% end %>

data/demographics.csv

@@ -1,11 +1,11 @@
-Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL
-1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged - N,ELL
-2,Asian or Pacific Islander,1,Female,Economically Disadvantaged - Y,Not ELL
-3,Black or African American,4,Non-Binary,Unknown,Unknown
-4,Hispanic or Latinx,99,Unknown,,
-5,White or Caucasian,,,,
-6,Prefer not to disclose,,,,
-7,Prefer to self-describe,,,,
-8,Middle Eastern,,,,
-99,Race/Ethnicity Not Listed,,,,
-100,Multiracial,,,,
+Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL,Special Ed Status
+1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged - N,ELL,Special Education
+2,Asian or Pacific Islander,1,Female,Economically Disadvantaged - Y,Not ELL,Not Special Education
+3,Black or African American,4,Non-Binary,Unknown,Unknown,Unknown
+4,Hispanic or Latinx,99,Unknown,,,
+5,White or Caucasian,,,,,
+6,Prefer not to disclose,,,,,
+7,Prefer to self-describe,,,,,
+8,Middle Eastern,,,,,
+99,Race/Ethnicity Not Listed,,,,,
+100,Multiracial,,,,,

db/migrate/20231004191828_create_speds.rb

@@ -0,0 +1,12 @@
+class CreateSpeds < ActiveRecord::Migration[7.0]
+  def change
+    create_table :speds do |t|
+      t.string :designation
+      t.string :slug
+
+      t.timestamps
+    end
+
+    add_index :speds, :designation
+  end
+end

db/migrate/20231004211430_add_sped_to_survey_item_response.rb

@@ -0,0 +1,5 @@
+class AddSpedToSurveyItemResponse < ActiveRecord::Migration[7.0]
+  def change
+    add_reference :survey_item_responses, :sped, foreign_key: true
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 20_230_912_223_701) do
+ActiveRecord::Schema[7.0].define(version: 2023_10_04_211430) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_stat_statements"
   enable_extension "plpgsql"
@@ -406,6 +406,14 @@ ActiveRecord::Schema[7.0].define(version: 20_230_912_223_701) do
     t.index ["school_id"], name: "index_scores_on_school_id"
   end
 
+  create_table "speds", force: :cascade do |t|
+    t.string "designation"
+    t.string "slug"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["designation"], name: "index_speds_on_designation"
+  end
+
   create_table "student_races", force: :cascade do |t|
     t.bigint "student_id", null: false
     t.bigint "race_id", null: false
@@ -448,14 +456,23 @@ ActiveRecord::Schema[7.0].define(version: 20_230_912_223_701) do
     t.datetime "recorded_date"
     t.bigint "income_id"
     t.bigint "ell_id"
+    t.bigint "sped_id"
     t.index ["academic_year_id"], name: "index_survey_item_responses_on_academic_year_id"
     t.index ["ell_id"], name: "index_survey_item_responses_on_ell_id"
     t.index ["gender_id"], name: "index_survey_item_responses_on_gender_id"
     t.index ["income_id"], name: "index_survey_item_responses_on_income_id"
     t.index ["response_id"], name: "index_survey_item_responses_on_response_id"
+<<<<<<< HEAD
     t.index %w[school_id academic_year_id survey_item_id], name: "by_school_year_and_survey_item"
     t.index %w[school_id academic_year_id], name: "index_survey_item_responses_on_school_id_and_academic_year_id"
     t.index %w[school_id survey_item_id academic_year_id grade], name: "index_survey_responses_on_grade"
+=======
+    t.index ["school_id", "academic_year_id", "survey_item_id"], name: "by_school_year_and_survey_item"
+    t.index ["school_id", "academic_year_id"], name: "index_survey_item_responses_on_school_id_and_academic_year_id"
+    t.index ["school_id", "survey_item_id", "academic_year_id", "grade"], name: "index_survey_responses_on_grade"
+    t.index ["school_id"], name: "index_survey_item_responses_on_school_id"
+    t.index ["sped_id"], name: "index_survey_item_responses_on_sped_id"
+>>>>>>> 48e795f (feat: add special education disaggregation)
     t.index ["student_id"], name: "index_survey_item_responses_on_student_id"
     t.index ["survey_item_id"], name: "index_survey_item_responses_on_survey_item_id"
   end
@@ -504,6 +521,7 @@ ActiveRecord::Schema[7.0].define(version: 20_230_912_223_701) do
   add_foreign_key "survey_item_responses", "genders"
   add_foreign_key "survey_item_responses", "incomes"
   add_foreign_key "survey_item_responses", "schools"
+  add_foreign_key "survey_item_responses", "speds"
   add_foreign_key "survey_item_responses", "students"
   add_foreign_key "survey_item_responses", "survey_items"
   add_foreign_key "survey_items", "scales"

spec/fixtures/sample_demographics.csv

@@ -1,11 +1,11 @@
-Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL
-1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged – N,ELL
-2,Asian or Pacific Islander,1,Female,Economically Disadvantaged – Y,Not ELL
-3,Black or African American,4,Non-Binary,Unknown,Unknown
-4,Hispanic or Latinx,99,Unknown,,
-5,White or Caucasian,,,,
-6,Prefer not to disclose,,,,
-7,Prefer to self-describe,,,,
-8,Middle Eastern,,,,
-99,Race/Ethnicity Not Listed,,,,
-100,Multiracial,,,,
+Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL,Special Ed Status
+1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged – N,ELL,Special Education
+2,Asian or Pacific Islander,1,Female,Economically Disadvantaged – Y,Not ELL,Not Special Education
+3,Black or African American,4,Non-Binary,Unknown,Unknown,Unknown
+4,Hispanic or Latinx,99,Unknown,,,
+5,White or Caucasian,,,,,
+6,Prefer not to disclose,,,,,
+7,Prefer to self-describe,,,,,
+8,Middle Eastern,,,,,
+99,Race/Ethnicity Not Listed,,,,,
+100,Multiracial,,,,,

spec/services/demographic_loader_spec.rb

@@ -21,6 +21,10 @@ describe DemographicLoader do
     ["ELL", "Not ELL", "Unknown"]
   end
 
+  let(:speds) do
+    ["Special Education", "Not Special Education", "Unknown"]
+  end
+
   before :each do
     DemographicLoader.load_data(filepath:)
   end
@@ -68,5 +72,12 @@ describe DemographicLoader do
         expect(Ell.find_by_designation(ell).designation).to eq ell
       end
     end
+
+    it "load all the special ed designations" do
+      expect(Sped.all.count).to eq 3
+      speds.each do |sped|
+        expect(Sped.find_by_designation(sped).designation).to eq sped
+      end
+    end
   end
 end

spec/services/survey_item_values_spec.rb

@@ -251,6 +251,44 @@ RSpec.describe SurveyItemValues, type: :model do
     end
   end
 
+  context ".sped" do
+    before :each do
+      attleboro
+      ay_2022_23
+    end
+
+    it 'translates "active" into "Special Education"' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "active" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.sped).to eq "Special Education"
+    end
+
+    it 'translates "exited" into "Not Special Education"' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "exited" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.sped).to eq "Not Special Education"
+    end
+    it 'translates blanks into "Not Special Education' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.sped).to eq "Not Special Education"
+    end
+
+    it 'tranlsates NA into "Unknown"' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "NA" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.sped).to eq "Unknown"
+
+      row = { "Raw SpEd" => "#NA" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.sped).to eq "Unknown"
+    end
+  end
+
   context ".valid_duration" do
     context "when duration is valid" do
       it "returns true" do