feat: add special education disaggregation

app/javascript/controllers/analyze_controller.js

@@ -30,7 +30,9 @@ export default class extends Controller {
       "&grades=" +
       this.selected_items("grade").join(",") +
       "&ells=" +
-      this.selected_items("ell").join(",");
+      this.selected_items("ell").join(",") +
+      "&speds=" +
+      this.selected_items("sped").join(",");
 
     this.go_to(url);
   }
@@ -124,19 +126,11 @@ export default class extends Controller {
         return item.id;
       })[0];
 
-    const groups = new Map([
-      ['gender', 'students-by-gender'],
-      ['grade', 'students-by-grade'],
-      ['income', 'students-by-income'],
-      ['race', 'students-by-race'],
-      ['ell', 'students-by-ell'],
-    ])
-
     if (target.name === 'slice' || target.name === 'group') {
       if (selected_slice === 'students-and-teachers') {
         return 'students-and-teachers';
       }
-      return groups.get(this.selected_group());
+      return `students-by-${this.selected_group()}`;
     }
 
     return window.graph;

app/models/sped.rb

@@ -0,0 +1,7 @@
+class Sped < ApplicationRecord
+  scope :by_designation, -> { all.map { |sped| [sped.designation, sped] }.to_h }
+
+  include FriendlyId
+
+  friendly_id :designation, use: [:slugged]
+end

app/models/survey_item_response.rb

@@ -11,6 +11,7 @@ class SurveyItemResponse < ActiveRecord::Base
   belongs_to :gender
   belongs_to :income
   belongs_to :ell
+  belongs_to :sped
 
   has_one :measure, through: :survey_item
 
@@ -38,6 +39,11 @@ class SurveyItemResponse < ActiveRecord::Base
                              academic_year:, ell:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score)
   }
 
+  scope :averages_for_sped, lambda { |survey_items, school, academic_year, sped|
+    SurveyItemResponse.where(survey_item: survey_items, school:,
+                             academic_year:, sped:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score)
+  }
+
   scope :averages_for_race, lambda { |school, academic_year, race|
     SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where(
       school:, academic_year:, grade: school.grades(academic_year:)

app/presenters/analyze/graph/column/ell_column/not_ell.rb

@@ -8,7 +8,7 @@ module Analyze
           include Analyze::Graph::Column::EllColumn::ScoreForEll
           include Analyze::Graph::Column::EllColumn::EllCount
           def label
-            %w[Not-ELL]
+            ["Not ELL"]
           end
 
           def basis

app/presenters/analyze/graph/column/sped_column/not_sped.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module SpedColumn
+        class NotSped < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::SpedColumn::ScoreForSped
+          include Analyze::Graph::Column::SpedColumn::SpedCount
+
+          def label
+            ["Not Special", "Education"]
+          end
+
+          def basis
+            "student"
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def sped
+            ::Sped.find_by_slug "not-special-education"
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/sped_column/score_for_sped.rb

@@ -0,0 +1,42 @@
+module Analyze
+  module Graph
+    module Column
+      module SpedColumn
+        module ScoreForSped
+          def score(year_index)
+            academic_year = academic_years[year_index]
+            meets_student_threshold = sufficient_student_responses?(academic_year:)
+            return Score::NIL_SCORE unless meets_student_threshold
+
+            averages = SurveyItemResponse.averages_for_sped(measure.student_survey_items, school, academic_year,
+                                                            sped)
+            average = bubble_up_averages(averages:).round(2)
+
+            Score.new(average:,
+                      meets_teacher_threshold: false,
+                      meets_student_threshold:,
+                      meets_admin_data_threshold: false)
+          end
+
+          def bubble_up_averages(averages:)
+            measure.student_scales.map do |scale|
+              scale.survey_items.map do |survey_item|
+                averages[survey_item]
+              end.remove_blanks.average
+            end.remove_blanks.average
+          end
+
+          def sufficient_student_responses?(academic_year:)
+            return false unless measure.subcategory.response_rate(school:, academic_year:).meets_student_threshold?
+
+            yearly_counts = SurveyItemResponse.where(school:, academic_year:,
+                                                     sped:, survey_item: measure.student_survey_items).group(:sped).select(:response_id).distinct(:response_id).count
+            yearly_counts.any? do |count|
+              count[1] >= 10
+            end
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/sped_column/sped.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module SpedColumn
+        class Sped < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::SpedColumn::ScoreForSped
+          include Analyze::Graph::Column::SpedColumn::SpedCount
+
+          def label
+            %w[Special Education]
+          end
+
+          def basis
+            "student"
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def sped
+            ::Sped.find_by_slug "special-education"
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/sped_column/sped_count.rb

@@ -0,0 +1,18 @@
+module Analyze
+  module Graph
+    module Column
+      module SpedColumn
+        module SpedCount
+          def type
+            :student
+          end
+
+          def n_size(year_index)
+            SurveyItemResponse.where(sped:, survey_item: measure.student_survey_items, school:, grade: grades(year_index),
+                                     academic_year: academic_years[year_index]).select(:response_id).distinct.count
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/sped_column/unknown.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module SpedColumn
+        class Unknown < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::SpedColumn::ScoreForSped
+          include Analyze::Graph::Column::SpedColumn::SpedCount
+
+          def label
+            %w[Unknown]
+          end
+
+          def basis
+            "student"
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def sped
+            ::Sped.find_by_slug "unknown"
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/students_by_ell.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
-#
+
 module Analyze
   module Graph
     class StudentsByEll
-      include Analyze::Graph::Column::GenderColumn
+      include Analyze::Graph::Column::EllColumn
       attr_reader :ells
 
       def initialize(ells:)

app/presenters/analyze/graph/students_by_sped.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    class StudentsBySped
+      include Analyze::Graph::Column::SpedColumn
+      attr_reader :speds
+
+      def initialize(speds:)
+        @speds = speds
+      end
+
+      def to_s
+        "Students by SpEd"
+      end
+
+      def slug
+        "students-by-sped"
+      end
+
+      def columns
+        [].tap do |array|
+          speds.each do |sped|
+            array << column_for_sped_code(code: sped.slug)
+          end
+          array << Analyze::Graph::Column::AllStudent
+        end
+      end
+
+      private
+
+      def column_for_sped_code(code:)
+        CFR[code]
+      end
+
+      CFR = {
+        "special-education" => Analyze::Graph::Column::SpedColumn::Sped,
+        "not-special-education" => Analyze::Graph::Column::SpedColumn::NotSped,
+        "unknown" => Analyze::Graph::Column::SpedColumn::Unknown
+      }.freeze
+    end
+  end
+end

app/presenters/analyze/group/sped.rb

@@ -0,0 +1,13 @@
+module Analyze
+  module Group
+    class Sped
+      def name
+        "SpEd"
+      end
+
+      def slug
+        "sped"
+      end
+    end
+  end
+end

app/presenters/analyze/presenter.rb

@@ -67,6 +67,19 @@ module Analyze
       end
     end
 
+    def speds
+      @speds ||= Sped.all.order(id: :ASC)
+    end
+
+    def selected_speds
+      @selected_speds ||= begin
+        sped_params = params[:speds]
+        return speds unless sped_params
+
+        sped_params.split(",").map { |sped| Sped.find_by_slug sped }.compact
+      end
+    end
+
     def graphs
       @graphs ||= [Analyze::Graph::AllData.new,
                    Analyze::Graph::StudentsAndTeachers.new,
@@ -74,7 +87,8 @@ module Analyze
                    Analyze::Graph::StudentsByGrade.new(grades: selected_grades),
                    Analyze::Graph::StudentsByGender.new(genders: selected_genders),
                    Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes),
-                   Analyze::Graph::StudentsByEll.new(ells: selected_ells)]
+                   Analyze::Graph::StudentsByEll.new(ells: selected_ells),
+                   Analyze::Graph::StudentsBySped.new(speds: selected_speds)]
     end
 
     def graph
@@ -107,7 +121,7 @@ module Analyze
 
     def groups
       @groups = [Analyze::Group::Ell.new, Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new,
-                 Analyze::Group::Race.new]
+                 Analyze::Group::Race.new, Analyze::Group::Sped.new]
     end
 
     def group

app/services/cleaner.rb

@@ -43,7 +43,7 @@ class Cleaner
     log_csv = []
     data = []
 
-    headers = CSV.parse(file.first).first.push("Raw Income").push("Income").push("Raw ELL").push("ELL")
+    headers = CSV.parse(file.first).first.push("Raw Income").push("Income").push("Raw ELL").push("ELL").push("Raw SpEd").push("SpEd")
     filtered_headers = include_all_headers(headers:)
     filtered_headers = remove_unwanted_headers(headers: filtered_headers)
     log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?",

app/services/demographic_loader.rb

@@ -5,8 +5,9 @@ class DemographicLoader
     CSV.parse(File.read(filepath), headers: true) do |row|
       process_race(row:)
       process_gender(row:)
-      process_income(row:)
-      process_ell(row:)
+      create_from_column(column: "Income", row:, model: Income)
+      create_from_column(column: "ELL", row:, model: Ell)
+      create_from_column(column: "Special Ed Status", row:, model: Sped)
     end
   end
 
@@ -31,18 +32,11 @@ class DemographicLoader
     gender.save
   end
 
-  def self.process_income(row:)
-    designation = row["Income"]
+  def self.create_from_column(column:, row:, model:)
+    designation = row[column]
     return unless designation
 
-    Income.find_or_create_by!(designation:)
-  end
-
-  def self.process_ell(row:)
-    designation = row["ELL"]
-    return unless designation
-
-    Ell.find_or_create_by!(designation:)
+    model.find_or_create_by!(designation:)
   end
 end
 

app/services/survey_item_values.rb

@@ -15,6 +15,8 @@ class SurveyItemValues
     row["Raw Income"] = raw_income
     row["Raw ELL"] = raw_ell
     row["ELL"] = ell
+    row["Raw SpEd"] = raw_sped
+    row["SpEd"] = sped
 
     copy_data_to_main_column(main: /Race/i, secondary: /Race Secondary|Race-1/i)
     copy_data_to_main_column(main: /Gender/i, secondary: /Gender Secondary|Gender-1/i)
@@ -163,6 +165,21 @@ class SurveyItemValues
              end
   end
 
+  def raw_sped
+    @raw_sped ||= value_from(pattern: /Special\s*Ed\s*Status|Raw\s*SpEd/i)
+  end
+
+  def sped
+    @sped ||= case raw_sped
+              in /active/i
+                "Special Education"
+              in /^NA$|^#NA$/i
+                "Unknown"
+              else
+                "Not Special Education"
+              end
+  end
+
   def value_from(pattern:)
     output = nil
     matches = headers.select do |header|

app/services/survey_responses_data_loader.rb

@@ -62,6 +62,10 @@ class SurveyResponsesDataLoader
     @ells ||= Ell.by_designation
   end
 
+  def speds
+    @speds ||= Sped.by_designation
+  end
+
   def process_row(row:, rules:)
     return unless row.dese_id?
     return unless row.school.present?
@@ -91,12 +95,14 @@ class SurveyResponsesDataLoader
     grade = row.grade
     income = incomes[row.income.parameterize]
     ell = ells[row.ell]
+    sped = speds[row.sped]
     if survey_item_response.present?
-      survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:)
+      survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:,
+                                   sped:)
       []
     else
       SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:,
-                             likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:)
+                             likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:)
     end
   end
 

app/views/analyze/_group_selectors.html.erb

@@ -25,3 +25,7 @@
 <% @presenter.ells.each do |ell| %>
   <%= render(partial: "checkboxes", locals: {id: "ell-#{ell.slug}", item: ell, selected_items: @presenter.selected_ells, name: "ell", label_text: ell.designation})  %>
 <% end %>
+
+<% @presenter.speds.each do |sped| %>
+  <%= render(partial: "checkboxes", locals: {id: "sped-#{sped.slug}", item: sped, selected_items: @presenter.selected_speds, name: "sped", label_text: sped.designation})  %>
+<% end %>

data/demographics.csv

@@ -1,11 +1,11 @@
-Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL
-1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged - N,ELL
-2,Asian or Pacific Islander,1,Female,Economically Disadvantaged - Y,Not ELL
-3,Black or African American,4,Non-Binary,Unknown,Unknown
-4,Hispanic or Latinx,99,Unknown,,
-5,White or Caucasian,,,,
-6,Prefer not to disclose,,,,
-7,Prefer to self-describe,,,,
-8,Middle Eastern,,,,
-99,Race/Ethnicity Not Listed,,,,
-100,Multiracial,,,,
+Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL,Special Ed Status
+1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged - N,ELL,Special Education
+2,Asian or Pacific Islander,1,Female,Economically Disadvantaged - Y,Not ELL,Not Special Education
+3,Black or African American,4,Non-Binary,Unknown,Unknown,Unknown
+4,Hispanic or Latinx,99,Unknown,,,
+5,White or Caucasian,,,,,
+6,Prefer not to disclose,,,,,
+7,Prefer to self-describe,,,,,
+8,Middle Eastern,,,,,
+99,Race/Ethnicity Not Listed,,,,,
+100,Multiracial,,,,,

db/migrate/20231004191828_create_speds.rb

@@ -0,0 +1,12 @@
+class CreateSpeds < ActiveRecord::Migration[7.0]
+  def change
+    create_table :speds do |t|
+      t.string :designation
+      t.string :slug
+
+      t.timestamps
+    end
+
+    add_index :speds, :designation
+  end
+end

db/migrate/20231004211430_add_sped_to_survey_item_response.rb

@@ -0,0 +1,5 @@
+class AddSpedToSurveyItemResponse < ActiveRecord::Migration[7.0]
+  def change
+    add_reference :survey_item_responses, :sped, foreign_key: true
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_09_12_223701) do
+ActiveRecord::Schema[7.0].define(version: 2023_10_04_211430) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -406,6 +406,14 @@ ActiveRecord::Schema[7.0].define(version: 2023_09_12_223701) do
     t.index ["school_id"], name: "index_scores_on_school_id"
   end
 
+  create_table "speds", force: :cascade do |t|
+    t.string "designation"
+    t.string "slug"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["designation"], name: "index_speds_on_designation"
+  end
+
   create_table "student_races", force: :cascade do |t|
     t.bigint "student_id", null: false
     t.bigint "race_id", null: false
@@ -448,6 +456,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_09_12_223701) do
     t.bigint "income_id"
     t.datetime "recorded_date"
     t.bigint "ell_id"
+    t.bigint "sped_id"
     t.index ["academic_year_id"], name: "index_survey_item_responses_on_academic_year_id"
     t.index ["ell_id"], name: "index_survey_item_responses_on_ell_id"
     t.index ["gender_id"], name: "index_survey_item_responses_on_gender_id"
@@ -457,6 +466,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_09_12_223701) do
     t.index ["school_id", "academic_year_id"], name: "index_survey_item_responses_on_school_id_and_academic_year_id"
     t.index ["school_id", "survey_item_id", "academic_year_id", "grade"], name: "index_survey_responses_on_grade"
     t.index ["school_id"], name: "index_survey_item_responses_on_school_id"
+    t.index ["sped_id"], name: "index_survey_item_responses_on_sped_id"
     t.index ["student_id"], name: "index_survey_item_responses_on_student_id"
     t.index ["survey_item_id"], name: "index_survey_item_responses_on_survey_item_id"
   end
@@ -505,6 +515,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_09_12_223701) do
   add_foreign_key "survey_item_responses", "genders"
   add_foreign_key "survey_item_responses", "incomes"
   add_foreign_key "survey_item_responses", "schools"
+  add_foreign_key "survey_item_responses", "speds"
   add_foreign_key "survey_item_responses", "students"
   add_foreign_key "survey_item_responses", "survey_items"
   add_foreign_key "survey_items", "scales"

spec/fixtures/sample_demographics.csv

@@ -1,11 +1,11 @@
-Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL
-1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged – N,ELL
-2,Asian or Pacific Islander,1,Female,Economically Disadvantaged – Y,Not ELL
-3,Black or African American,4,Non-Binary,Unknown,Unknown
-4,Hispanic or Latinx,99,Unknown,,
-5,White or Caucasian,,,,
-6,Prefer not to disclose,,,,
-7,Prefer to self-describe,,,,
-8,Middle Eastern,,,,
-99,Race/Ethnicity Not Listed,,,,
-100,Multiracial,,,,
+Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL,Special Ed Status
+1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged – N,ELL,Special Education
+2,Asian or Pacific Islander,1,Female,Economically Disadvantaged – Y,Not ELL,Not Special Education
+3,Black or African American,4,Non-Binary,Unknown,Unknown,Unknown
+4,Hispanic or Latinx,99,Unknown,,,
+5,White or Caucasian,,,,,
+6,Prefer not to disclose,,,,,
+7,Prefer to self-describe,,,,,
+8,Middle Eastern,,,,,
+99,Race/Ethnicity Not Listed,,,,,
+100,Multiracial,,,,,

spec/fixtures/test_2020-21_student_survey_responses.csv

@@ -1,8 +1,8 @@
-Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,LASID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district,school,DESE ID,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,#N/A,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,#N/A,#N/A,s-cure-q3,s-cure-q4,#N/A,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,#N/A,#N/A,s-grmi-q4,s-appa-q1,s-appa-q2,#N/A,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q1-1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-acst-q4,s-acst-q5,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,race,What is your race/ethnicity?(Please select all that apply) - Selected Choice,grade,gender,Raw Income,Income,Raw ELL,ELL
-2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,1500025,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888,11th,1,Free Lunch,Economically Disadvantaged – Y,Does not apply,Not ELL
-2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,"2,3,4",888,10,,Not Eligible,Economically Disadvantaged – N,,Unknown
-2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1500505,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,6,888,8,2,Reduced Lunch,Economically Disadvantaged – Y,,Unknown
-2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1500505,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888,8,3,,Unknown,LEP student not 1st year,ELL
-2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888,7,4,Free Lunch,Economically Disadvantaged – Y,EL Student First Year,ELL
-2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888,3,NA,Not Eligible,Economically Disadvantaged – N,Unknown,Unknown
-2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,,,4,,Reduced Lunch,Economically Disadvantaged – Y,#N/A,Unknown
+Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,LASID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district,school,DESE ID,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,#N/A,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,#N/A,#N/A,s-cure-q3,s-cure-q4,#N/A,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,#N/A,#N/A,s-grmi-q4,s-appa-q1,s-appa-q2,#N/A,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q1-1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-acst-q4,s-acst-q5,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,race,What is your race/ethnicity?(Please select all that apply) - Selected Choice,grade,gender,Raw Income,Income,Raw ELL,ELL,Raw SpEd
+2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,1500025,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888,11th,1,Free Lunch,Economically Disadvantaged – Y,Does not apply,Not ELL,exited
+2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,"2,3,4",888,10,,Not Eligible,Economically Disadvantaged – N,,Unknown,Unknown
+2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1500505,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,6,888,8,2,Reduced Lunch,Economically Disadvantaged – Y,,Unknown,Unknown
+2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1500505,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888,8,3,,Unknown,LEP student not 1st year,ELL,active
+2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888,7,4,Free Lunch,Economically Disadvantaged – Y,EL Student First Year,ELL,active
+2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888,3,NA,Not Eligible,Economically Disadvantaged – N,Unknown,Unknown,NA
+2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,,,4,,Reduced Lunch,Economically Disadvantaged – Y,#N/A,Unknown,

spec/services/cleaner_spec.rb

@@ -219,7 +219,7 @@ def reads_headers_from_raw_csv(processed_data)
                                      "s-grit-q1", "s-grit-q2", "s-grit-q3", "s-grit-q4", "s-expa-q1", "s-poaf-q1", "s-poaf-q2", "s-poaf-q3",
                                      "s-poaf-q4", "s-tint-q1-1", "s-tint-q2-1", "s-tint-q3-1", "s-tint-q4-1", "s-tint-q5-1", "s-acpr-q1-1",
                                      "s-acpr-q2-1", "s-acpr-q3-1", "s-acpr-q4-1", "s-peff-q1-1", "s-peff-q2-1", "s-peff-q3-1", "s-peff-q4-1",
-                                     "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL"].to_set.sort
+                                     "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd"].to_set.sort
 end
 
 def invalid_rows_are_rejected_for_the_correct_reasons(data)

spec/services/demographic_loader_spec.rb

@@ -21,6 +21,10 @@ describe DemographicLoader do
     ["ELL", "Not ELL", "Unknown"]
   end
 
+  let(:speds) do
+    ["Special Education", "Not Special Education", "Unknown"]
+  end
+
   before :each do
     DemographicLoader.load_data(filepath:)
   end
@@ -68,5 +72,12 @@ describe DemographicLoader do
         expect(Ell.find_by_designation(ell).designation).to eq ell
       end
     end
+
+    it "load all the special ed designations" do
+      expect(Sped.all.count).to eq 3
+      speds.each do |sped|
+        expect(Sped.find_by_designation(sped).designation).to eq sped
+      end
+    end
   end
 end

spec/services/survey_item_values_spec.rb

@@ -252,6 +252,44 @@ RSpec.describe SurveyItemValues, type: :model do
     end
   end
 
+  context ".sped" do
+    before :each do
+      attleboro
+      ay_2022_23
+    end
+
+    it 'translates "active" into "Special Education"' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "active" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.sped).to eq "Special Education"
+    end
+
+    it 'translates "exited" into "Not Special Education"' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "exited" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.sped).to eq "Not Special Education"
+    end
+    it 'translates blanks into "Not Special Education' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.sped).to eq "Not Special Education"
+    end
+
+    it 'tranlsates NA into "Unknown"' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "NA" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.sped).to eq "Unknown"
+
+      row = { "Raw SpEd" => "#NA" }
+      values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
+      expect(values.sped).to eq "Unknown"
+    end
+  end
+
   context ".valid_duration" do
     context "when duration is valid" do
       it "returns true" do