Finishes #183092367. Adds ui and graphs to show grade level disaggregations

app/controllers/analyze_controller.rb

@@ -2,7 +2,8 @@
 
 class AnalyzeController < SqmApplicationController
   before_action :assign_categories, :assign_subcategories, :assign_measures, :assign_academic_years,
-                :response_rate_timestamp, :races, :selected_races, :graph, :graphs, :background, :race_score_timestamp, only: [:index]
+                :response_rate_timestamp, :races, :selected_races, :graph, :graphs, :background, :race_score_timestamp,
+                :sources, :group, :groups, :selected_grades, :grades, :slice, only: [:index]
   def index; end
 
   private
@@ -67,14 +68,15 @@ class AnalyzeController < SqmApplicationController
 
   def graph
     graphs.each do |graph|
-      @graph = graph if graph.value == params[:graph]
+      @graph = graph if graph.slug == params[:graph]
     end
 
     @graph ||= graphs.first
   end
 
   def graphs
-    @graphs ||= [Analyze::Graph::StudentsAndTeachers.new, Analyze::Graph::StudentsByGroup.new(races: selected_races)]
+    @graphs ||= [Analyze::Graph::StudentsAndTeachers.new, Analyze::Graph::StudentsByRace.new(races: selected_races),
+                 Analyze::Graph::StudentsByGrade.new(grades: selected_grades)]
   end
 
   def background
@@ -88,4 +90,61 @@ class AnalyzeController < SqmApplicationController
       score.updated_at
     end
   end
+
+  def sources
+    @sources = [Analyze::Source::SurveyData.new(slices:)]
+  end
+
+  def slices
+    students_and_teachers = Analyze::Slice::StudentsAndTeachers.new
+    students_by_group = Analyze::Slice::StudentsByGroup.new(races:, grades:)
+    [students_and_teachers, students_by_group]
+  end
+
+  def group
+    groups.each do |group|
+      @group = group if group.slug == params[:group]
+    end
+
+    @group ||= groups.first
+  end
+
+  def groups
+    @groups = [Analyze::Group::Race.new, Analyze::Group::Grade.new]
+  end
+
+  def selected_grades
+    @selected_grades ||= begin
+      grade_params = params[:grades]
+      return @selected_grades = grades unless grade_params
+
+      grade_list = grade_params.split(',') if grade_params
+      if grade_list
+        grade_list = grade_list.map do |grade|
+          grade.to_i
+        end
+      end
+      grade_list
+    end
+  end
+
+  def grades
+    @grades ||= SurveyItemResponse.where(school: @school, academic_year: @academic_year)
+                                  .where.not(grade: nil)
+                                  .group(:grade)
+                                  .select(:response_id)
+                                  .distinct(:response_id)
+                                  .count.reject do |_key, value|
+                                    value < 10
+                                  end.keys
+  end
+
+  def slice
+    slice_param = params[:slice]
+    slices.each do |slice|
+      @slice = slice if slice.slug == slice_param
+    end
+
+    @slice ||= slices.first
+  end
 end

app/javascript/controllers/analyze_controller.js

@@ -10,10 +10,16 @@ export default class extends Controller {
       base_url +
       "&academic_years=" +
       this.selected_years().join(",") +
+      "&group=" +
+      this.selected_group() +
+      "&slice=" +
+      this.selected_slice() +
       "&graph=" +
       this.selected_graph() +
       "&races=" +
-      this.selected_races().join(",");
+      this.selected_races().join(",") +
+      "&grades=" +
+      this.selected_grades().join(",");
 
     this.go_to(url);
   }
@@ -36,17 +42,50 @@ export default class extends Controller {
     return years;
   }
 
+  selected_group() {
+    let groups = [...document.getElementsByName("group-option")];
+    let selected_group = groups
+      .filter((item) => {
+        return item.selected;
+      })
+      .map((item) => {
+        return item.id;
+      });
+
+    return selected_group[0];
+  }
+
+  selected_slice() {
+    let slices = [...document.getElementsByName("slice")];
+    let selected_slice = slices
+      .filter((item) => {
+        return item.checked;
+      })
+      .map((item) => {
+        return item.id;
+      });
+
+    return selected_slice[0];
+  }
+
   selected_graph() {
-    let graphs = [...document.getElementsByName("graph")];
+    let graphs = [...document.getElementsByName("slice")];
     let selected_graph = graphs
       .filter((item) => {
         return item.checked;
       })
       .map((item) => {
         return item.id;
-      });
+      })[0];
+    if (selected_graph === 'students-and-teachers') {
+      return selected_graph;
+    }
 
-    return selected_graph[0];
+    if (this.selected_group() === 'race') {
+      return 'students-by-race'
+    } else {
+      return 'students-by-grade'
+    }
   }
 
   selected_races() {
@@ -61,4 +100,17 @@ export default class extends Controller {
 
     return races;
   }
+
+  selected_grades() {
+    let grade_checkboxes = [...document.getElementsByName("grade-checkbox")]
+    let grades = grade_checkboxes
+      .filter((item) => {
+        return item.checked;
+      })
+      .map((item) => {
+        return item.id.replace('grade-', '');
+      });
+
+    return grades;
+  }
 }

app/models/measure.rb

@@ -145,7 +145,7 @@ class Measure < ActiveRecord::Base
     meets_student_threshold = sufficient_student_data?(school:, academic_year:)
     meets_teacher_threshold = sufficient_teacher_data?(school:, academic_year:)
     meets_admin_data_threshold = any_admin_data_collected?(school:, academic_year:)
-    Score.new(average, meets_teacher_threshold, meets_student_threshold, meets_admin_data_threshold)
+    Score.new(average:, meets_teacher_threshold:, meets_student_threshold:, meets_admin_data_threshold:)
   end
 
   def collect_survey_item_average(survey_items:, school:, academic_year:)

app/models/score.rb

@@ -1,7 +1,20 @@
 # frozen_string_literal: true
 
-class Score < Struct.new(:average, :meets_teacher_threshold?, :meets_student_threshold?, :meets_admin_data_threshold?)
-  NIL_SCORE = Score.new(nil, false, false, false)
+class Score < ApplicationRecord
+  belongs_to :measure
+  belongs_to :school
+  belongs_to :academic_year
+  belongs_to :race
+
+  NIL_SCORE = Score.new(average: nil, meets_teacher_threshold: false, meets_student_threshold:  false, meets_admin_data_threshold: false)
+
+  enum group: {
+    all_students: 0,
+    race: 1,
+    grade: 2,
+    gender: 3
+  }
+
   def in_zone?(zone:)
     return false if average.nil? || average.is_a?(Float) && average.nan?
 

app/models/survey_item_response.rb

@@ -15,4 +15,10 @@ class SurveyItemResponse < ActiveRecord::Base
                            boston = District.find_by_name('Boston')
                            where.not(school: boston.schools) if boston.present?
                          }
+
+  scope :averages_for_grade, ->(survey_items, school, academic_year, grade) {
+            SurveyItemResponse.where(survey_item: survey_items, school:,
+                                                academic_year: , grade:).group(:survey_item).average(:likert_score)
+  }
+
 end

app/presenters/analyze/graph/column/grade/eight.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class Eight < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 8'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            8
+          end
+        end
+      end
+    end
+  end
+end
+

app/presenters/analyze/graph/column/grade/eleven.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class Eleven < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 11'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            11
+          end
+        end
+      end
+    end
+  end
+end
+

app/presenters/analyze/graph/column/grade/five.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class Five < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 5'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            5
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/grade/four.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class Four < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 4'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            4
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/grade/nine.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class Nine < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 9'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            9
+          end
+        end
+      end
+    end
+  end
+end
+
+

app/presenters/analyze/graph/column/grade/one.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class One < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 1'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            1
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/grade/score_for_grade.rb

@@ -0,0 +1,27 @@
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        module ScoreForGrade
+          def score(year_index)
+            averages = SurveyItemResponse.averages_for_grade(measure.student_survey_items, school, academic_years[year_index], grade)
+            average = bubble_up_averages(averages:)
+
+            Score.new(average:,
+                      meets_teacher_threshold: false,
+                      meets_student_threshold: true,
+                      meets_admin_data_threshold: false)
+          end
+
+          def bubble_up_averages(averages:)
+            measure.student_scales.map do |scale|
+              scale.survey_items.map do |survey_item|
+                averages[survey_item]
+              end.remove_blanks.average
+            end.remove_blanks.average
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/grade/seven.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class Seven < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 7'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            7
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/grade/six.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class Six < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 6'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            6
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/grade/ten.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class Ten < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 10'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            10
+          end
+        end
+      end
+    end
+  end
+end
+
+

app/presenters/analyze/graph/column/grade/three.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class Three < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 3'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            3
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/grade/twelve.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class Twelve < GroupedBarColumnPresenter
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 12'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            12
+          end
+        end
+      end
+    end
+  end
+end
+
+

app/presenters/analyze/graph/column/grade/two.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Analyze
+  module Graph
+    module Column
+      module Grade
+        class Two < GroupedBarColumnPresenter
+          attr_reader :sufficient_responses
+
+          include Analyze::Graph::Column::Grade::ScoreForGrade
+          def label
+            'Grade 2'
+          end
+
+          def basis
+            'student'
+          end
+
+          def show_irrelevancy_message?
+            false
+          end
+
+          def show_insufficient_data_message?
+            false
+          end
+
+          def grade
+            2
+          end
+        end
+      end
+    end
+  end
+end

app/presenters/analyze/graph/column/score_for_race.rb

@@ -8,7 +8,10 @@ module Analyze
           average ||= 0
           meets_student_threshold = s.meets_student_threshold? unless s.nil?
           meets_student_threshold ||= false
-          Score.new(average, false, meets_student_threshold, false)
+          Score.new(average:,
+                    meets_teacher_threshold: false,
+                    meets_student_threshold:,
+                    meets_admin_data_threshold: false)
         end
       end
     end

app/presenters/analyze/graph/students_and_teachers.rb

@@ -8,7 +8,7 @@ module Analyze
         'Students & Teachers'
       end
 
-      def value
+      def slug
         'students-and-teachers'
       end
 

app/presenters/analyze/graph/students_by_grade.rb

@@ -0,0 +1,50 @@
+module Analyze
+  module Graph
+    class StudentsByGrade
+      include Analyze::Graph::Column::Grade
+      attr_reader :grades
+
+      def initialize(grades:)
+        @grades = grades
+      end
+
+      def to_s
+        'Students by Grade'
+      end
+
+      def slug
+        'students-by-grade'
+      end
+
+      def columns
+        [].tap do |array|
+          grades.each do |grade|
+            array << column_for_grade_code(code: grade)
+          end
+          array << Analyze::Graph::Column::AllStudent
+        end
+      end
+
+      private
+
+      def column_for_grade_code(code:)
+        CFR[code]
+      end
+
+      CFR = {
+        1 => One,
+        2 => Two,
+        3 => Three,
+        4 => Four,
+        5 => Five,
+        6 => Six,
+        7 => Seven,
+        8 => Eight,
+        9 => Nine,
+        10 => Ten,
+        11 => Eleven,
+        12 => Twelve,
+      }.freeze
+    end
+  end
+end

app/presenters/analyze/graph/students_by_race.rb

@@ -1,6 +1,6 @@
 module Analyze
   module Graph
-    class StudentsByGroup
+    class StudentsByRace
       attr_reader :races
 
       def initialize(races:)
@@ -8,11 +8,11 @@ module Analyze
       end
 
       def to_s
-        'Students by Group'
+        'Students by Race'
       end
 
-      def value
-        'students-by-group'
+      def slug
+        'students-by-race'
       end
 
       def columns
@@ -39,7 +39,7 @@ module Analyze
         '8' => Analyze::Graph::Column::MiddleEastern,
         '99' => Analyze::Graph::Column::Unknown,
         '100' => Analyze::Graph::Column::Multiracial
-      }
+      }.freeze
     end
   end
 end

app/presenters/analyze/group/grade.rb

@@ -0,0 +1,13 @@
+module Analyze
+  module Group
+    class Grade
+      def name
+        'Grade'
+      end
+
+      def slug
+        'grade'
+      end
+    end
+  end
+end

app/presenters/analyze/group/race.rb

@@ -0,0 +1,13 @@
+module Analyze
+  module Group
+    class Race
+      def name
+        'Race'
+      end
+
+      def slug
+        'race'
+      end
+    end
+  end
+end

app/presenters/analyze/slice/students_and_teachers.rb

@@ -0,0 +1,17 @@
+module Analyze
+  module Slice
+    class StudentsAndTeachers
+      def to_s
+        'Students & Teachers'
+      end
+
+      def slug
+        'students-and-teachers'
+      end
+
+      def graphs
+        [Analyze::Graph::StudentsAndTeachers.new]
+      end
+    end
+  end
+end

app/presenters/analyze/slice/students_by_group.rb

@@ -0,0 +1,24 @@
+module Analyze
+  module Slice
+    class StudentsByGroup
+      attr_reader :races, :grades
+
+      def initialize(races:, grades:)
+        @races = races
+        @grades = grades
+      end
+
+      def to_s
+        'Students by Group'
+      end
+
+      def slug
+        'students-by-group'
+      end
+
+      def graphs
+        [Analyze::Graph::StudentsByRace.new(races:), Analyze::Graph::StudentsByGrade.new(grades:)]
+      end
+    end
+  end
+end

app/presenters/analyze/source/survey_data.rb

@@ -0,0 +1,20 @@
+module Analyze
+  module Source
+    class SurveyData
+      attr_reader :slices
+
+      include Analyze::Slice
+
+      def initialize(slices:)
+        @slices = slices
+      end
+      # def to_s
+      #   'Survey Data Only'
+      # end
+
+      # def value
+      #   'survey-data-only'
+      # end
+    end
+  end
+end

app/presenters/zones.rb

@@ -42,6 +42,6 @@ class Zones
   end
 
   def zone_for_score(score)
-    all_zones.find { |zone| Score.new(score).in_zone?(zone:) } || insufficient_data
+    all_zones.find { |zone| Score.new(average: score).in_zone?(zone:) } || insufficient_data
   end
 end

app/services/race_score_loader.rb

@@ -45,15 +45,13 @@ class RaceScoreLoader
 
   def self.race_score(measure:, school:, academic_year:, race:)
     rate = response_rate(school:, academic_year:, measure:)
-    return Score.new(0, false, false, false) unless rate.meets_student_threshold
+    return Score.new(average: 0, meets_teacher_threshold: false, meets_student_threshold: false, meets_admin_data_threshold: false) unless rate.meets_student_threshold
 
     survey_items = measure.student_survey_items
 
-    # students = StudentRace.where(race:).pluck(:student_id).uniq
     averages = grouped_responses(school:, academic_year:, survey_items:, race:)
     meets_student_threshold = sufficient_responses(school:, academic_year:, race:)
     scorify(responses: averages, meets_student_threshold:, measure:)
-    # binding.break
   end
 
   def self.grouped_responses(school:, academic_year:, survey_items:, race:)
@@ -83,7 +81,7 @@ class RaceScoreLoader
 
     average = 0 unless meets_student_threshold
 
-    Score.new(average, false, meets_student_threshold, false)
+    Score.new(average:, meets_teacher_threshold: false, meets_student_threshold:, meets_admin_data_threshold: false)
   end
 
   def self.sufficient_responses(school:, academic_year:, race:)

app/services/survey_responses_data_loader.rb

@@ -6,11 +6,10 @@ class SurveyResponsesDataLoader
   def self.load_data(filepath:)
     File.open(filepath) do |file|
       headers = file.first
-      survey_items = SurveyItem.where(survey_item_id: get_survey_item_ids_from_headers(headers:))
 
       file.lazy.each_slice(1000) do |lines|
         survey_item_responses = CSV.parse(lines.join, headers:).map do |row|
-          process_row row: row, survey_items: survey_items
+          process_row row: Values.new(row:, headers:)
         end
 
         SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 1000
@@ -20,89 +19,102 @@ class SurveyResponsesDataLoader
 
   private
 
-  def self.process_row(row:, survey_items:)
-    id = dese_id(row)
-    return unless dese_id?(id)
+  def self.process_row(row:)
+    return unless row.dese_id?
+    return unless row.school.present?
 
-    school = School.find_by_dese_id(id)
-    return unless school.present?
-
-    process_survey_items(row:, survey_items:, school:)
+    process_survey_items(row:)
   end
 
-  def self.process_survey_items(row:, survey_items:, school:)
-    id = response_id(row)
-    survey_items.map do |survey_item|
-      likert_score = row[survey_item.survey_item_id] || next
+  def self.process_survey_items(row:)
+    row.survey_items.map do |survey_item|
+      likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next
 
       unless likert_score.valid_likert_score?
-        puts "Response ID: #{id}, Likert score: #{likert_score} rejected" unless likert_score == 'NA'
+        puts "Response ID: #{row.response_id}, Likert score: #{likert_score} rejected" unless likert_score == 'NA'
         next
       end
-      response = survey_item_response(response_id: id, survey_item:)
-      create_or_update_response(survey_item_response: response, likert_score:, school:, row:, survey_item:)
+      response = row.survey_item_response(survey_item:)
+      create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:)
     end.compact
   end
 
-  def self.create_or_update_response(survey_item_response:, likert_score:, school:, row:, survey_item:)
+  def self.create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:)
     if survey_item_response.present?
-      survey_item_response.update!(likert_score:) if survey_item_response.likert_score != likert_score
+      survey_item_response.update!(likert_score:)
       []
     else
-      SurveyItemResponse.new(response_id: response_id(row), academic_year: academic_year(row), school:, survey_item:,
-                             likert_score:)
+      SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:,
+                             likert_score:, grade: row.grade)
     end
   end
 
-  def self.get_survey_item_ids_from_headers(headers:)
-    CSV.parse(headers, headers: true).headers
-       .filter(&:present?)
-       .filter { |header| header.start_with? 't-' or header.start_with? 's-' }
+  private_class_method :process_row
+  private_class_method :process_survey_items
+  private_class_method :create_or_update_response
+end
+
+class Values
+  attr_reader :row, :headers
+
+  def initialize(row:, headers:)
+    @row = row
+    @headers = headers
   end
 
-  def self.dese_id?(dese_id)
+  def dese_id?
     dese_id.present?
   end
 
-  def self.response_date(row)
-    Date.parse(row['Recorded Date'] || row['RecordedDate'])
+  def response_date
+    @response_date ||= Date.parse(row['Recorded Date'] || row['RecordedDate'])
   end
 
-  def self.academic_year(row)
-    AcademicYear.find_by_date response_date(row)
+  def academic_year
+    @academic_year ||= AcademicYear.find_by_date response_date
   end
 
-  def self.survey_item_response(response_id:, survey_item:)
+  def survey_item_response(survey_item:)
     SurveyItemResponse.find_by(response_id:, survey_item:)
   end
 
-  def self.response_id(row)
-    row['Response ID'] || row['ResponseId'] || row['ResponseID']
+  def response_id
+    @response_id ||= row['Response ID'] || row['ResponseId'] || row['ResponseID']
   end
 
-  def self.dese_id(row)
-    row['DESE ID' || 'Dese ID'] || row['DeseId'] || row['DeseID']
+  def dese_id
+    @dese_id ||= (row['DESE ID' || 'Dese ID'] || row['DeseId'] || row['DeseID']).to_i
   end
 
-  private_class_method :process_row
-  private_class_method :process_survey_items
-  private_class_method :get_survey_item_ids_from_headers
-  private_class_method :dese_id?
-  private_class_method :create_or_update_response
-  private_class_method :response_date
-  private_class_method :academic_year
-  private_class_method :survey_item_response
-  private_class_method :response_id
-  private_class_method :dese_id
-end
+  def likert_score(survey_item_id:)
+    row[survey_item_id]
+  end
 
-module StringMonkeyPatches
-  def integer?
-    to_i.to_s == self
+  def school
+    @school ||= School.find_by_dese_id(dese_id)
+  end
+
+  def survey_items
+    @survey_items ||= SurveyItem.where(survey_item_id: get_survey_item_ids_from_headers(headers:))
+  end
+
+  def get_survey_item_ids_from_headers(headers:)
+    CSV.parse(headers, headers: true).headers
+       .filter(&:present?)
+       .filter { |header| header.start_with? 't-' or header.start_with? 's-' }
   end
 
+  def grade
+    @grade ||= begin
+      raw_grade = (row['grade'] || row['Grade'] || row['What grade are you in?']).to_i
+      raw_grade == 0 ? nil : raw_grade
+    end
+  end
+end
+
+module StringMonkeyPatches
   def valid_likert_score?
-    integer? and to_i.between? 1, 5
+    to_i.between? 1, 5
   end
 end
 

app/views/analyze/_data_filters.html.erb

@@ -1,23 +1,28 @@
-<%# TODO Hook up the buttons so that only the selected races are shown for each column %>
 <h3 class="sub-header-4 mt-5">Data Filters</h3>
 <div class="bg-gray p-3" data-controller="analyze">
-  <% @graphs.each do |graph| %>
+<% @sources.first.slices.each do |slice| %>
   <div>
     <input type="radio"
-           id="<%= graph.value %>"
+           id="<%= slice.slug %>"
            class="form-check-input"
-           name="graph"
+           name="slice"
             value="<%= base_url %>"
             data-action="click->analyze#refresh"
-            <%= graph.value == @graph.value ? "checked" : "" %>>
-    <label for="<%= graph.value %>"><%= graph.to_s %></label>
+            <%= slice.slug == @slice.slug ? "checked" : "" %>>
+            <label for="<%= slice.slug %>"><%= slice.to_s %></label>
   </div>
+<% end %>
+
+<select id="select-group" class="mx-3 form-select" data-id="group-dropdown" data-action="analyze#refresh">
+  <% @groups.each do |group| %>
+    <option id="<%= group.slug %>" name="group-option" value="<%= base_url %>" <%= group.slug == @group.slug ? "Selected": "" %>><%= group.name %> </option>
   <% end %>
+</select>
 
   <p class="sub-header-5 mt-3 font-size-14"> Select a group </p>
 
     <% @races.each do |race | %>
-      <div class="d-flex align-items-center <%= race.slug %>">
+      <div class="d-flex align-items-center">
         <input
           id="<%= race.slug %>"
           class="m-3 race-checkbox form-check-input"
@@ -26,9 +31,33 @@
           value="<%= base_url %>"
           data-action="click->analyze#refresh"
           <%= @selected_races.map(&:slug).include?(race.slug)  ? "checked" : "" %>
-          <%= @graph.value == 'students-and-teachers' ? "disabled" : "" %>>
+          <%= @graph.slug == 'students-and-teachers' ? "disabled" : "" %>
+          <%= @group.slug == 'race' ? "" : "hidden" %>>
+
+          <label for="<%= race.qualtrics_code %>"
+            <%= @group.slug == 'race' ? "" : "hidden" %>>
+              <%= race.designation %>
+          </label>
+      </div>
+    <% end %>
+
+    <% @grades.each do |grade | %>
+      <div class="d-flex align-items-center">
+        <input
+          id="grade-<%= grade %>"
+          class="m-3 grade-checkbox form-check-input"
+          type="checkbox"
+          name="grade-checkbox"
+          value="<%= base_url %>"
+          data-action="click->analyze#refresh"
+          <%= @selected_grades.include?(grade)  ? "checked" : "" %>
+          <%= @graph.slug == 'students-and-teachers' ? "disabled" : "" %>
+          <%= @group.slug == 'grade' ? "" : "hidden" %>>
 
-      <label for="<%= race.qualtrics_code %>"><%= race.designation %></label>
+          <label for="grade-<%= grade %>"
+            <%= @group.slug == 'grade' ? "" : "hidden" %>>
+              <%= grade %>
+          </label>
       </div>
     <% end %>
 </div>

app/views/analyze/index.html.erb

@@ -1,6 +1,7 @@
 <% content_for :title do %>
   <h1 class="sub-header-2 color-white m-0"> Analysis of <%= @school.name %> </h1>
 <% end %>
+
 <div class="graph-content">
   <div class="breadcrumbs sub-header-4">
     <%= @category.category_id %>:<%= @category.name %> > <%= @subcategory.subcategory_id %>:<%= @subcategory.name %>

db/migrate/20220927213911_add_grade_to_survey_item_response.rb

@@ -0,0 +1,5 @@
+class AddGradeToSurveyItemResponse < ActiveRecord::Migration[7.0]
+  def change
+    add_column :survey_item_responses, :grade, :integer
+  end
+end

db/migrate/20221015002312_create_scores.rb

@@ -0,0 +1,18 @@
+class CreateScores < ActiveRecord::Migration[7.0]
+  def change
+    create_table :scores do |t|
+      t.float :average
+      t.boolean :meets_teacher_threshold
+      t.boolean :meets_student_threshold
+      t.boolean :meets_admin_data_threshold
+      t.integer :group
+      t.references :measure, null: false, foreign_key: true
+      t.references :school, null: false, foreign_key: true
+      t.references :academic_year, null: false, foreign_key: true
+      t.integer :grade
+      t.references :race, null: false, foreign_key: true
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20221015023621_add_grade_index_to_survey_item_response.rb

@@ -0,0 +1,5 @@
+class AddGradeIndexToSurveyItemResponse < ActiveRecord::Migration[7.0]
+  def change
+    add_index :survey_item_responses, [:school_id, :survey_item_id, :academic_year_id, :grade], name: "index_survey_responses_on_grade"
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2022_08_22_214951) do
+ActiveRecord::Schema[7.0].define(version: 2022_10_15_023621) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_stat_statements"
   enable_extension "plpgsql"
@@ -339,7 +339,6 @@ ActiveRecord::Schema[7.0].define(version: 2022_08_22_214951) do
     t.datetime "updated_at", null: false
     t.index ["academic_year_id"], name: "index_response_rates_on_academic_year_id"
     t.index ["school_id", "subcategory_id"], name: "index_response_rates_on_school_id_and_subcategory_id"
-    t.index ["school_id"], name: "index_response_rates_on_school_id"
     t.index ["subcategory_id"], name: "index_response_rates_on_subcategory_id"
   end
 
@@ -366,6 +365,25 @@ ActiveRecord::Schema[7.0].define(version: 2022_08_22_214951) do
     t.index ["dese_id"], name: "index_schools_on_dese_id", unique: true
   end
 
+  create_table "scores", force: :cascade do |t|
+    t.float "average"
+    t.boolean "meets_teacher_threshold"
+    t.boolean "meets_student_threshold"
+    t.boolean "meets_admin_data_threshold"
+    t.integer "group"
+    t.bigint "measure_id", null: false
+    t.bigint "school_id", null: false
+    t.bigint "academic_year_id", null: false
+    t.integer "grade"
+    t.bigint "race_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["academic_year_id"], name: "index_scores_on_academic_year_id"
+    t.index ["measure_id"], name: "index_scores_on_measure_id"
+    t.index ["race_id"], name: "index_scores_on_race_id"
+    t.index ["school_id"], name: "index_scores_on_school_id"
+  end
+
   create_table "student_races", force: :cascade do |t|
     t.bigint "student_id", null: false
     t.bigint "race_id", null: false
@@ -403,10 +421,11 @@ ActiveRecord::Schema[7.0].define(version: 2022_08_22_214951) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.bigint "student_id"
+    t.integer "grade"
     t.index ["academic_year_id"], name: "index_survey_item_responses_on_academic_year_id"
     t.index ["response_id"], name: "index_survey_item_responses_on_response_id"
     t.index ["school_id", "academic_year_id"], name: "index_survey_item_responses_on_school_id_and_academic_year_id"
-    t.index ["school_id"], name: "index_survey_item_responses_on_school_id"
+    t.index ["school_id", "survey_item_id", "academic_year_id", "grade"], name: "index_survey_responses_on_grade"
     t.index ["student_id"], name: "index_survey_item_responses_on_student_id"
     t.index ["survey_item_id"], name: "index_survey_item_responses_on_survey_item_id"
   end
@@ -457,6 +476,10 @@ ActiveRecord::Schema[7.0].define(version: 2022_08_22_214951) do
   add_foreign_key "response_rates", "subcategories"
   add_foreign_key "scales", "measures"
   add_foreign_key "schools", "districts"
+  add_foreign_key "scores", "academic_years"
+  add_foreign_key "scores", "measures"
+  add_foreign_key "scores", "races"
+  add_foreign_key "scores", "schools"
   add_foreign_key "student_races", "races"
   add_foreign_key "student_races", "students"
   add_foreign_key "subcategories", "categories"

lib/tasks/one_off.rake

@@ -40,14 +40,10 @@ namespace :one_off do
     end
   end
 
-  task change_overall_performance_measure_id: :environment do
-    measure_4aii = Measure.where(name: 'Overall Performance')[0]
-    measure_4aii.update! measure_id: '4A-i'
-  end
-
   desc 'load a single file'
   task load_single_file: :environment do
-    filepath = Rails.root.join('data', 'survey_responses', '2016-17_student_survey_responses.csv')
+    filepath = Rails.root.join('data', 'survey_responses',
+                               '2021-22_revere_somerville_wareham_student_survey_responses.csv')
     puts "=====================> Loading data from csv at path: #{filepath}"
     SurveyResponsesDataLoader.load_data filepath: filepath
     puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses"
@@ -78,44 +74,6 @@ namespace :one_off do
     puts "=====================> Completed loading #{Student.count} survey responses"
   end
 
-  desc 'load revere somerville warehame results for 2021-22'
-  task load_revere: :environment do
-    ['2021-22_revere_somerville_wareham_student_survey_responses.csv',
-     '2021-22_revere_somerville_wareham_teacher_survey_responses.csv'].each do |filepath|
-      filepath = Rails.root.join('data', 'survey_responses', filepath)
-      puts "=====================> Loading data from csv at path: #{filepath}"
-      SurveyResponsesDataLoader.load_data filepath:
-    end
-    puts 'Resetting response rates'
-    revere = District.find_by_name 'Revere'
-    somerville = District.find_by_name 'Somerville'
-    wareham = District.find_by_name 'Wareham'
-    academic_year = AcademicYear.find_by_range '2021-22'
-    ResponseRateLoader.reset(schools: revere.schools, academic_years: [academic_year])
-    ResponseRateLoader.reset(schools: somerville.schools, academic_years: [academic_year])
-    ResponseRateLoader.reset(schools: wareham.schools, academic_years: [academic_year])
-    Rails.cache.clear
-    puts "=====================> Completed recalculating #{ResponseRate.count} response rates"
-  end
-
-  desc 'load lowell and milford results for 2021-22'
-  task load_lowell: :environment do
-    ['2021-22_lowell_milford_student_survey_responses.csv',
-     '2021-22_lowell_milford_teacher_survey_responses.csv'].each do |filepath|
-      filepath = Rails.root.join('data', 'survey_responses', filepath)
-      puts "=====================> Loading data from csv at path: #{filepath}"
-      SurveyResponsesDataLoader.load_data filepath:
-    end
-    puts 'Resetting response rates'
-    lowell = District.find_by_name 'Lowell'
-    milford = District.find_by_name 'Milford'
-    academic_year = AcademicYear.find_by_range '2021-22'
-    ResponseRateLoader.reset(schools: lowell.schools, academic_years: [academic_year])
-    ResponseRateLoader.reset(schools: milford.schools, academic_years: [academic_year])
-    Rails.cache.clear
-    puts "=====================> Completed recalculating #{ResponseRate.count} response rates"
-  end
-
   desc 'reset race score calculations'
   task reset_race_scores_2021: :environment do
     puts 'Resetting race scores'
@@ -125,51 +83,6 @@ namespace :one_off do
     puts "=====================> Completed loading #{RaceScore.count} race scores"
   end
 
-  desc 'reset race score calculations'
-  task reset_race_scores_2020: :environment do
-    puts 'Resetting race scores'
-    academic_years = [AcademicYear.find_by_range('2020-21')]
-    RaceScoreLoader.reset(academic_years:, fast_processing: false)
-    Rails.cache.clear
-    puts "=====================> Completed loading #{RaceScore.count} race scores"
-  end
-
-  desc 'reset race score calculations'
-  task reset_race_scores_2019: :environment do
-    puts 'Resetting race scores'
-    academic_years = [AcademicYear.find_by_range('2019-20')]
-    RaceScoreLoader.reset(academic_years:, fast_processing: false)
-    Rails.cache.clear
-    puts "=====================> Completed loading #{RaceScore.count} race scores"
-  end
-
-  desc 'reset race score calculations'
-  task reset_race_scores_2018: :environment do
-    puts 'Resetting race scores'
-    academic_years = [AcademicYear.find_by_range('2018-19')]
-    RaceScoreLoader.reset(academic_years:, fast_processing: false)
-    Rails.cache.clear
-    puts "=====================> Completed loading #{RaceScore.count} race scores"
-  end
-
-  desc 'reset race score calculations'
-  task reset_race_scores_2017: :environment do
-    puts 'Resetting race scores'
-    academic_years = [AcademicYear.find_by_range('2017-18')]
-    RaceScoreLoader.reset(academic_years:, fast_processing: false)
-    Rails.cache.clear
-    puts "=====================> Completed loading #{RaceScore.count} race scores"
-  end
-
-  desc 'reset race score calculations'
-  task reset_race_scores_2016: :environment do
-    puts 'Resetting race scores'
-    academic_years = [AcademicYear.find_by_range('2016-17')]
-    RaceScoreLoader.reset(academic_years:, fast_processing: false)
-    Rails.cache.clear
-    puts "=====================> Completed loading #{RaceScore.count} race scores"
-  end
-
   desc 'list scales that have no survey responses'
   task list_scales_that_lack_survey_responses: :environment do
     output = AcademicYear.all.map do |academic_year|
@@ -202,24 +115,4 @@ namespace :one_off do
     end
     pp output
   end
-
-  desc 'delete invalid scale'
-  task delete_s_grmi_scale_from_2016_17: :environment do
-    academic_year = AcademicYear.find_by_range '2016-17'
-    survey_items = SurveyItem.where('survey_item_id LIKE ?', 's-grmi%')
-    SurveyItemResponse.joins(:survey_item).where(academic_year:, survey_item: survey_items).delete_all
-  end
-
-  desc 'reset admin data values'
-  task reset_admin_data_values: :environment do
-    puts "Initial count of admin data values #{AdminDataValue.all.count}"
-    AdminDataValue.delete_all
-    puts 'Deleted all admin data values'
-
-    Dir.glob(Rails.root.join('data', 'admin_data', '*.csv')).each do |filepath|
-      puts "=====================> Loading data from csv at path: #{filepath}"
-      AdminDataLoader.load_data filepath:
-    end
-    puts "=====================> Completed loading #{AdminDataValue.count} survey responses"
-  end
 end

spec/fixtures/test_2020-21_student_survey_responses.csv

@@ -1,8 +1,8 @@
-Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,LASID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district,school,DESE ID,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,#N/A,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,#N/A,#N/A,s-cure-q3,s-cure-q4,#N/A,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,#N/A,#N/A,s-grmi-q4,s-appa-q1,s-appa-q2,#N/A,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,#N/A,#N/A,s-vale-q4,#N/A,s-acst-q2,s-acst-q3,#N/A,#N/A,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,race,What is your race/ethnicity?(Please select all that apply) - Selected Choice
-2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,160505,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888
-2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,"2,3,4",888
-2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1600310,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,6,888
-2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1600310,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888
-2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1600310,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888
-2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1600310,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888
-2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1600310,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,,
+Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,LASID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district,school,DESE ID,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,#N/A,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,#N/A,#N/A,s-cure-q3,s-cure-q4,#N/A,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,#N/A,#N/A,s-grmi-q4,s-appa-q1,s-appa-q2,#N/A,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,#N/A,#N/A,s-vale-q4,#N/A,s-acst-q2,s-acst-q3,#N/A,#N/A,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,race,What is your race/ethnicity?(Please select all that apply) - Selected Choice,grade
+2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,160505,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888,11th
+2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,"2,3,4",888,10
+2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1600310,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1.00,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,6,888,8
+2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1600310,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888,8
+2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1600310,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888,7
+2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1600310,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888,3
+2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1600310,6,15,109,3710,7,1,,2.0,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,,,4

spec/models/score_spec.rb

@@ -0,0 +1,58 @@
+require 'rails_helper'
+
+RSpec.describe Score, type: :model do
+  let(:zones) do
+    Zones.new(watch_low_benchmark: 1.5, growth_low_benchmark: 2, approval_low_benchmark: 3,
+              ideal_low_benchmark: 4)
+  end
+
+  let(:warning) do
+    zones.warning_zone
+  end
+  let(:watch) do
+    zones.watch_zone
+  end
+  let(:growth) do
+    zones.growth_zone
+  end
+  let(:approval) do
+    zones.approval_zone
+  end
+  let(:ideal) do
+    zones.ideal_zone
+  end
+
+  context '.in_zone?' do
+    it('returns true if the score is in the warning zone') do
+      score = Score.new(average: 1)
+      expect(score.in_zone?(zone: warning)).to eq true
+      expect(score.in_zone?(zone: watch)).to eq false
+      expect(score.in_zone?(zone: growth)).to eq false
+      expect(score.in_zone?(zone: approval)).to eq false
+      expect(score.in_zone?(zone: ideal)).to eq false
+    end
+
+    it('returns true if the score is in the watch zone') do
+      score = Score.new(average: 1.5)
+      expect(score.in_zone?(zone: warning)).to eq true
+      expect(score.in_zone?(zone: watch)).to eq true
+      expect(score.in_zone?(zone: growth)).to eq false
+      expect(score.in_zone?(zone: approval)).to eq false
+      expect(score.in_zone?(zone: ideal)).to eq false
+    end
+  end
+  context '.blank?' do
+    it 'returns true if the average is nil zero or not a number' do
+      expect(Score.new(average: 0).blank?).to eq true
+      expect(Score.new(average: nil).blank?).to eq true
+      nan = Float::NAN
+      expect(Score.new(average: nan).blank?).to eq true
+    end
+
+    it 'returns false if the average is a non-zero float' do
+      expect(Score.new(average: 1).blank?).to eq false
+      expect(Score.new(average: 0.1).blank?).to eq false
+      expect(Score.new(average: -0.1).blank?).to eq false
+    end
+  end
+end

spec/presenters/analyze/graph/students_by_race_spec.rb

@@ -1,7 +1,7 @@
 require 'rails_helper'
 include Analyze::Graph
 include Analyze::Graph::Column
-describe StudentsByGroup do
+describe StudentsByRace do
   let(:american_indian) { create(:race, qualtrics_code: 1) }
   let(:asian) { create(:race, qualtrics_code: 2) }
   let(:black) { create(:race, qualtrics_code: 3) }
@@ -12,11 +12,11 @@ describe StudentsByGroup do
   context 'when initialized with a list of races' do
     it 'generates corresponding race columns' do
       races = [american_indian]
-      expect(StudentsByGroup.new(races:).columns).to eq [AmericanIndian, AllStudent]
+      expect(StudentsByRace.new(races:).columns).to eq [AmericanIndian, AllStudent]
       races = [american_indian, asian]
-      expect(StudentsByGroup.new(races:).columns).to eq [AmericanIndian, Asian, AllStudent]
+      expect(StudentsByRace.new(races:).columns).to eq [AmericanIndian, Asian, AllStudent]
       races = [black, hispanic, multiracial]
-      expect(StudentsByGroup.new(races:).columns).to eq [Black, Hispanic, Multiracial, AllStudent]
+      expect(StudentsByRace.new(races:).columns).to eq [Black, Hispanic, Multiracial, AllStudent]
     end
   end
 end

spec/presenters/variance_chart_row_presenter_spec.rb

@@ -40,7 +40,7 @@ describe VarianceChartRowPresenter do
   end
 
   context 'when the score is in the Ideal zone' do
-    let(:score) { Score.new(4.4, true, true) }
+    let(:score) { Score.new(average: 4.4, meets_teacher_threshold: true, meets_student_threshold: true) }
 
     it_behaves_like 'measure_name'
 
@@ -58,7 +58,7 @@ describe VarianceChartRowPresenter do
   end
 
   context 'when the score is in the Approval zone' do
-    let(:score) { Score.new(3.7, true, true) }
+    let(:score) { Score.new(average: 3.7, meets_teacher_threshold: true, meets_student_threshold: true) }
 
     it_behaves_like 'measure_name'
 
@@ -76,7 +76,7 @@ describe VarianceChartRowPresenter do
   end
 
   context 'when the score is in the Growth zone' do
-    let(:score) { Score.new(3.2, true, true) }
+    let(:score) { Score.new(average: 3.2, meets_teacher_threshold: true, meets_student_threshold: true) }
 
     it_behaves_like 'measure_name'
 
@@ -96,7 +96,7 @@ describe VarianceChartRowPresenter do
   end
 
   context 'when the score is in the Watch zone' do
-    let(:score) { Score.new(2.9, true, true) }
+    let(:score) { Score.new(average: 2.9, meets_teacher_threshold: true, meets_student_threshold: true) }
 
     it_behaves_like 'measure_name'
 
@@ -116,7 +116,7 @@ describe VarianceChartRowPresenter do
   end
 
   context 'when the score is in the Warning zone' do
-    let(:score) { Score.new(1.0, true, true) }
+    let(:score) { Score.new(average: 1.0, meets_teacher_threshold: true, meets_student_threshold: true) }
 
     it_behaves_like 'measure_name'
 
@@ -136,7 +136,7 @@ describe VarianceChartRowPresenter do
   end
 
   context 'when a measure does not contain admin data items' do
-    let(:score) { Score.new(nil, false, false) }
+    let(:score) { Score.new(average: nil, meets_teacher_threshold: false,  meets_student_threshold: false) }
 
     it 'it does not show a partial data indicator' do
       presenter_without_admin_data = VarianceChartRowPresenter.new measure: measure_without_admin_data_items,
@@ -149,7 +149,7 @@ describe VarianceChartRowPresenter do
     before :each do
     end
 
-    let(:score) { Score.new(nil, false, false) }
+    let(:score) { Score.new(average: nil, meets_teacher_threshold:  false, meets_student_threshold: false) }
 
     it 'shows a partial data indicator' do
       measure_with_admin_data = create(
@@ -165,7 +165,7 @@ describe VarianceChartRowPresenter do
              ideal_low_benchmark: ideal_low_benchmark
       admin_data_presenter = VarianceChartRowPresenter.new measure: measure_with_admin_data,
                                                            score: Score.new(
-                                                             3.7, true, true
+                                                             average: 3.7, meets_teacher_threshold: true, meets_student_threshold: true
                                                            )
       expect(admin_data_presenter.show_partial_data_indicator?).to be true
       expect(admin_data_presenter.partial_data_sources).to eq ['administrative data']
@@ -179,7 +179,7 @@ describe VarianceChartRowPresenter do
     end
 
     context 'when there are insufficient teacher survey item responses' do
-      let(:score) { Score.new(nil, false, true) }
+      let(:score) { Score.new(average: nil, meets_teacher_threshold: false, meets_student_threshold: true) }
       it 'shows a partial data indicator' do
         expect(presenter.show_partial_data_indicator?).to be true
         expect(presenter.partial_data_sources).to eq ['teacher survey results']
@@ -187,7 +187,7 @@ describe VarianceChartRowPresenter do
     end
 
     context 'when there are sufficient teacher survey item responses' do
-      let(:score) { Score.new(nil, true, true) }
+      let(:score) { Score.new(average: nil, meets_teacher_threshold:  true, meets_student_threshold: true) }
       it 'does not show a partial data indicator' do
         expect(presenter.show_partial_data_indicator?).to be false
       end
@@ -201,7 +201,7 @@ describe VarianceChartRowPresenter do
     end
 
     context 'when there are insufficient student survey item responses' do
-      let(:score) { Score.new(nil, true, false) }
+      let(:score) { Score.new(average: nil, meets_teacher_threshold: true, meets_student_threshold: false) }
       it 'shows a partial data indicator' do
         expect(presenter.show_partial_data_indicator?).to be true
         expect(presenter.partial_data_sources).to eq ['student survey results']
@@ -220,7 +220,7 @@ describe VarianceChartRowPresenter do
     end
 
     context 'When there are sufficient student survey item responses' do
-      let(:score) { Score.new(nil, true, true) }
+      let(:score) { Score.new(average: nil, meets_teacher_threshold: true, meets_student_threshold: true) }
       it 'does not show a partial data indicator' do
         expect(presenter.show_partial_data_indicator?).to be false
       end
@@ -236,15 +236,15 @@ describe VarianceChartRowPresenter do
              growth_low_benchmark:,
              approval_low_benchmark:,
              ideal_low_benchmark:)
-      approval_presenter = VarianceChartRowPresenter.new measure: measure, score: Score.new(3.7, true, true)
-      ideal_presenter = VarianceChartRowPresenter.new measure: measure, score: Score.new(4.4, true, true)
+      approval_presenter = VarianceChartRowPresenter.new measure: measure, score: Score.new(average: 3.7, meets_teacher_threshold: true,meets_student_threshold: true)
+      ideal_presenter = VarianceChartRowPresenter.new measure: measure, score: Score.new(average: 4.4, meets_teacher_threshold: true, meets_student_threshold: true)
       expect(ideal_presenter <=> approval_presenter).to be < 0
       expect([approval_presenter, ideal_presenter].sort).to eq [ideal_presenter, approval_presenter]
     end
 
     it 'selects a warning bar below a ideal bar' do
-      warning_presenter = VarianceChartRowPresenter.new measure: measure, score: Score.new(1.0, true, true)
-      ideal_presenter = VarianceChartRowPresenter.new measure: measure, score: Score.new(5.0, true, true)
+      warning_presenter = VarianceChartRowPresenter.new measure: measure, score: Score.new(average: 1.0, meets_teacher_threshold: true, meets_student_threshold: true)
+      ideal_presenter = VarianceChartRowPresenter.new measure: measure, score: Score.new(average: 5.0, meets_teacher_threshold: true, meets_student_threshold: true)
       expect(warning_presenter <=> ideal_presenter).to be > 0
       expect([warning_presenter, ideal_presenter].sort).to eq [ideal_presenter, warning_presenter]
     end

spec/services/survey_responses_data_loader_spec.rb

@@ -29,15 +29,10 @@ describe SurveyResponsesDataLoader do
       end
       it 'ensures teacher responses load correctly' do
         assigns_academic_year_to_survey_item_responses
-
         assigns_school_to_the_survey_item_responses
-
         loads_survey_item_responses_for_a_given_survey_response
-
         loads_all_survey_item_responses_for_a_given_survey_item
-
         captures_likert_scores_for_survey_item_responses
-
         is_idempotent
       end
     end
@@ -53,16 +48,23 @@ describe SurveyResponsesDataLoader do
         loads_student_survey_item_response_values
         student_survey_item_response_count_matches_expected
         captures_likert_scores_for_student_survey_item_responses
+        assigns_grade_level_to_responses
         is_idempotent_for_students
       end
 
       context 'when updating student survey responses from another csv file' do
-        it 'updates the likert score to the score on the new csv file' do
+        before do
           SurveyResponsesDataLoader.load_data filepath: Rails.root.join('spec', 'fixtures',
                                                                         'secondary_test_2020-21_student_survey_responses.csv')
-          expect(SurveyItemResponse.joins(:survey_item).where(response_id: 'student_survey_response_3').where("survey_item.survey_item_id": 's-emsa-q1').first.likert_score).to eq 1
-          expect(SurveyItemResponse.joins(:survey_item).where(response_id: 'student_survey_response_4').where("survey_item.survey_item_id": 's-emsa-q1').first.likert_score).to eq 1
-          expect(SurveyItemResponse.joins(:survey_item).where(response_id: 'student_survey_response_5').where("survey_item.survey_item_id": 's-emsa-q1').first.likert_score).to eq 1
+        end
+        it 'updates the likert score to the score on the new csv file' do
+          s_emsa_q1 = SurveyItem.find_by_survey_item_id 's-emsa-q1'
+          expect(SurveyItemResponse.where(response_id: 'student_survey_response_3',
+                                          survey_item: s_emsa_q1).first.likert_score).to eq 1
+          expect(SurveyItemResponse.where(response_id: 'student_survey_response_4',
+                                          survey_item: s_emsa_q1).first.likert_score).to eq 1
+          expect(SurveyItemResponse.where(response_id: 'student_survey_response_5',
+                                          survey_item: s_emsa_q1).first.likert_score).to eq 1
         end
       end
     end
@@ -160,3 +162,17 @@ def is_idempotent_for_students
 
   expect(SurveyItemResponse.count).to eq number_of_survey_item_responses
 end
+
+def assigns_grade_level_to_responses
+  results = { 'student_survey_response_1' => 11,
+              'student_survey_response_3' => 8,
+              'student_survey_response_4' => 8,
+              'student_survey_response_5' => 7,
+              'student_survey_response_6' => 3,
+              'student_survey_response_7' => 4 }
+  results.each do |key, value|
+    expect(SurveyItemResponse.where(response_id: key).all? do |response|
+             response.grade == value
+           end).to eq true
+  end
+end

spec/views/analyze/index.html.erb_spec.rb

@@ -13,7 +13,7 @@ describe 'analyze/index' do
   end
   let(:graph) { StudentsAndTeachers.new }
   let(:graphs) do
-    [StudentsAndTeachers.new, StudentsByGroup.new(races:)]
+    [StudentsAndTeachers.new, StudentsByRace.new(races:)]
   end
   let(:background) { BackgroundPresenter.new(num_of_columns: graph.columns.count) }
   let(:selected_races) { races }
@@ -54,6 +54,36 @@ describe 'analyze/index' do
     measure
   end
 
+  let(:sources) do
+    [Analyze::Source::SurveyData.new(slices:)]
+  end
+
+  let(:slices) do
+    students_and_teachers = Analyze::Slice::StudentsAndTeachers.new
+    students_by_group = Analyze::Slice::StudentsByGroup.new(races:, grades:)
+    [students_and_teachers, students_by_group]
+  end
+
+  let(:slice) do
+    slices.first
+  end
+
+  let(:groups) do
+    [Analyze::Group::Race.new, Analyze::Group::Grade.new]
+  end
+
+  let(:group) do
+    groups.first
+  end
+
+  let(:grades) do
+    (1..12).to_a
+  end
+
+  let(:selected_grades) do
+    grades
+  end
+
   before :each do
     assign :races, races
     assign :selected_races, selected_races
@@ -70,6 +100,12 @@ describe 'analyze/index' do
     assign :subcategory, subcategory
     assign :subcategories, category.subcategories
     assign :measures, [support_for_teaching, effective_leadership, professional_qualifications]
+    assign :sources, sources
+    assign :groups, groups
+    assign :group, group
+    assign :slice, slice
+    assign :grades, grades
+    assign :selected_grades, selected_grades
     create(:respondent, school:, academic_year:)
     create(:survey, school:, academic_year:)
   end
@@ -88,7 +124,7 @@ describe 'analyze/index' do
     #          ideal_low_benchmark: 4.5)
     #   [
     #     GroupedBarColumnPresenter.new(measure:,
-    #                                   score: Score.new(rand))
+    #                                   score: Score.new(average: rand))
     #   ]
     # end
 

spec/views/overview/index.html.erb_spec.rb

@@ -58,7 +58,7 @@ describe 'overview/index' do
     let(:variance_chart_row_presenters) do
       [
         VarianceChartRowPresenter.new(measure: support_for_teaching, score: Score.new),
-        VarianceChartRowPresenter.new(measure: effective_leadership, score: Score.new(rand)),
+        VarianceChartRowPresenter.new(measure: effective_leadership, score: Score.new(average: rand)),
         VarianceChartRowPresenter.new(measure: professional_qualifications, score: Score.new)
       ]
     end
@@ -90,7 +90,7 @@ describe 'overview/index' do
              ideal_low_benchmark: 4.5)
       [
         VarianceChartRowPresenter.new(measure:,
-                                      score: Score.new(rand))
+                                      score: Score.new(average: rand))
       ]
     end
 

spec/views/overview/variance_chart.html.erb_spec.rb

@@ -27,8 +27,8 @@ describe 'overview/_variance_chart.html.erb' do
 
     before :each do
       presenters = [
-        VarianceChartRowPresenter.new(measure: lower_scoring_measure, score: Score.new(1)),
-        VarianceChartRowPresenter.new(measure: higher_scoring_measure, score: Score.new(5))
+        VarianceChartRowPresenter.new(measure: lower_scoring_measure, score: Score.new(average: 1)),
+        VarianceChartRowPresenter.new(measure: higher_scoring_measure, score: Score.new(average: 5))
       ]
 
       render partial: 'variance_chart', locals: { presenters: }
@@ -53,8 +53,8 @@ describe 'overview/_variance_chart.html.erb' do
       measure_lacking_score = create(:measure)
       another_measure_lacking_score = create(:measure)
       presenters = [
-        VarianceChartRowPresenter.new(measure: measure_lacking_score, score: Score.new(nil)),
-        VarianceChartRowPresenter.new(measure: another_measure_lacking_score, score: Score.new(nil))
+        VarianceChartRowPresenter.new(measure: measure_lacking_score, score: Score.new(average: nil)),
+        VarianceChartRowPresenter.new(measure: another_measure_lacking_score, score: Score.new(average: nil))
       ]
 
       render partial: 'variance_chart', locals: { presenters: }