From 76ebcc6ef3d049a3b9c2ec3e0a7284431f48c98f Mon Sep 17 00:00:00 2001 From: rebuilt Date: Tue, 20 Jun 2023 14:50:33 -0700 Subject: [PATCH] feat: Add income table to the database. Add seeder for income. Add a reference to income from survey item response. Update the loader to import income data from the survey response csv. Refactor analyze controller to extract presenter. Add corresponding specs. Add income graph to analyze page --- app/controllers/analyze_controller.rb | 170 +----- app/helpers/analyze_helper.rb | 4 +- .../controllers/analyze_controller.js | 59 +-- app/models/income.rb | 7 + app/models/survey_item_response.rb | 6 + app/presenters/analyze/bar_presenter.rb | 85 +++ .../column/grouped_bar_column_presenter.rb | 8 +- .../column/income_column/disadvantaged.rb | 1 + .../column/income_column/not_disadvantaged.rb | 1 + .../column/income_column/score_for_income.rb | 43 ++ .../graph/column/income_column/unknown.rb | 1 + .../analyze/graph/column/score_for_race.rb | 64 ++- .../analyze/graph/students_by_income.rb | 40 ++ app/presenters/analyze/group/income.rb | 13 + app/presenters/analyze/presenter.rb | 153 ++++++ app/presenters/analyze/ui.rb | 8 - app/presenters/analyze_bar_presenter.rb | 83 --- app/services/cleaner.rb | 115 ++-- app/services/demographic_loader.rb | 8 + app/services/disaggregation_loader.rb | 30 ++ app/services/disaggregation_row.rb | 35 ++ app/services/survey_item_values.rb | 55 +- app/services/survey_responses_data_loader.rb | 21 +- app/views/analyze/_checkboxes.html.erb | 17 + app/views/analyze/_data_filters.html.erb | 14 +- app/views/analyze/_focus_area.html.erb | 4 +- app/views/analyze/_group_selectors.html.erb | 66 +-- app/views/analyze/_grouped_bar_chart.html.erb | 6 +- app/views/analyze/_school_years.html.erb | 6 +- app/views/analyze/index.html.erb | 12 +- data/demographics.csv | 22 +- db/migrate/20230620221633_create_incomes.rb | 9 + ...702_add_income_to_survey_item_responses.rb | 5 + .../20230630215110_add_slug_to_income.rb | 6 + db/schema.rb | 32 +- lib/tasks/one_off.rake | 22 +- package.json | 2 +- spec/factories.rb | 4 + .../sample_maynard_disaggregation_data.csv | 501 ++++++++++++++++++ .../raw/sample_maynard_raw_student_survey.csv | 36 ++ spec/fixtures/sample_demographics.csv | 22 +- .../test_2020-21_student_survey_responses.csv | 18 +- spec/models/income_spec.rb | 5 + spec/presenters/analyze/presenter_spec.rb | 468 ++++++++++++++++ .../grouped_bar_column_presenter_spec.rb | 12 +- spec/services/cleaner_spec.rb | 267 ++++++++-- spec/services/demographic_loader_spec.rb | 11 + spec/services/disaggregation_loader_spec.rb | 35 ++ spec/services/race_score_loader_spec.rb | 77 --- spec/services/survey_item_values_spec.rb | 306 ++++++----- .../survey_responses_data_loader_spec.rb | 287 +++++----- spec/views/analyze/index.html.erb_spec.rb | 159 +++--- yarn.lock | 9 +- 53 files changed, 2412 insertions(+), 1038 deletions(-) create mode 100644 app/models/income.rb create mode 100644 app/presenters/analyze/bar_presenter.rb create mode 100644 app/presenters/analyze/graph/column/income_column/score_for_income.rb create mode 100644 app/presenters/analyze/graph/students_by_income.rb create mode 100644 app/presenters/analyze/group/income.rb create mode 100644 app/presenters/analyze/presenter.rb delete mode 100644 app/presenters/analyze/ui.rb delete mode 100644 app/presenters/analyze_bar_presenter.rb create mode 100644 app/services/disaggregation_loader.rb create mode 100644 app/services/disaggregation_row.rb create mode 100644 app/views/analyze/_checkboxes.html.erb create mode 100644 db/migrate/20230620221633_create_incomes.rb create mode 100644 db/migrate/20230620223702_add_income_to_survey_item_responses.rb create mode 100644 db/migrate/20230630215110_add_slug_to_income.rb create mode 100644 spec/fixtures/disaggregation/sample_maynard_disaggregation_data.csv create mode 100644 spec/fixtures/raw/sample_maynard_raw_student_survey.csv create mode 100644 spec/models/income_spec.rb create mode 100644 spec/presenters/analyze/presenter_spec.rb create mode 100644 spec/services/disaggregation_loader_spec.rb delete mode 100644 spec/services/race_score_loader_spec.rb diff --git a/app/controllers/analyze_controller.rb b/app/controllers/analyze_controller.rb index a0f1469b..f46fa907 100644 --- a/app/controllers/analyze_controller.rb +++ b/app/controllers/analyze_controller.rb @@ -1,172 +1,8 @@ # frozen_string_literal: true class AnalyzeController < SqmApplicationController - before_action :assign_categories, :assign_subcategories, :assign_measures, :assign_academic_years, - :races, :selected_races, :graph, :graphs, :background, :race_score_timestamp, - :source, :sources, :group, :groups, :selected_grades, :grades, :slice, :selected_genders, :genders, only: [:index] - def index; end - - private - - def assign_categories - @category ||= Category.find_by_category_id(params[:category]) - @category ||= Category.order(:category_id).first - @categories = Category.all.order(:category_id) - end - - def assign_subcategories - @subcategories = @category.subcategories.order(:subcategory_id) - @subcategory ||= Subcategory.find_by_subcategory_id(params[:subcategory]) - @subcategory ||= @subcategories.first - end - - def assign_measures - @measures = @subcategory.measures.order(:measure_id).includes(%i[admin_data_items subcategory]) - end - - def assign_academic_years - @available_academic_years = AcademicYear.order(:range).all - year_params = params[:academic_years] - @academic_year_params = year_params.split(',') if year_params - @selected_academic_years = [] - @academic_year_params ||= [] - @academic_year_params.each do |year| - @selected_academic_years << AcademicYear.find_by_range(year) - end - end - - def races - @races ||= Race.all.order(designation: :ASC) - end - - def selected_races - @selected_races ||= begin - race_params = params[:races] - return @selected_races = races unless race_params - - race_list = race_params.split(',') if race_params - if race_list - race_list = race_list.map do |race| - Race.find_by_slug race - end - end - race_list - end - end - - def graph - graphs.each do |graph| - @graph = graph if graph.slug == params[:graph] - end - - @graph ||= graphs.first - end - - def graphs - @graphs ||= [Analyze::Graph::AllData.new, Analyze::Graph::StudentsAndTeachers.new, Analyze::Graph::StudentsByRace.new(races: selected_races), - Analyze::Graph::StudentsByGrade.new(grades: selected_grades), Analyze::Graph::StudentsByGender.new(genders: selected_genders)] - end - - def background - @background ||= BackgroundPresenter.new(num_of_columns: graph.columns.count) - end - - def race_score_timestamp - @race_score_timestamp ||= begin - score = RaceScore.where(school: @school, - academic_year: @academic_year).order(updated_at: :DESC).first || Today.new - score.updated_at - end - end - - def source - source_param = params[:source] - sources.each do |source| - @source = source if source.slug == source_param - end - - @source ||= sources.first - end - - def sources - all_data_slices = [Analyze::Slice::AllData.new] - all_data_source = Analyze::Source::AllData.new(slices: all_data_slices) - - students_and_teachers = Analyze::Slice::StudentsAndTeachers.new - students_by_group = Analyze::Slice::StudentsByGroup.new(races:, grades:) - survey_data_slices = [students_and_teachers, students_by_group] - survey_data_source = Analyze::Source::SurveyData.new(slices: survey_data_slices) - - @sources = [all_data_source, survey_data_source] - end - - def slice - slice_param = params[:slice] - slices.each do |slice| - @slice = slice if slice.slug == slice_param - end - - @slice ||= slices.first - end - - def slices - source.slices - end - - def group - groups.each do |group| - @group = group if group.slug == params[:group] - end - - @group ||= groups.first - end - - def groups - @groups = [Analyze::Group::Race.new, Analyze::Group::Grade.new, Analyze::Group::Gender.new] - end - - def selected_grades - @selected_grades ||= begin - grade_params = params[:grades] - return @selected_grades = grades unless grade_params - - grade_list = grade_params.split(',') if grade_params - if grade_list - grade_list = grade_list.map do |grade| - grade.to_i - end - end - grade_list - end - end - - def grades - @grades ||= SurveyItemResponse.where(school: @school, academic_year: @academic_year) - .where.not(grade: nil) - .group(:grade) - .select(:response_id) - .distinct(:response_id) - .count.reject do |_key, value| - value < 10 - end.keys - end - - def selected_genders - @selected_genders ||= begin - gender_params = params[:genders] - return @selected_genders = genders unless gender_params - - gender_list = gender_params.split(',') if gender_params - if gender_list - gender_list = gender_list.map do |gender| - Gender.find_by_designation(gender) - end - end - gender_list - end - end - - def genders - @genders ||= Gender.all + def index + @presenter = Analyze::Presenter.new(params:, school: @school, academic_year: @academic_year) + @background ||= BackgroundPresenter.new(num_of_columns: @presenter.graph.columns.count) end end diff --git a/app/helpers/analyze_helper.rb b/app/helpers/analyze_helper.rb index 7ac34581..e253167c 100644 --- a/app/helpers/analyze_helper.rb +++ b/app/helpers/analyze_helper.rb @@ -62,7 +62,7 @@ module AnalyzeHelper end def base_url - analyze_subcategory_link(district: @district, school: @school, academic_year: @academic_year, category: @category, - subcategory: @subcategory) + analyze_subcategory_link(district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, + subcategory: @presenter.subcategory) end end diff --git a/app/javascript/controllers/analyze_controller.js b/app/javascript/controllers/analyze_controller.js index ed11416a..bf290c9b 100644 --- a/app/javascript/controllers/analyze_controller.js +++ b/app/javascript/controllers/analyze_controller.js @@ -22,16 +22,17 @@ export default class extends Controller { "&graph=" + this.selected_graph(target) + "&races=" + - this.selected_races().join(",") + + this.selected_items("race").join(",") + "&genders=" + - this.selected_genders().join(",") + + this.selected_items("gender").join(",") + + "&incomes=" + + this.selected_items("income").join(",") + "&grades=" + - this.selected_grades().join(","); + this.selected_items("grade").join(","); this.go_to(url); } - go_to(location) { window.location = location; } @@ -121,58 +122,34 @@ export default class extends Controller { return item.id; })[0]; + const groups = new Map([ + ['gender', 'students-by-gender'], + ['grade', 'students-by-grade'], + ['income', 'students-by-income'], + ['race', 'students-by-race'] + ]) + if (target.name === 'slice' || target.name === 'group') { if (selected_slice === 'students-and-teachers') { return 'students-and-teachers'; - } else if (this.selected_group() === 'race') { - return 'students-by-race'; - } else if (this.selected_group() === 'gender') { - return 'students-by-gender'; - } else if (this.selected_group() === 'grade') { - return 'students-by-grade'; } + return groups.get(this.selected_group()); } return window.graph; } - selected_races() { - let race_checkboxes = [...document.getElementsByName("race-checkbox")] - let races = race_checkboxes - .filter((item) => { - return item.checked; - }) - .map((item) => { - return item.id; - }); - - return races; - } - - selected_grades() { - let grade_checkboxes = [...document.getElementsByName("grade-checkbox")] - let grades = grade_checkboxes - .filter((item) => { - return item.checked; - }) - .map((item) => { - return item.id.replace('grade-', ''); - }); - - return grades; - } - - selected_genders() { - let gender_checkboxes = [...document.getElementsByName("gender-checkbox")] - let genders = gender_checkboxes + selected_items(type) { + let checkboxes = [...document.getElementsByName(`${type}-checkbox`)] + let items = checkboxes .filter((item) => { return item.checked; }) .map((item) => { - return item.id.replace('gender-', ''); + return item.id.replace(`${type}-`, ''); }); - return genders; + return items; } } diff --git a/app/models/income.rb b/app/models/income.rb new file mode 100644 index 00000000..447f8465 --- /dev/null +++ b/app/models/income.rb @@ -0,0 +1,7 @@ +class Income < ApplicationRecord + scope :by_designation, -> { all.map { |income| [income.designation, income] }.to_h } + + include FriendlyId + + friendly_id :designation, use: [:slugged] +end diff --git a/app/models/survey_item_response.rb b/app/models/survey_item_response.rb index cf3986bc..50810867 100644 --- a/app/models/survey_item_response.rb +++ b/app/models/survey_item_response.rb @@ -9,6 +9,7 @@ class SurveyItemResponse < ActiveRecord::Base belongs_to :survey_item, counter_cache: true belongs_to :student, foreign_key: :student_id, optional: true belongs_to :gender + belongs_to :income has_one :measure, through: :survey_item @@ -31,4 +32,9 @@ class SurveyItemResponse < ActiveRecord::Base SurveyItemResponse.where(survey_item: survey_items, school:, academic_year:, income:, grade: school.grades(academic_year:)).group(:survey_item).average(:likert_score) } + + scope :averages_for_income, lambda { |survey_items, school, academic_year, income| + SurveyItemResponse.where(survey_item: survey_items, school:, + academic_year:, income:).group(:survey_item).average(:likert_score) + } end diff --git a/app/presenters/analyze/bar_presenter.rb b/app/presenters/analyze/bar_presenter.rb new file mode 100644 index 00000000..7dc9bdc1 --- /dev/null +++ b/app/presenters/analyze/bar_presenter.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Analyze + class BarPresenter + include AnalyzeHelper + attr_reader :score, :x_position, :academic_year, :measure_id, :measure, :color + + MINIMUM_BAR_HEIGHT = 2 + + def initialize(measure:, academic_year:, score:, x_position:, color:) + @score = score + @x_position = x_position + @academic_year = academic_year + @measure = measure + @measure_id = measure.measure_id + @color = color + end + + def y_offset + benchmark_height = analyze_zone_height * 2 + case zone.type + when :ideal, :approval + benchmark_height - bar_height_percentage + else + benchmark_height + end + end + + def bar_color + "fill-#{zone.type}" + end + + def bar_height_percentage + bar_height = send("#{zone.type}_bar_height_percentage") || 0 + enforce_minimum_height(bar_height:) + end + + def percentage + low_benchmark = zone.low_benchmark + (score.average - low_benchmark) / (zone.high_benchmark - low_benchmark) + end + + def zone + zones = Zones.new( + watch_low_benchmark: measure.watch_low_benchmark, + growth_low_benchmark: measure.growth_low_benchmark, + approval_low_benchmark: measure.approval_low_benchmark, + ideal_low_benchmark: measure.ideal_low_benchmark + ) + zones.zone_for_score(score.average) + end + + def average + average = score.average || 0 + + average.round(6) + end + + private + + def enforce_minimum_height(bar_height:) + bar_height < MINIMUM_BAR_HEIGHT ? MINIMUM_BAR_HEIGHT : bar_height + end + + def ideal_bar_height_percentage + (percentage * zone_height_percentage + zone_height_percentage) * 100 + end + + def approval_bar_height_percentage + (percentage * zone_height_percentage) * 100 + end + + def growth_bar_height_percentage + ((1 - percentage) * zone_height_percentage) * 100 + end + + def watch_bar_height_percentage + ((1 - percentage) * zone_height_percentage + zone_height_percentage) * 100 + end + + def warning_bar_height_percentage + ((1 - percentage) * zone_height_percentage + zone_height_percentage + zone_height_percentage) * 100 + end + end +end diff --git a/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb b/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb index 9e826c88..e796af97 100644 --- a/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb +++ b/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb @@ -27,10 +27,10 @@ module Analyze def bars @bars ||= yearly_scores.map.each_with_index do |yearly_score, index| year = yearly_score.year - AnalyzeBarPresenter.new(measure:, academic_year: year, - score: yearly_score.score, - x_position: bar_x(index), - color: bar_color(year)) + Analyze::BarPresenter.new(measure:, academic_year: year, + score: yearly_score.score, + x_position: bar_x(index), + color: bar_color(year)) end end diff --git a/app/presenters/analyze/graph/column/income_column/disadvantaged.rb b/app/presenters/analyze/graph/column/income_column/disadvantaged.rb index c6569611..ad304ce9 100644 --- a/app/presenters/analyze/graph/column/income_column/disadvantaged.rb +++ b/app/presenters/analyze/graph/column/income_column/disadvantaged.rb @@ -27,3 +27,4 @@ module Analyze end end end + diff --git a/app/presenters/analyze/graph/column/income_column/not_disadvantaged.rb b/app/presenters/analyze/graph/column/income_column/not_disadvantaged.rb index e48c653f..4a1d4c21 100644 --- a/app/presenters/analyze/graph/column/income_column/not_disadvantaged.rb +++ b/app/presenters/analyze/graph/column/income_column/not_disadvantaged.rb @@ -27,3 +27,4 @@ module Analyze end end end + diff --git a/app/presenters/analyze/graph/column/income_column/score_for_income.rb b/app/presenters/analyze/graph/column/income_column/score_for_income.rb new file mode 100644 index 00000000..3e1140c3 --- /dev/null +++ b/app/presenters/analyze/graph/column/income_column/score_for_income.rb @@ -0,0 +1,43 @@ +module Analyze + module Graph + module Column + module IncomeColumn + module ScoreForIncome + def score(year_index) + academic_year = academic_years[year_index] + averages = SurveyItemResponse.averages_for_income(measure.student_survey_items, school, academic_year, + income) + average = bubble_up_averages(averages:).round(2) + + scorify(average:, meets_student_threshold: sufficient_student_responses?(academic_year:)) + end + + def bubble_up_averages(averages:) + measure.student_scales.map do |scale| + scale.survey_items.map do |survey_item| + averages[survey_item] + end.remove_blanks.average + end.remove_blanks.average + end + + def scorify(average:, meets_student_threshold:) + return Score::NIL_SCORE unless meets_student_threshold + + Score.new(average:, + meets_teacher_threshold: false, + meets_student_threshold: true, + meets_admin_data_threshold: false) + end + + def sufficient_student_responses?(academic_year:) + yearly_counts = SurveyItemResponse.where(school:, academic_year:, + income:, survey_item: measure.student_survey_items).group(:income).select(:response_id).distinct(:response_id).count + yearly_counts.any? do |count| + count[1] >= 10 + end + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/column/income_column/unknown.rb b/app/presenters/analyze/graph/column/income_column/unknown.rb index eee7c8be..e6e0ba8b 100644 --- a/app/presenters/analyze/graph/column/income_column/unknown.rb +++ b/app/presenters/analyze/graph/column/income_column/unknown.rb @@ -27,3 +27,4 @@ module Analyze end end end + diff --git a/app/presenters/analyze/graph/column/score_for_race.rb b/app/presenters/analyze/graph/column/score_for_race.rb index 207e198f..36bee8a5 100644 --- a/app/presenters/analyze/graph/column/score_for_race.rb +++ b/app/presenters/analyze/graph/column/score_for_race.rb @@ -3,16 +3,64 @@ module Analyze module Column module ScoreForRace def score(year_index) - s = ::RaceScore.find_by(measure:, school:, academic_year: academic_years[year_index], race:) - average = s.average.round(2) unless s.nil? - average ||= 0 - meets_student_threshold = s.meets_student_threshold? unless s.nil? - meets_student_threshold ||= false - Score.new(average:, - meets_teacher_threshold: false, - meets_student_threshold:, + academic_year = academic_year_for_year_index(year_index) + rate = response_rate(school:, academic_year:, measure:) + return Score::NIL_SCORE unless rate.meets_student_threshold + + survey_items = measure.student_survey_items + + averages = grouped_responses(school:, academic_year:, survey_items:, race:) + meets_student_threshold = sufficient_responses(school:, academic_year:, race:) + scorify(responses: averages, meets_student_threshold:, measure:) + end + + def grouped_responses(school:, academic_year:, survey_items:, race:) + @grouped_responses ||= Hash.new do |memo, (school, academic_year, survey_items, race)| + memo[[school, academic_year, survey_items, race]] = + SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where( + school:, academic_year:, grade: school.grades(academic_year:) + ).where("student_races.race_id": race.id).group(:survey_item_id).having("count(*) >= 10").average(:likert_score) + end + + @grouped_responses[[school, academic_year, survey_items, race]] + end + + def response_rate(school:, academic_year:, measure:) + subcategory = measure.subcategory + @response_rate ||= Hash.new do |memo, (school, academic_year, subcategory)| + memo[[school, academic_year, subcategory]] = subcategory.response_rate(school:, academic_year:) + end + + @response_rate[[school, academic_year, subcategory]] + end + + def scorify(responses:, meets_student_threshold:, measure:) + averages = bubble_up_averages(responses:, measure:) + average = averages.average.round(2) + + average = 0 unless meets_student_threshold + + Score.new(average:, meets_teacher_threshold: false, meets_student_threshold:, meets_admin_data_threshold: false) end + + def sufficient_responses(school:, academic_year:, race:) + @sufficient_responses ||= Hash.new do |memo, (school, academic_year, race)| + number_of_students_for_a_racial_group = SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where( + school:, academic_year: + ).where("student_races.race_id": race.id).distinct.pluck(:student_id).count + memo[[school, academic_year, race]] = number_of_students_for_a_racial_group >= 10 + end + @sufficient_responses[[school, academic_year, race]] + end + + def bubble_up_averages(responses:, measure:) + measure.student_scales.map do |scale| + scale.survey_items.map do |survey_item| + responses[survey_item.id] + end.remove_blanks.average + end.remove_blanks + end end end end diff --git a/app/presenters/analyze/graph/students_by_income.rb b/app/presenters/analyze/graph/students_by_income.rb new file mode 100644 index 00000000..148b22c1 --- /dev/null +++ b/app/presenters/analyze/graph/students_by_income.rb @@ -0,0 +1,40 @@ +module Analyze + module Graph + class StudentsByIncome + attr_reader :incomes + + def initialize(incomes:) + @incomes = incomes + end + + def to_s + "Students by income" + end + + def slug + "students-by-income" + end + + def columns + [].tap do |array| + incomes.each do |income| + array << column_for_income_code(code: income.slug) + end + array << Analyze::Graph::Column::AllStudent + end + end + + private + + def column_for_income_code(code:) + CFR[code.to_s] + end + + CFR = { + "economically-disadvantaged-y" => Analyze::Graph::Column::IncomeColumn::Disadvantaged, + "economically-disadvantaged-n" => Analyze::Graph::Column::IncomeColumn::NotDisadvantaged, + "unknown" => Analyze::Graph::Column::IncomeColumn::Unknown + }.freeze + end + end +end diff --git a/app/presenters/analyze/group/income.rb b/app/presenters/analyze/group/income.rb new file mode 100644 index 00000000..21e364b4 --- /dev/null +++ b/app/presenters/analyze/group/income.rb @@ -0,0 +1,13 @@ +module Analyze + module Group + class Income + def name + 'Income' + end + + def slug + 'income' + end + end + end +end diff --git a/app/presenters/analyze/presenter.rb b/app/presenters/analyze/presenter.rb new file mode 100644 index 00000000..f68f2997 --- /dev/null +++ b/app/presenters/analyze/presenter.rb @@ -0,0 +1,153 @@ +module Analyze + class Presenter + attr_reader :params, :school, :academic_year + + def initialize(params:, school:, academic_year:) + @params = params + @school = school + @academic_year = academic_year + end + + def category + @category ||= Category.find_by_category_id(params[:category]) || Category.order(:category_id).first + end + + def categories + @categories = Category.all.order(:category_id) + end + + def subcategory + @subcategory ||= Subcategory.find_by_subcategory_id(params[:subcategory]) || subcategories.first + end + + def subcategories + @subcategories = category.subcategories.order(:subcategory_id) + end + + def measures + @measures = subcategory.measures.order(:measure_id).includes(%i[admin_data_items subcategory]) + end + + def academic_years + @academic_years = AcademicYear.order(:range).all + end + + def selected_academic_years + @selected_academic_years ||= begin + year_params = params[:academic_years] + return [] unless year_params + + year_params.split(",").map { |year| AcademicYear.find_by_range(year) }.compact + end + end + + def races + @races ||= Race.all.order(designation: :ASC) + end + + def selected_races + @selected_races ||= begin + race_params = params[:races] + return races unless race_params + + race_params.split(",").map { |race| Race.find_by_slug race }.compact + end + end + + def graphs + @graphs ||= [Analyze::Graph::AllData.new, Analyze::Graph::StudentsAndTeachers.new, Analyze::Graph::StudentsByRace.new(races: selected_races), + Analyze::Graph::StudentsByGrade.new(grades: selected_grades), Analyze::Graph::StudentsByGender.new(genders: selected_genders), Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes)] + end + + def graph + @graph ||= graphs.reduce(graphs.first) do |acc, graph| + graph.slug == params[:graph] ? graph : acc + end + end + + def selected_grades + @selected_grades ||= begin + grade_params = params[:grades] + return grades unless grade_params + + grade_params.split(",").map(&:to_i) + end + end + + def selected_genders + @selected_genders ||= begin + gender_params = params[:genders] + return genders unless gender_params + + gender_params.split(",").map { |gender| Gender.find_by_designation(gender) }.compact + end + end + + def genders + @genders ||= Gender.all + end + + def groups + @groups = [Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new, + Analyze::Group::Race.new] + end + + def group + @group ||= groups.reduce(groups.first) do |acc, group| + group.slug == params[:group] ? group : acc + end + end + + def slice + @slice ||= slices.reduce(slices.first) do |acc, slice| + slice.slug == params[:slice] ? slice : acc + end + end + + def slices + source.slices + end + + def source + @source ||= sources.reduce(sources.first) do |acc, source| + source.slug == params[:source] ? source : acc + end + end + + def sources + all_data_slices = [Analyze::Slice::AllData.new] + all_data_source = Analyze::Source::AllData.new(slices: all_data_slices) + + students_and_teachers = Analyze::Slice::StudentsAndTeachers.new + students_by_group = Analyze::Slice::StudentsByGroup.new(races:, grades:) + survey_data_slices = [students_and_teachers, students_by_group] + survey_data_source = Analyze::Source::SurveyData.new(slices: survey_data_slices) + + @sources = [all_data_source, survey_data_source] + end + + def grades + @grades ||= SurveyItemResponse.where(school:, academic_year:) + .where.not(grade: nil) + .group(:grade) + .select(:response_id) + .distinct(:response_id) + .count.reject do |_key, value| + value < 10 + end.keys + end + + def incomes + @incomes ||= Income.all + end + + def selected_incomes + @selected_incomes ||= begin + income_params = params[:incomes] + return incomes unless income_params + + income_params.split(",").map { |income| Income.find_by_slug(income) }.compact + end + end + end +end diff --git a/app/presenters/analyze/ui.rb b/app/presenters/analyze/ui.rb deleted file mode 100644 index cf440140..00000000 --- a/app/presenters/analyze/ui.rb +++ /dev/null @@ -1,8 +0,0 @@ -module Analyze - class Ui - attr_reader :params - def initialize(params:) - @params = params - end - end -end diff --git a/app/presenters/analyze_bar_presenter.rb b/app/presenters/analyze_bar_presenter.rb deleted file mode 100644 index 32582411..00000000 --- a/app/presenters/analyze_bar_presenter.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -class AnalyzeBarPresenter - include AnalyzeHelper - attr_reader :score, :x_position, :academic_year, :measure_id, :measure, :color - - MINIMUM_BAR_HEIGHT = 2 - - def initialize(measure:, academic_year:, score:, x_position:, color:) - @score = score - @x_position = x_position - @academic_year = academic_year - @measure = measure - @measure_id = measure.measure_id - @color = color - end - - def y_offset - benchmark_height = analyze_zone_height * 2 - case zone.type - when :ideal, :approval - benchmark_height - bar_height_percentage - else - benchmark_height - end - end - - def bar_color - "fill-#{zone.type}" - end - - def bar_height_percentage - bar_height = send("#{zone.type}_bar_height_percentage") || 0 - enforce_minimum_height(bar_height:) - end - - def percentage - low_benchmark = zone.low_benchmark - (score.average - low_benchmark) / (zone.high_benchmark - low_benchmark) - end - - def zone - zones = Zones.new( - watch_low_benchmark: measure.watch_low_benchmark, - growth_low_benchmark: measure.growth_low_benchmark, - approval_low_benchmark: measure.approval_low_benchmark, - ideal_low_benchmark: measure.ideal_low_benchmark - ) - zones.zone_for_score(score.average) - end - - def average - average = score.average || 0 - - average.round(6) - end - - private - - def enforce_minimum_height(bar_height:) - bar_height < MINIMUM_BAR_HEIGHT ? MINIMUM_BAR_HEIGHT : bar_height - end - - def ideal_bar_height_percentage - (percentage * zone_height_percentage + zone_height_percentage) * 100 - end - - def approval_bar_height_percentage - (percentage * zone_height_percentage) * 100 - end - - def growth_bar_height_percentage - ((1 - percentage) * zone_height_percentage) * 100 - end - - def watch_bar_height_percentage - ((1 - percentage) * zone_height_percentage + zone_height_percentage) * 100 - end - - def warning_bar_height_percentage - ((1 - percentage) * zone_height_percentage + zone_height_percentage + zone_height_percentage) * 100 - end -end diff --git a/app/services/cleaner.rb b/app/services/cleaner.rb index 29d51904..f5a8d155 100644 --- a/app/services/cleaner.rb +++ b/app/services/cleaner.rb @@ -1,57 +1,34 @@ require "fileutils" class Cleaner - attr_reader :input_filepath, :output_filepath, :log_filepath, :clean_csv, :log_csv + attr_reader :input_filepath, :output_filepath, :log_filepath, :disaggregation_filepath - def initialize(input_filepath:, output_filepath:, log_filepath:) + def initialize(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:) @input_filepath = input_filepath @output_filepath = output_filepath @log_filepath = log_filepath + @disaggregation_filepath = disaggregation_filepath initialize_directories end - def initialize_directories - create_ouput_directory - create_log_directory - end - def clean Dir.glob(Rails.root.join(input_filepath, "*.csv")).each do |filepath| puts filepath File.open(filepath) do |file| - clean_csv = [] - log_csv = [] - data = [] - - headers = CSV.parse(file.first).first - filtered_headers = remove_unwanted_headers(headers:) - log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?", - "Valid Standard Deviation?"]).flatten - - clean_csv << filtered_headers - log_csv << log_headers - - all_survey_items = survey_items(headers:) - - file.lazy.each_slice(1000) do |lines| - CSV.parse(lines.join, headers:).map do |row| - values = SurveyItemValues.new(row:, headers:, genders:, - survey_items: all_survey_items, schools:) - next unless values.valid_school? - - data << values - values.valid? ? clean_csv << values.to_a : log_csv << (values.to_a << values.valid_duration?.to_s << values.valid_progress?.to_s << values.valid_grade?.to_s << values.valid_sd?.to_s) - end - end - - unless data.empty? - filename = filename(headers:, data:) - write_csv(data: clean_csv, output_filepath:, filename:) - write_csv(data: log_csv, output_filepath: log_filepath, prefix: "removed.", filename:) - end + processed_data = process_raw_file(file:, disaggregation_data:) + processed_data in [headers, clean_csv, log_csv, data] + return if data.empty? + + filename = filename(headers:, data:) + write_csv(data: clean_csv, output_filepath:, filename:) + write_csv(data: log_csv, output_filepath: log_filepath, prefix: "removed.", filename:) end end end + def disaggregation_data + @disaggregation_data ||= DisaggregationLoader.new(path: disaggregation_filepath).load + end + def filename(headers:, data:) survey_item_ids = headers.filter(&:present?).filter do |header| header.start_with?("s-", "t-") @@ -66,6 +43,41 @@ class Cleaner districts.join(".").to_s + "." + survey_type.to_s + "." + range + ".csv" end + def process_raw_file(file:, disaggregation_data:) + clean_csv = [] + log_csv = [] + data = [] + + headers = (CSV.parse(file.first).first << "Raw Income") << "Income" + filtered_headers = remove_unwanted_headers(headers:) + log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?", + "Valid Standard Deviation?"]).flatten + + clean_csv << filtered_headers + log_csv << log_headers + + all_survey_items = survey_items(headers:) + + file.lazy.each_slice(1000) do |lines| + CSV.parse(lines.join, headers:).map do |row| + values = SurveyItemValues.new(row:, headers:, genders:, + survey_items: all_survey_items, schools:, disaggregation_data:) + next unless values.valid_school? + + data << values + values.valid? ? clean_csv << values.to_a : log_csv << (values.to_a << values.valid_duration?.to_s << values.valid_progress?.to_s << values.valid_grade?.to_s << values.valid_sd?.to_s) + end + end + [headers, clean_csv, log_csv, data] + end + + private + + def initialize_directories + create_ouput_directory + create_log_directory + end + def remove_unwanted_headers(headers:) headers.to_set.to_a.compact.reject do |item| item.start_with? "Q" @@ -81,34 +93,19 @@ class Cleaner File.write(output_filepath.join(prefix + filename), csv) end - def process_row(row:) - clean_csv << row.to_csv - log_csv << row.to_csv - end - def schools @schools ||= School.school_hash end def genders - @genders ||= begin - gender_hash = {} - - Gender.all.each do |gender| - gender_hash[gender.qualtrics_code] = gender - end - gender_hash - end + @genders ||= Gender.gender_hash end def survey_items(headers:) - @survey_items ||= SurveyItem.where(survey_item_id: get_survey_item_ids_from_headers(headers:)) - end - - def get_survey_item_ids_from_headers(headers:) - headers - .filter(&:present?) - .filter { |header| header.start_with? "t-", "s-" } + survey_item_ids = headers + .filter(&:present?) + .filter { |header| header.start_with? "t-", "s-" } + @survey_items ||= SurveyItem.where(survey_item_id: survey_item_ids) end def create_ouput_directory @@ -118,8 +115,4 @@ class Cleaner def create_log_directory FileUtils.mkdir_p log_filepath end - - def create_file(path:, filename:) - FileUtils.touch path.join(filename) - end end diff --git a/app/services/demographic_loader.rb b/app/services/demographic_loader.rb index 80373a0a..8ad705ae 100644 --- a/app/services/demographic_loader.rb +++ b/app/services/demographic_loader.rb @@ -7,6 +7,7 @@ class DemographicLoader CSV.parse(File.read(filepath), headers: true) do |row| process_race(row:) process_gender(row:) + process_income(row:) end end @@ -30,6 +31,13 @@ class DemographicLoader gender = ::Gender.find_or_create_by!(qualtrics_code:, designation:) gender.save end + + def self.process_income(row:) + designation = row['Income'] + return unless designation + + Income.find_or_create_by!(designation:) + end end class KnownRace diff --git a/app/services/disaggregation_loader.rb b/app/services/disaggregation_loader.rb new file mode 100644 index 00000000..0b67c541 --- /dev/null +++ b/app/services/disaggregation_loader.rb @@ -0,0 +1,30 @@ +class DisaggregationLoader + attr_reader :path + + def initialize(path:) + @path = path + initialize_directory + end + + def load + data = {} + Dir.glob(Rails.root.join(path, "*.csv")).each do |filepath| + puts filepath + File.open(filepath) do |file| + headers = CSV.parse(file.first).first + + file.lazy.each_slice(1000) do |lines| + CSV.parse(lines.join, headers:).map do |row| + values = DisaggregationRow.new(row:, headers:) + data[[values.lasid, values.district, values.academic_year]] = values + end + end + end + end + data + end + + def initialize_directory + FileUtils.mkdir_p(path) + end +end diff --git a/app/services/disaggregation_row.rb b/app/services/disaggregation_row.rb new file mode 100644 index 00000000..0fefe53f --- /dev/null +++ b/app/services/disaggregation_row.rb @@ -0,0 +1,35 @@ +class DisaggregationRow + attr_reader :row, :headers + + def initialize(row:, headers:) + @row = row + @headers = headers + end + + def district + @district ||= value_from(pattern: /District/i) + end + + def academic_year + @academic_year ||= value_from(pattern: /Academic\s*Year/i) + end + + def income + @income ||= value_from(pattern: /Low\s*Income/i) + end + + def lasid + @lasid ||= value_from(pattern: /LASID/i) + end + + def value_from(pattern:) + output = nil + matches = headers.select do |header| + pattern.match(header) + end.map { |item| item.delete("\n") } + matches.each do |match| + output ||= row[match] + end + output + end +end diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index 99bbd96e..80a56e63 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -1,12 +1,13 @@ class SurveyItemValues - attr_reader :row, :headers, :genders, :survey_items, :schools + attr_reader :row, :headers, :genders, :survey_items, :schools, :disaggregation_data - def initialize(row:, headers:, genders:, survey_items:, schools:) + def initialize(row:, headers:, genders:, survey_items:, schools:, disaggregation_data: nil) @row = row @headers = headers @genders = genders @survey_items = survey_items @schools = schools + @disaggregation_data = disaggregation_data end def dese_id? @@ -93,6 +94,37 @@ class SurveyItemValues genders[gender_code] end + def lasid + @lasid ||= value_from(pattern: /LASID/i) + end + + def raw_income + @raw_income ||= value_from(pattern: /Low\s*Income|Raw\s*Income/i) + return @raw_income if @raw_income.present? + + return "Unknown" unless disaggregation_data.present? + + disaggregation = disaggregation_data[[lasid, district.name, academic_year.range]] + return "Unknown" unless disaggregation.present? + + @raw_income ||= disaggregation.income + end + + # TODO: - rename these cases + def income + @income ||= value_from(pattern: /^Income$/i) + return @income if @income.present? + + @income ||= case raw_income + in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income/i + "Economically Disadvantaged - Y" + in /Not\s*Eligible/i + "Economically Disadvantaged - N" + else + "Unknown" + end + end + def value_from(pattern:) output = nil matches = headers.select do |header| @@ -106,10 +138,12 @@ class SurveyItemValues def to_a copy_likert_scores_from_variant_survey_items + row["Income"] = income + row["Raw Income"] = raw_income headers.select(&:present?) - .reject { |key, _value| key.start_with? "Q" } - .reject { |key, _value| key.end_with? "-1" } - .map { |header| row[header] } + .reject { |key, _value| key.start_with? "Q" } + .reject { |key, _value| key.end_with? "-1" } + .map { |header| row[header] } end def duration @@ -122,17 +156,17 @@ class SurveyItemValues def respondent_type return :teacher if headers - .filter(&:present?) - .filter { |header| header.start_with? "t-" }.count > 0 + .filter(&:present?) + .filter { |header| header.start_with? "t-" }.count > 0 :student end def survey_type survey_item_ids = headers - .filter(&:present?) - .reject { |header| header.end_with?("-1") } - .filter { |header| header.start_with?("t-", "s-") } + .filter(&:present?) + .reject { |header| header.end_with?("-1") } + .filter { |header| header.start_with?("t-", "s-") } SurveyItem.survey_type(survey_item_ids:) end @@ -200,3 +234,4 @@ class SurveyItemValues end end end + diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index d7946ce0..5d063764 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -7,12 +7,13 @@ class SurveyResponsesDataLoader headers_array = CSV.parse(headers).first genders = Gender.gender_hash schools = School.school_hash + incomes = Income.by_designation all_survey_items = survey_items(headers:) file.lazy.each_slice(500) do |lines| survey_item_responses = CSV.parse(lines.join, headers:).map do |row| process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:), - rules:) + rules:, incomes:) end SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500 end @@ -24,6 +25,7 @@ class SurveyResponsesDataLoader headers_array = CSV.parse(headers).first genders = Gender.gender_hash schools = School.school_hash + incomes = Income.by_designation all_survey_items = survey_items(headers:) survey_item_responses = [] @@ -34,7 +36,7 @@ class SurveyResponsesDataLoader CSV.parse(line, headers:).map do |row| survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:), - rules:) + rules:, incomes:) end row_count += 1 @@ -50,7 +52,7 @@ class SurveyResponsesDataLoader private - def self.process_row(row:, rules:) + def self.process_row(row:, rules:, incomes:) return unless row.dese_id? return unless row.school.present? @@ -58,10 +60,10 @@ class SurveyResponsesDataLoader return if rule.new(row:).skip_row? end - process_survey_items(row:) + process_survey_items(row:, incomes:) end - def self.process_survey_items(row:) + def self.process_survey_items(row:, incomes:) row.survey_items.map do |survey_item| likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next @@ -70,19 +72,20 @@ class SurveyResponsesDataLoader next end response = row.survey_item_response(survey_item:) - create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:) + create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, incomes:) end.compact end - def self.create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:) + def self.create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, incomes:) gender = row.gender grade = row.grade + income = incomes[row.income] if survey_item_response.present? - survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date) + survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:) [] else SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:, - likert_score:, grade:, gender:, recorded_date: row.recorded_date) + likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:) end end diff --git a/app/views/analyze/_checkboxes.html.erb b/app/views/analyze/_checkboxes.html.erb new file mode 100644 index 00000000..d64d7e6b --- /dev/null +++ b/app/views/analyze/_checkboxes.html.erb @@ -0,0 +1,17 @@ +
+ + <%= @presenter.graph.slug == 'students-and-teachers' || @presenter.source.slug == 'all-data' ? "disabled" : "" %> + <%= @presenter.group.slug == name ? "" : "hidden" %>> + + +
diff --git a/app/views/analyze/_data_filters.html.erb b/app/views/analyze/_data_filters.html.erb index e7011a36..0cba336f 100644 --- a/app/views/analyze/_data_filters.html.erb +++ b/app/views/analyze/_data_filters.html.erb @@ -1,7 +1,7 @@

Data Filters

- <% @sources.each do |source| %> + <% @presenter.sources.each do |source| %> > + <%= source.slug == @presenter.source.slug ? "checked" : "" %>> <% source.slices.each do | slice | %> @@ -20,7 +20,7 @@ name="slice" value="<%= base_url %>" data-action="click->analyze#refresh" - <%= slice.slug == @slice.slug ? "checked" : "" %> + <%= slice.slug == @presenter.slice.slug ? "checked" : "" %> <%= slice.slug == "all-data" ? "hidden" : "" %>>