diff --git a/app/controllers/analyze_controller.rb b/app/controllers/analyze_controller.rb index a0f1469b..f46fa907 100644 --- a/app/controllers/analyze_controller.rb +++ b/app/controllers/analyze_controller.rb @@ -1,172 +1,8 @@ # frozen_string_literal: true class AnalyzeController < SqmApplicationController - before_action :assign_categories, :assign_subcategories, :assign_measures, :assign_academic_years, - :races, :selected_races, :graph, :graphs, :background, :race_score_timestamp, - :source, :sources, :group, :groups, :selected_grades, :grades, :slice, :selected_genders, :genders, only: [:index] - def index; end - - private - - def assign_categories - @category ||= Category.find_by_category_id(params[:category]) - @category ||= Category.order(:category_id).first - @categories = Category.all.order(:category_id) - end - - def assign_subcategories - @subcategories = @category.subcategories.order(:subcategory_id) - @subcategory ||= Subcategory.find_by_subcategory_id(params[:subcategory]) - @subcategory ||= @subcategories.first - end - - def assign_measures - @measures = @subcategory.measures.order(:measure_id).includes(%i[admin_data_items subcategory]) - end - - def assign_academic_years - @available_academic_years = AcademicYear.order(:range).all - year_params = params[:academic_years] - @academic_year_params = year_params.split(',') if year_params - @selected_academic_years = [] - @academic_year_params ||= [] - @academic_year_params.each do |year| - @selected_academic_years << AcademicYear.find_by_range(year) - end - end - - def races - @races ||= Race.all.order(designation: :ASC) - end - - def selected_races - @selected_races ||= begin - race_params = params[:races] - return @selected_races = races unless race_params - - race_list = race_params.split(',') if race_params - if race_list - race_list = race_list.map do |race| - Race.find_by_slug race - end - end - race_list - end - end - - def graph - graphs.each do |graph| - @graph = graph if graph.slug == params[:graph] - end - - @graph ||= graphs.first - end - - def graphs - @graphs ||= [Analyze::Graph::AllData.new, Analyze::Graph::StudentsAndTeachers.new, Analyze::Graph::StudentsByRace.new(races: selected_races), - Analyze::Graph::StudentsByGrade.new(grades: selected_grades), Analyze::Graph::StudentsByGender.new(genders: selected_genders)] - end - - def background - @background ||= BackgroundPresenter.new(num_of_columns: graph.columns.count) - end - - def race_score_timestamp - @race_score_timestamp ||= begin - score = RaceScore.where(school: @school, - academic_year: @academic_year).order(updated_at: :DESC).first || Today.new - score.updated_at - end - end - - def source - source_param = params[:source] - sources.each do |source| - @source = source if source.slug == source_param - end - - @source ||= sources.first - end - - def sources - all_data_slices = [Analyze::Slice::AllData.new] - all_data_source = Analyze::Source::AllData.new(slices: all_data_slices) - - students_and_teachers = Analyze::Slice::StudentsAndTeachers.new - students_by_group = Analyze::Slice::StudentsByGroup.new(races:, grades:) - survey_data_slices = [students_and_teachers, students_by_group] - survey_data_source = Analyze::Source::SurveyData.new(slices: survey_data_slices) - - @sources = [all_data_source, survey_data_source] - end - - def slice - slice_param = params[:slice] - slices.each do |slice| - @slice = slice if slice.slug == slice_param - end - - @slice ||= slices.first - end - - def slices - source.slices - end - - def group - groups.each do |group| - @group = group if group.slug == params[:group] - end - - @group ||= groups.first - end - - def groups - @groups = [Analyze::Group::Race.new, Analyze::Group::Grade.new, Analyze::Group::Gender.new] - end - - def selected_grades - @selected_grades ||= begin - grade_params = params[:grades] - return @selected_grades = grades unless grade_params - - grade_list = grade_params.split(',') if grade_params - if grade_list - grade_list = grade_list.map do |grade| - grade.to_i - end - end - grade_list - end - end - - def grades - @grades ||= SurveyItemResponse.where(school: @school, academic_year: @academic_year) - .where.not(grade: nil) - .group(:grade) - .select(:response_id) - .distinct(:response_id) - .count.reject do |_key, value| - value < 10 - end.keys - end - - def selected_genders - @selected_genders ||= begin - gender_params = params[:genders] - return @selected_genders = genders unless gender_params - - gender_list = gender_params.split(',') if gender_params - if gender_list - gender_list = gender_list.map do |gender| - Gender.find_by_designation(gender) - end - end - gender_list - end - end - - def genders - @genders ||= Gender.all + def index + @presenter = Analyze::Presenter.new(params:, school: @school, academic_year: @academic_year) + @background ||= BackgroundPresenter.new(num_of_columns: @presenter.graph.columns.count) end end diff --git a/app/helpers/analyze_helper.rb b/app/helpers/analyze_helper.rb index 7ac34581..e253167c 100644 --- a/app/helpers/analyze_helper.rb +++ b/app/helpers/analyze_helper.rb @@ -62,7 +62,7 @@ module AnalyzeHelper end def base_url - analyze_subcategory_link(district: @district, school: @school, academic_year: @academic_year, category: @category, - subcategory: @subcategory) + analyze_subcategory_link(district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, + subcategory: @presenter.subcategory) end end diff --git a/app/javascript/controllers/analyze_controller.js b/app/javascript/controllers/analyze_controller.js index ed11416a..bf290c9b 100644 --- a/app/javascript/controllers/analyze_controller.js +++ b/app/javascript/controllers/analyze_controller.js @@ -22,16 +22,17 @@ export default class extends Controller { "&graph=" + this.selected_graph(target) + "&races=" + - this.selected_races().join(",") + + this.selected_items("race").join(",") + "&genders=" + - this.selected_genders().join(",") + + this.selected_items("gender").join(",") + + "&incomes=" + + this.selected_items("income").join(",") + "&grades=" + - this.selected_grades().join(","); + this.selected_items("grade").join(","); this.go_to(url); } - go_to(location) { window.location = location; } @@ -121,58 +122,34 @@ export default class extends Controller { return item.id; })[0]; + const groups = new Map([ + ['gender', 'students-by-gender'], + ['grade', 'students-by-grade'], + ['income', 'students-by-income'], + ['race', 'students-by-race'] + ]) + if (target.name === 'slice' || target.name === 'group') { if (selected_slice === 'students-and-teachers') { return 'students-and-teachers'; - } else if (this.selected_group() === 'race') { - return 'students-by-race'; - } else if (this.selected_group() === 'gender') { - return 'students-by-gender'; - } else if (this.selected_group() === 'grade') { - return 'students-by-grade'; } + return groups.get(this.selected_group()); } return window.graph; } - selected_races() { - let race_checkboxes = [...document.getElementsByName("race-checkbox")] - let races = race_checkboxes - .filter((item) => { - return item.checked; - }) - .map((item) => { - return item.id; - }); - - return races; - } - - selected_grades() { - let grade_checkboxes = [...document.getElementsByName("grade-checkbox")] - let grades = grade_checkboxes - .filter((item) => { - return item.checked; - }) - .map((item) => { - return item.id.replace('grade-', ''); - }); - - return grades; - } - - selected_genders() { - let gender_checkboxes = [...document.getElementsByName("gender-checkbox")] - let genders = gender_checkboxes + selected_items(type) { + let checkboxes = [...document.getElementsByName(`${type}-checkbox`)] + let items = checkboxes .filter((item) => { return item.checked; }) .map((item) => { - return item.id.replace('gender-', ''); + return item.id.replace(`${type}-`, ''); }); - return genders; + return items; } } diff --git a/app/models/income.rb b/app/models/income.rb new file mode 100644 index 00000000..447f8465 --- /dev/null +++ b/app/models/income.rb @@ -0,0 +1,7 @@ +class Income < ApplicationRecord + scope :by_designation, -> { all.map { |income| [income.designation, income] }.to_h } + + include FriendlyId + + friendly_id :designation, use: [:slugged] +end diff --git a/app/models/survey_item_response.rb b/app/models/survey_item_response.rb index cf3986bc..50810867 100644 --- a/app/models/survey_item_response.rb +++ b/app/models/survey_item_response.rb @@ -9,6 +9,7 @@ class SurveyItemResponse < ActiveRecord::Base belongs_to :survey_item, counter_cache: true belongs_to :student, foreign_key: :student_id, optional: true belongs_to :gender + belongs_to :income has_one :measure, through: :survey_item @@ -31,4 +32,9 @@ class SurveyItemResponse < ActiveRecord::Base SurveyItemResponse.where(survey_item: survey_items, school:, academic_year:, income:, grade: school.grades(academic_year:)).group(:survey_item).average(:likert_score) } + + scope :averages_for_income, lambda { |survey_items, school, academic_year, income| + SurveyItemResponse.where(survey_item: survey_items, school:, + academic_year:, income:).group(:survey_item).average(:likert_score) + } end diff --git a/app/presenters/analyze/bar_presenter.rb b/app/presenters/analyze/bar_presenter.rb new file mode 100644 index 00000000..7dc9bdc1 --- /dev/null +++ b/app/presenters/analyze/bar_presenter.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Analyze + class BarPresenter + include AnalyzeHelper + attr_reader :score, :x_position, :academic_year, :measure_id, :measure, :color + + MINIMUM_BAR_HEIGHT = 2 + + def initialize(measure:, academic_year:, score:, x_position:, color:) + @score = score + @x_position = x_position + @academic_year = academic_year + @measure = measure + @measure_id = measure.measure_id + @color = color + end + + def y_offset + benchmark_height = analyze_zone_height * 2 + case zone.type + when :ideal, :approval + benchmark_height - bar_height_percentage + else + benchmark_height + end + end + + def bar_color + "fill-#{zone.type}" + end + + def bar_height_percentage + bar_height = send("#{zone.type}_bar_height_percentage") || 0 + enforce_minimum_height(bar_height:) + end + + def percentage + low_benchmark = zone.low_benchmark + (score.average - low_benchmark) / (zone.high_benchmark - low_benchmark) + end + + def zone + zones = Zones.new( + watch_low_benchmark: measure.watch_low_benchmark, + growth_low_benchmark: measure.growth_low_benchmark, + approval_low_benchmark: measure.approval_low_benchmark, + ideal_low_benchmark: measure.ideal_low_benchmark + ) + zones.zone_for_score(score.average) + end + + def average + average = score.average || 0 + + average.round(6) + end + + private + + def enforce_minimum_height(bar_height:) + bar_height < MINIMUM_BAR_HEIGHT ? MINIMUM_BAR_HEIGHT : bar_height + end + + def ideal_bar_height_percentage + (percentage * zone_height_percentage + zone_height_percentage) * 100 + end + + def approval_bar_height_percentage + (percentage * zone_height_percentage) * 100 + end + + def growth_bar_height_percentage + ((1 - percentage) * zone_height_percentage) * 100 + end + + def watch_bar_height_percentage + ((1 - percentage) * zone_height_percentage + zone_height_percentage) * 100 + end + + def warning_bar_height_percentage + ((1 - percentage) * zone_height_percentage + zone_height_percentage + zone_height_percentage) * 100 + end + end +end diff --git a/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb b/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb index 9e826c88..e796af97 100644 --- a/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb +++ b/app/presenters/analyze/graph/column/grouped_bar_column_presenter.rb @@ -27,10 +27,10 @@ module Analyze def bars @bars ||= yearly_scores.map.each_with_index do |yearly_score, index| year = yearly_score.year - AnalyzeBarPresenter.new(measure:, academic_year: year, - score: yearly_score.score, - x_position: bar_x(index), - color: bar_color(year)) + Analyze::BarPresenter.new(measure:, academic_year: year, + score: yearly_score.score, + x_position: bar_x(index), + color: bar_color(year)) end end diff --git a/app/presenters/analyze/graph/column/income_column/disadvantaged.rb b/app/presenters/analyze/graph/column/income_column/disadvantaged.rb index c6569611..ad304ce9 100644 --- a/app/presenters/analyze/graph/column/income_column/disadvantaged.rb +++ b/app/presenters/analyze/graph/column/income_column/disadvantaged.rb @@ -27,3 +27,4 @@ module Analyze end end end + diff --git a/app/presenters/analyze/graph/column/income_column/not_disadvantaged.rb b/app/presenters/analyze/graph/column/income_column/not_disadvantaged.rb index e48c653f..4a1d4c21 100644 --- a/app/presenters/analyze/graph/column/income_column/not_disadvantaged.rb +++ b/app/presenters/analyze/graph/column/income_column/not_disadvantaged.rb @@ -27,3 +27,4 @@ module Analyze end end end + diff --git a/app/presenters/analyze/graph/column/income_column/score_for_income.rb b/app/presenters/analyze/graph/column/income_column/score_for_income.rb new file mode 100644 index 00000000..3e1140c3 --- /dev/null +++ b/app/presenters/analyze/graph/column/income_column/score_for_income.rb @@ -0,0 +1,43 @@ +module Analyze + module Graph + module Column + module IncomeColumn + module ScoreForIncome + def score(year_index) + academic_year = academic_years[year_index] + averages = SurveyItemResponse.averages_for_income(measure.student_survey_items, school, academic_year, + income) + average = bubble_up_averages(averages:).round(2) + + scorify(average:, meets_student_threshold: sufficient_student_responses?(academic_year:)) + end + + def bubble_up_averages(averages:) + measure.student_scales.map do |scale| + scale.survey_items.map do |survey_item| + averages[survey_item] + end.remove_blanks.average + end.remove_blanks.average + end + + def scorify(average:, meets_student_threshold:) + return Score::NIL_SCORE unless meets_student_threshold + + Score.new(average:, + meets_teacher_threshold: false, + meets_student_threshold: true, + meets_admin_data_threshold: false) + end + + def sufficient_student_responses?(academic_year:) + yearly_counts = SurveyItemResponse.where(school:, academic_year:, + income:, survey_item: measure.student_survey_items).group(:income).select(:response_id).distinct(:response_id).count + yearly_counts.any? do |count| + count[1] >= 10 + end + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/column/income_column/unknown.rb b/app/presenters/analyze/graph/column/income_column/unknown.rb index eee7c8be..e6e0ba8b 100644 --- a/app/presenters/analyze/graph/column/income_column/unknown.rb +++ b/app/presenters/analyze/graph/column/income_column/unknown.rb @@ -27,3 +27,4 @@ module Analyze end end end + diff --git a/app/presenters/analyze/graph/column/score_for_race.rb b/app/presenters/analyze/graph/column/score_for_race.rb index 207e198f..36bee8a5 100644 --- a/app/presenters/analyze/graph/column/score_for_race.rb +++ b/app/presenters/analyze/graph/column/score_for_race.rb @@ -3,16 +3,64 @@ module Analyze module Column module ScoreForRace def score(year_index) - s = ::RaceScore.find_by(measure:, school:, academic_year: academic_years[year_index], race:) - average = s.average.round(2) unless s.nil? - average ||= 0 - meets_student_threshold = s.meets_student_threshold? unless s.nil? - meets_student_threshold ||= false - Score.new(average:, - meets_teacher_threshold: false, - meets_student_threshold:, + academic_year = academic_year_for_year_index(year_index) + rate = response_rate(school:, academic_year:, measure:) + return Score::NIL_SCORE unless rate.meets_student_threshold + + survey_items = measure.student_survey_items + + averages = grouped_responses(school:, academic_year:, survey_items:, race:) + meets_student_threshold = sufficient_responses(school:, academic_year:, race:) + scorify(responses: averages, meets_student_threshold:, measure:) + end + + def grouped_responses(school:, academic_year:, survey_items:, race:) + @grouped_responses ||= Hash.new do |memo, (school, academic_year, survey_items, race)| + memo[[school, academic_year, survey_items, race]] = + SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where( + school:, academic_year:, grade: school.grades(academic_year:) + ).where("student_races.race_id": race.id).group(:survey_item_id).having("count(*) >= 10").average(:likert_score) + end + + @grouped_responses[[school, academic_year, survey_items, race]] + end + + def response_rate(school:, academic_year:, measure:) + subcategory = measure.subcategory + @response_rate ||= Hash.new do |memo, (school, academic_year, subcategory)| + memo[[school, academic_year, subcategory]] = subcategory.response_rate(school:, academic_year:) + end + + @response_rate[[school, academic_year, subcategory]] + end + + def scorify(responses:, meets_student_threshold:, measure:) + averages = bubble_up_averages(responses:, measure:) + average = averages.average.round(2) + + average = 0 unless meets_student_threshold + + Score.new(average:, meets_teacher_threshold: false, meets_student_threshold:, meets_admin_data_threshold: false) end + + def sufficient_responses(school:, academic_year:, race:) + @sufficient_responses ||= Hash.new do |memo, (school, academic_year, race)| + number_of_students_for_a_racial_group = SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where( + school:, academic_year: + ).where("student_races.race_id": race.id).distinct.pluck(:student_id).count + memo[[school, academic_year, race]] = number_of_students_for_a_racial_group >= 10 + end + @sufficient_responses[[school, academic_year, race]] + end + + def bubble_up_averages(responses:, measure:) + measure.student_scales.map do |scale| + scale.survey_items.map do |survey_item| + responses[survey_item.id] + end.remove_blanks.average + end.remove_blanks + end end end end diff --git a/app/presenters/analyze/graph/students_by_income.rb b/app/presenters/analyze/graph/students_by_income.rb new file mode 100644 index 00000000..148b22c1 --- /dev/null +++ b/app/presenters/analyze/graph/students_by_income.rb @@ -0,0 +1,40 @@ +module Analyze + module Graph + class StudentsByIncome + attr_reader :incomes + + def initialize(incomes:) + @incomes = incomes + end + + def to_s + "Students by income" + end + + def slug + "students-by-income" + end + + def columns + [].tap do |array| + incomes.each do |income| + array << column_for_income_code(code: income.slug) + end + array << Analyze::Graph::Column::AllStudent + end + end + + private + + def column_for_income_code(code:) + CFR[code.to_s] + end + + CFR = { + "economically-disadvantaged-y" => Analyze::Graph::Column::IncomeColumn::Disadvantaged, + "economically-disadvantaged-n" => Analyze::Graph::Column::IncomeColumn::NotDisadvantaged, + "unknown" => Analyze::Graph::Column::IncomeColumn::Unknown + }.freeze + end + end +end diff --git a/app/presenters/analyze/group/income.rb b/app/presenters/analyze/group/income.rb new file mode 100644 index 00000000..21e364b4 --- /dev/null +++ b/app/presenters/analyze/group/income.rb @@ -0,0 +1,13 @@ +module Analyze + module Group + class Income + def name + 'Income' + end + + def slug + 'income' + end + end + end +end diff --git a/app/presenters/analyze/presenter.rb b/app/presenters/analyze/presenter.rb new file mode 100644 index 00000000..f68f2997 --- /dev/null +++ b/app/presenters/analyze/presenter.rb @@ -0,0 +1,153 @@ +module Analyze + class Presenter + attr_reader :params, :school, :academic_year + + def initialize(params:, school:, academic_year:) + @params = params + @school = school + @academic_year = academic_year + end + + def category + @category ||= Category.find_by_category_id(params[:category]) || Category.order(:category_id).first + end + + def categories + @categories = Category.all.order(:category_id) + end + + def subcategory + @subcategory ||= Subcategory.find_by_subcategory_id(params[:subcategory]) || subcategories.first + end + + def subcategories + @subcategories = category.subcategories.order(:subcategory_id) + end + + def measures + @measures = subcategory.measures.order(:measure_id).includes(%i[admin_data_items subcategory]) + end + + def academic_years + @academic_years = AcademicYear.order(:range).all + end + + def selected_academic_years + @selected_academic_years ||= begin + year_params = params[:academic_years] + return [] unless year_params + + year_params.split(",").map { |year| AcademicYear.find_by_range(year) }.compact + end + end + + def races + @races ||= Race.all.order(designation: :ASC) + end + + def selected_races + @selected_races ||= begin + race_params = params[:races] + return races unless race_params + + race_params.split(",").map { |race| Race.find_by_slug race }.compact + end + end + + def graphs + @graphs ||= [Analyze::Graph::AllData.new, Analyze::Graph::StudentsAndTeachers.new, Analyze::Graph::StudentsByRace.new(races: selected_races), + Analyze::Graph::StudentsByGrade.new(grades: selected_grades), Analyze::Graph::StudentsByGender.new(genders: selected_genders), Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes)] + end + + def graph + @graph ||= graphs.reduce(graphs.first) do |acc, graph| + graph.slug == params[:graph] ? graph : acc + end + end + + def selected_grades + @selected_grades ||= begin + grade_params = params[:grades] + return grades unless grade_params + + grade_params.split(",").map(&:to_i) + end + end + + def selected_genders + @selected_genders ||= begin + gender_params = params[:genders] + return genders unless gender_params + + gender_params.split(",").map { |gender| Gender.find_by_designation(gender) }.compact + end + end + + def genders + @genders ||= Gender.all + end + + def groups + @groups = [Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new, + Analyze::Group::Race.new] + end + + def group + @group ||= groups.reduce(groups.first) do |acc, group| + group.slug == params[:group] ? group : acc + end + end + + def slice + @slice ||= slices.reduce(slices.first) do |acc, slice| + slice.slug == params[:slice] ? slice : acc + end + end + + def slices + source.slices + end + + def source + @source ||= sources.reduce(sources.first) do |acc, source| + source.slug == params[:source] ? source : acc + end + end + + def sources + all_data_slices = [Analyze::Slice::AllData.new] + all_data_source = Analyze::Source::AllData.new(slices: all_data_slices) + + students_and_teachers = Analyze::Slice::StudentsAndTeachers.new + students_by_group = Analyze::Slice::StudentsByGroup.new(races:, grades:) + survey_data_slices = [students_and_teachers, students_by_group] + survey_data_source = Analyze::Source::SurveyData.new(slices: survey_data_slices) + + @sources = [all_data_source, survey_data_source] + end + + def grades + @grades ||= SurveyItemResponse.where(school:, academic_year:) + .where.not(grade: nil) + .group(:grade) + .select(:response_id) + .distinct(:response_id) + .count.reject do |_key, value| + value < 10 + end.keys + end + + def incomes + @incomes ||= Income.all + end + + def selected_incomes + @selected_incomes ||= begin + income_params = params[:incomes] + return incomes unless income_params + + income_params.split(",").map { |income| Income.find_by_slug(income) }.compact + end + end + end +end diff --git a/app/presenters/analyze/ui.rb b/app/presenters/analyze/ui.rb deleted file mode 100644 index cf440140..00000000 --- a/app/presenters/analyze/ui.rb +++ /dev/null @@ -1,8 +0,0 @@ -module Analyze - class Ui - attr_reader :params - def initialize(params:) - @params = params - end - end -end diff --git a/app/presenters/analyze_bar_presenter.rb b/app/presenters/analyze_bar_presenter.rb deleted file mode 100644 index 32582411..00000000 --- a/app/presenters/analyze_bar_presenter.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -class AnalyzeBarPresenter - include AnalyzeHelper - attr_reader :score, :x_position, :academic_year, :measure_id, :measure, :color - - MINIMUM_BAR_HEIGHT = 2 - - def initialize(measure:, academic_year:, score:, x_position:, color:) - @score = score - @x_position = x_position - @academic_year = academic_year - @measure = measure - @measure_id = measure.measure_id - @color = color - end - - def y_offset - benchmark_height = analyze_zone_height * 2 - case zone.type - when :ideal, :approval - benchmark_height - bar_height_percentage - else - benchmark_height - end - end - - def bar_color - "fill-#{zone.type}" - end - - def bar_height_percentage - bar_height = send("#{zone.type}_bar_height_percentage") || 0 - enforce_minimum_height(bar_height:) - end - - def percentage - low_benchmark = zone.low_benchmark - (score.average - low_benchmark) / (zone.high_benchmark - low_benchmark) - end - - def zone - zones = Zones.new( - watch_low_benchmark: measure.watch_low_benchmark, - growth_low_benchmark: measure.growth_low_benchmark, - approval_low_benchmark: measure.approval_low_benchmark, - ideal_low_benchmark: measure.ideal_low_benchmark - ) - zones.zone_for_score(score.average) - end - - def average - average = score.average || 0 - - average.round(6) - end - - private - - def enforce_minimum_height(bar_height:) - bar_height < MINIMUM_BAR_HEIGHT ? MINIMUM_BAR_HEIGHT : bar_height - end - - def ideal_bar_height_percentage - (percentage * zone_height_percentage + zone_height_percentage) * 100 - end - - def approval_bar_height_percentage - (percentage * zone_height_percentage) * 100 - end - - def growth_bar_height_percentage - ((1 - percentage) * zone_height_percentage) * 100 - end - - def watch_bar_height_percentage - ((1 - percentage) * zone_height_percentage + zone_height_percentage) * 100 - end - - def warning_bar_height_percentage - ((1 - percentage) * zone_height_percentage + zone_height_percentage + zone_height_percentage) * 100 - end -end diff --git a/app/services/cleaner.rb b/app/services/cleaner.rb index 29d51904..f5a8d155 100644 --- a/app/services/cleaner.rb +++ b/app/services/cleaner.rb @@ -1,57 +1,34 @@ require "fileutils" class Cleaner - attr_reader :input_filepath, :output_filepath, :log_filepath, :clean_csv, :log_csv + attr_reader :input_filepath, :output_filepath, :log_filepath, :disaggregation_filepath - def initialize(input_filepath:, output_filepath:, log_filepath:) + def initialize(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:) @input_filepath = input_filepath @output_filepath = output_filepath @log_filepath = log_filepath + @disaggregation_filepath = disaggregation_filepath initialize_directories end - def initialize_directories - create_ouput_directory - create_log_directory - end - def clean Dir.glob(Rails.root.join(input_filepath, "*.csv")).each do |filepath| puts filepath File.open(filepath) do |file| - clean_csv = [] - log_csv = [] - data = [] - - headers = CSV.parse(file.first).first - filtered_headers = remove_unwanted_headers(headers:) - log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?", - "Valid Standard Deviation?"]).flatten - - clean_csv << filtered_headers - log_csv << log_headers - - all_survey_items = survey_items(headers:) - - file.lazy.each_slice(1000) do |lines| - CSV.parse(lines.join, headers:).map do |row| - values = SurveyItemValues.new(row:, headers:, genders:, - survey_items: all_survey_items, schools:) - next unless values.valid_school? - - data << values - values.valid? ? clean_csv << values.to_a : log_csv << (values.to_a << values.valid_duration?.to_s << values.valid_progress?.to_s << values.valid_grade?.to_s << values.valid_sd?.to_s) - end - end - - unless data.empty? - filename = filename(headers:, data:) - write_csv(data: clean_csv, output_filepath:, filename:) - write_csv(data: log_csv, output_filepath: log_filepath, prefix: "removed.", filename:) - end + processed_data = process_raw_file(file:, disaggregation_data:) + processed_data in [headers, clean_csv, log_csv, data] + return if data.empty? + + filename = filename(headers:, data:) + write_csv(data: clean_csv, output_filepath:, filename:) + write_csv(data: log_csv, output_filepath: log_filepath, prefix: "removed.", filename:) end end end + def disaggregation_data + @disaggregation_data ||= DisaggregationLoader.new(path: disaggregation_filepath).load + end + def filename(headers:, data:) survey_item_ids = headers.filter(&:present?).filter do |header| header.start_with?("s-", "t-") @@ -66,6 +43,41 @@ class Cleaner districts.join(".").to_s + "." + survey_type.to_s + "." + range + ".csv" end + def process_raw_file(file:, disaggregation_data:) + clean_csv = [] + log_csv = [] + data = [] + + headers = (CSV.parse(file.first).first << "Raw Income") << "Income" + filtered_headers = remove_unwanted_headers(headers:) + log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?", + "Valid Standard Deviation?"]).flatten + + clean_csv << filtered_headers + log_csv << log_headers + + all_survey_items = survey_items(headers:) + + file.lazy.each_slice(1000) do |lines| + CSV.parse(lines.join, headers:).map do |row| + values = SurveyItemValues.new(row:, headers:, genders:, + survey_items: all_survey_items, schools:, disaggregation_data:) + next unless values.valid_school? + + data << values + values.valid? ? clean_csv << values.to_a : log_csv << (values.to_a << values.valid_duration?.to_s << values.valid_progress?.to_s << values.valid_grade?.to_s << values.valid_sd?.to_s) + end + end + [headers, clean_csv, log_csv, data] + end + + private + + def initialize_directories + create_ouput_directory + create_log_directory + end + def remove_unwanted_headers(headers:) headers.to_set.to_a.compact.reject do |item| item.start_with? "Q" @@ -81,34 +93,19 @@ class Cleaner File.write(output_filepath.join(prefix + filename), csv) end - def process_row(row:) - clean_csv << row.to_csv - log_csv << row.to_csv - end - def schools @schools ||= School.school_hash end def genders - @genders ||= begin - gender_hash = {} - - Gender.all.each do |gender| - gender_hash[gender.qualtrics_code] = gender - end - gender_hash - end + @genders ||= Gender.gender_hash end def survey_items(headers:) - @survey_items ||= SurveyItem.where(survey_item_id: get_survey_item_ids_from_headers(headers:)) - end - - def get_survey_item_ids_from_headers(headers:) - headers - .filter(&:present?) - .filter { |header| header.start_with? "t-", "s-" } + survey_item_ids = headers + .filter(&:present?) + .filter { |header| header.start_with? "t-", "s-" } + @survey_items ||= SurveyItem.where(survey_item_id: survey_item_ids) end def create_ouput_directory @@ -118,8 +115,4 @@ class Cleaner def create_log_directory FileUtils.mkdir_p log_filepath end - - def create_file(path:, filename:) - FileUtils.touch path.join(filename) - end end diff --git a/app/services/demographic_loader.rb b/app/services/demographic_loader.rb index 80373a0a..8ad705ae 100644 --- a/app/services/demographic_loader.rb +++ b/app/services/demographic_loader.rb @@ -7,6 +7,7 @@ class DemographicLoader CSV.parse(File.read(filepath), headers: true) do |row| process_race(row:) process_gender(row:) + process_income(row:) end end @@ -30,6 +31,13 @@ class DemographicLoader gender = ::Gender.find_or_create_by!(qualtrics_code:, designation:) gender.save end + + def self.process_income(row:) + designation = row['Income'] + return unless designation + + Income.find_or_create_by!(designation:) + end end class KnownRace diff --git a/app/services/disaggregation_loader.rb b/app/services/disaggregation_loader.rb new file mode 100644 index 00000000..0b67c541 --- /dev/null +++ b/app/services/disaggregation_loader.rb @@ -0,0 +1,30 @@ +class DisaggregationLoader + attr_reader :path + + def initialize(path:) + @path = path + initialize_directory + end + + def load + data = {} + Dir.glob(Rails.root.join(path, "*.csv")).each do |filepath| + puts filepath + File.open(filepath) do |file| + headers = CSV.parse(file.first).first + + file.lazy.each_slice(1000) do |lines| + CSV.parse(lines.join, headers:).map do |row| + values = DisaggregationRow.new(row:, headers:) + data[[values.lasid, values.district, values.academic_year]] = values + end + end + end + end + data + end + + def initialize_directory + FileUtils.mkdir_p(path) + end +end diff --git a/app/services/disaggregation_row.rb b/app/services/disaggregation_row.rb new file mode 100644 index 00000000..0fefe53f --- /dev/null +++ b/app/services/disaggregation_row.rb @@ -0,0 +1,35 @@ +class DisaggregationRow + attr_reader :row, :headers + + def initialize(row:, headers:) + @row = row + @headers = headers + end + + def district + @district ||= value_from(pattern: /District/i) + end + + def academic_year + @academic_year ||= value_from(pattern: /Academic\s*Year/i) + end + + def income + @income ||= value_from(pattern: /Low\s*Income/i) + end + + def lasid + @lasid ||= value_from(pattern: /LASID/i) + end + + def value_from(pattern:) + output = nil + matches = headers.select do |header| + pattern.match(header) + end.map { |item| item.delete("\n") } + matches.each do |match| + output ||= row[match] + end + output + end +end diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index 99bbd96e..80a56e63 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -1,12 +1,13 @@ class SurveyItemValues - attr_reader :row, :headers, :genders, :survey_items, :schools + attr_reader :row, :headers, :genders, :survey_items, :schools, :disaggregation_data - def initialize(row:, headers:, genders:, survey_items:, schools:) + def initialize(row:, headers:, genders:, survey_items:, schools:, disaggregation_data: nil) @row = row @headers = headers @genders = genders @survey_items = survey_items @schools = schools + @disaggregation_data = disaggregation_data end def dese_id? @@ -93,6 +94,37 @@ class SurveyItemValues genders[gender_code] end + def lasid + @lasid ||= value_from(pattern: /LASID/i) + end + + def raw_income + @raw_income ||= value_from(pattern: /Low\s*Income|Raw\s*Income/i) + return @raw_income if @raw_income.present? + + return "Unknown" unless disaggregation_data.present? + + disaggregation = disaggregation_data[[lasid, district.name, academic_year.range]] + return "Unknown" unless disaggregation.present? + + @raw_income ||= disaggregation.income + end + + # TODO: - rename these cases + def income + @income ||= value_from(pattern: /^Income$/i) + return @income if @income.present? + + @income ||= case raw_income + in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income/i + "Economically Disadvantaged - Y" + in /Not\s*Eligible/i + "Economically Disadvantaged - N" + else + "Unknown" + end + end + def value_from(pattern:) output = nil matches = headers.select do |header| @@ -106,10 +138,12 @@ class SurveyItemValues def to_a copy_likert_scores_from_variant_survey_items + row["Income"] = income + row["Raw Income"] = raw_income headers.select(&:present?) - .reject { |key, _value| key.start_with? "Q" } - .reject { |key, _value| key.end_with? "-1" } - .map { |header| row[header] } + .reject { |key, _value| key.start_with? "Q" } + .reject { |key, _value| key.end_with? "-1" } + .map { |header| row[header] } end def duration @@ -122,17 +156,17 @@ class SurveyItemValues def respondent_type return :teacher if headers - .filter(&:present?) - .filter { |header| header.start_with? "t-" }.count > 0 + .filter(&:present?) + .filter { |header| header.start_with? "t-" }.count > 0 :student end def survey_type survey_item_ids = headers - .filter(&:present?) - .reject { |header| header.end_with?("-1") } - .filter { |header| header.start_with?("t-", "s-") } + .filter(&:present?) + .reject { |header| header.end_with?("-1") } + .filter { |header| header.start_with?("t-", "s-") } SurveyItem.survey_type(survey_item_ids:) end @@ -200,3 +234,4 @@ class SurveyItemValues end end end + diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index d7946ce0..5d063764 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -7,12 +7,13 @@ class SurveyResponsesDataLoader headers_array = CSV.parse(headers).first genders = Gender.gender_hash schools = School.school_hash + incomes = Income.by_designation all_survey_items = survey_items(headers:) file.lazy.each_slice(500) do |lines| survey_item_responses = CSV.parse(lines.join, headers:).map do |row| process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:), - rules:) + rules:, incomes:) end SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500 end @@ -24,6 +25,7 @@ class SurveyResponsesDataLoader headers_array = CSV.parse(headers).first genders = Gender.gender_hash schools = School.school_hash + incomes = Income.by_designation all_survey_items = survey_items(headers:) survey_item_responses = [] @@ -34,7 +36,7 @@ class SurveyResponsesDataLoader CSV.parse(line, headers:).map do |row| survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:), - rules:) + rules:, incomes:) end row_count += 1 @@ -50,7 +52,7 @@ class SurveyResponsesDataLoader private - def self.process_row(row:, rules:) + def self.process_row(row:, rules:, incomes:) return unless row.dese_id? return unless row.school.present? @@ -58,10 +60,10 @@ class SurveyResponsesDataLoader return if rule.new(row:).skip_row? end - process_survey_items(row:) + process_survey_items(row:, incomes:) end - def self.process_survey_items(row:) + def self.process_survey_items(row:, incomes:) row.survey_items.map do |survey_item| likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next @@ -70,19 +72,20 @@ class SurveyResponsesDataLoader next end response = row.survey_item_response(survey_item:) - create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:) + create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, incomes:) end.compact end - def self.create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:) + def self.create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, incomes:) gender = row.gender grade = row.grade + income = incomes[row.income] if survey_item_response.present? - survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date) + survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:) [] else SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:, - likert_score:, grade:, gender:, recorded_date: row.recorded_date) + likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:) end end diff --git a/app/views/analyze/_checkboxes.html.erb b/app/views/analyze/_checkboxes.html.erb new file mode 100644 index 00000000..d64d7e6b --- /dev/null +++ b/app/views/analyze/_checkboxes.html.erb @@ -0,0 +1,17 @@ +
+ + <%= @presenter.graph.slug == 'students-and-teachers' || @presenter.source.slug == 'all-data' ? "disabled" : "" %> + <%= @presenter.group.slug == name ? "" : "hidden" %>> + + +
diff --git a/app/views/analyze/_data_filters.html.erb b/app/views/analyze/_data_filters.html.erb index e7011a36..0cba336f 100644 --- a/app/views/analyze/_data_filters.html.erb +++ b/app/views/analyze/_data_filters.html.erb @@ -1,7 +1,7 @@

Data Filters

- <% @sources.each do |source| %> + <% @presenter.sources.each do |source| %> > + <%= source.slug == @presenter.source.slug ? "checked" : "" %>> <% source.slices.each do | slice | %> @@ -20,7 +20,7 @@ name="slice" value="<%= base_url %>" data-action="click->analyze#refresh" - <%= slice.slug == @slice.slug ? "checked" : "" %> + <%= slice.slug == @presenter.slice.slug ? "checked" : "" %> <%= slice.slug == "all-data" ? "hidden" : "" %>>