Add disaggregation by ELL

This commit is contained in:
rebuilt 2023-08-30 15:18:38 -07:00
parent 8d33095a48
commit 060d7aa55a
41 changed files with 707 additions and 376 deletions

View file

@ -28,7 +28,9 @@ export default class extends Controller {
"&incomes=" +
this.selected_items("income").join(",") +
"&grades=" +
this.selected_items("grade").join(",");
this.selected_items("grade").join(",") +
"&ells=" +
this.selected_items("ell").join(",");
this.go_to(url);
}
@ -126,7 +128,8 @@ export default class extends Controller {
['gender', 'students-by-gender'],
['grade', 'students-by-grade'],
['income', 'students-by-income'],
['race', 'students-by-race']
['race', 'students-by-race'],
['ell', 'students-by-ell'],
])
if (target.name === 'slice' || target.name === 'group') {

7
app/models/ell.rb Normal file
View file

@ -0,0 +1,7 @@
class Ell < ApplicationRecord
scope :by_designation, -> { all.map { |ell| [ell.designation, ell] }.to_h }
include FriendlyId
friendly_id :designation, use: [:slugged]
end

View file

@ -1,5 +1,5 @@
class Gender < ApplicationRecord
scope :gender_hash, lambda {
scope :by_qualtrics_code, lambda {
all.map { |gender| [gender.qualtrics_code, gender] }.to_h
}
end

View file

@ -1,5 +1,6 @@
class Income < ApplicationRecord
scope :by_designation, -> { all.map { |income| [income.designation, income] }.to_h }
scope :by_slug, -> { all.map { |income| [income.slug, income] }.to_h }
include FriendlyId

View file

@ -10,6 +10,7 @@ class SurveyItemResponse < ActiveRecord::Base
belongs_to :student, foreign_key: :student_id, optional: true
belongs_to :gender
belongs_to :income
belongs_to :ell
has_one :measure, through: :survey_item
@ -32,6 +33,11 @@ class SurveyItemResponse < ActiveRecord::Base
academic_year:, income:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score)
}
scope :averages_for_ell, lambda { |survey_items, school, academic_year, ell|
SurveyItemResponse.where(survey_item: survey_items, school:,
academic_year:, ell:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score)
}
scope :averages_for_race, lambda { |school, academic_year, race|
SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where(
school:, academic_year:, grade: school.grades(academic_year:)

View file

@ -0,0 +1,33 @@
# frozen_string_literal: true
module Analyze
module Graph
module Column
module EllColumn
class Ell < GroupedBarColumnPresenter
include Analyze::Graph::Column::EllColumn::ScoreForEll
include Analyze::Graph::Column::EllColumn::EllCount
def label
%w[ELL]
end
def basis
"student"
end
def show_irrelevancy_message?
false
end
def show_insufficient_data_message?
false
end
def ell
::Ell.find_by_slug "ell"
end
end
end
end
end
end

View file

@ -0,0 +1,18 @@
module Analyze
module Graph
module Column
module EllColumn
module EllCount
def type
:student
end
def n_size(year_index)
SurveyItemResponse.where(ell:, survey_item: measure.student_survey_items, school:, grade: grades(year_index),
academic_year: academic_years[year_index]).select(:response_id).distinct.count
end
end
end
end
end
end

View file

@ -0,0 +1,33 @@
# frozen_string_literal: true
module Analyze
module Graph
module Column
module EllColumn
class NotEll < GroupedBarColumnPresenter
include Analyze::Graph::Column::EllColumn::ScoreForEll
include Analyze::Graph::Column::EllColumn::EllCount
def label
%w[Not-ELL]
end
def basis
"student"
end
def show_irrelevancy_message?
false
end
def show_insufficient_data_message?
false
end
def ell
::Ell.find_by_slug "not-ell"
end
end
end
end
end
end

View file

@ -0,0 +1,42 @@
module Analyze
module Graph
module Column
module EllColumn
module ScoreForEll
def score(year_index)
academic_year = academic_years[year_index]
meets_student_threshold = sufficient_student_responses?(academic_year:)
return Score::NIL_SCORE unless meets_student_threshold
averages = SurveyItemResponse.averages_for_ell(measure.student_survey_items, school, academic_year,
ell)
average = bubble_up_averages(averages:).round(2)
Score.new(average:,
meets_teacher_threshold: false,
meets_student_threshold:,
meets_admin_data_threshold: false)
end
def bubble_up_averages(averages:)
measure.student_scales.map do |scale|
scale.survey_items.map do |survey_item|
averages[survey_item]
end.remove_blanks.average
end.remove_blanks.average
end
def sufficient_student_responses?(academic_year:)
return false unless measure.subcategory.response_rate(school:, academic_year:).meets_student_threshold?
yearly_counts = SurveyItemResponse.where(school:, academic_year:,
ell:, survey_item: measure.student_survey_items).group(:ell).select(:response_id).distinct(:response_id).count
yearly_counts.any? do |count|
count[1] >= 10
end
end
end
end
end
end
end

View file

@ -0,0 +1,33 @@
# frozen_string_literal: true
module Analyze
module Graph
module Column
module EllColumn
class Unknown < GroupedBarColumnPresenter
include Analyze::Graph::Column::EllColumn::ScoreForEll
include Analyze::Graph::Column::EllColumn::EllCount
def label
%w[Unknown]
end
def basis
"student"
end
def show_irrelevancy_message?
false
end
def show_insufficient_data_message?
false
end
def ell
::Ell.find_by_slug "unknown"
end
end
end
end
end
end

View file

@ -0,0 +1,44 @@
# frozen_string_literal: true
#
module Analyze
module Graph
class StudentsByEll
include Analyze::Graph::Column::GenderColumn
attr_reader :ells
def initialize(ells:)
@ells = ells
end
def to_s
"Students by Ell"
end
def slug
"students-by-ell"
end
def columns
[].tap do |array|
ells.each do |ell|
array << column_for_ell_code(code: ell.slug)
end
array.sort_by!(&:to_s)
array << Analyze::Graph::Column::AllStudent
end
end
private
def column_for_ell_code(code:)
CFR[code]
end
CFR = {
"ell" => Analyze::Graph::Column::EllColumn::Ell,
"not-ell" => Analyze::Graph::Column::EllColumn::NotEll,
"unknown" => Analyze::Graph::Column::EllColumn::Unknown
}.freeze
end
end
end

View file

@ -1,3 +1,5 @@
# frozen_string_literal: true
module Analyze
module Graph
class StudentsByGender

View file

@ -1,3 +1,5 @@
# frozen_string_literal: true
module Analyze
module Graph
class StudentsByGrade
@ -9,11 +11,11 @@ module Analyze
end
def to_s
'Students by Grade'
"Students by Grade"
end
def slug
'students-by-grade'
"students-by-grade"
end
def columns

View file

@ -1,3 +1,5 @@
# frozen_string_literal: true
module Analyze
module Graph
class StudentsByIncome

View file

@ -1,3 +1,5 @@
# frozen_string_literal: true
module Analyze
module Graph
class StudentsByRace
@ -8,11 +10,11 @@ module Analyze
end
def to_s
'Students by Race'
"Students by Race"
end
def slug
'students-by-race'
"students-by-race"
end
def columns
@ -31,14 +33,14 @@ module Analyze
end
CFR = {
'1' => Analyze::Graph::Column::RaceColumn::AmericanIndian,
'2' => Analyze::Graph::Column::RaceColumn::Asian,
'3' => Analyze::Graph::Column::RaceColumn::Black,
'4' => Analyze::Graph::Column::RaceColumn::Hispanic,
'5' => Analyze::Graph::Column::RaceColumn::White,
'8' => Analyze::Graph::Column::RaceColumn::MiddleEastern,
'99' => Analyze::Graph::Column::RaceColumn::Unknown,
'100' => Analyze::Graph::Column::RaceColumn::Multiracial
"1" => Analyze::Graph::Column::RaceColumn::AmericanIndian,
"2" => Analyze::Graph::Column::RaceColumn::Asian,
"3" => Analyze::Graph::Column::RaceColumn::Black,
"4" => Analyze::Graph::Column::RaceColumn::Hispanic,
"5" => Analyze::Graph::Column::RaceColumn::White,
"8" => Analyze::Graph::Column::RaceColumn::MiddleEastern,
"99" => Analyze::Graph::Column::RaceColumn::Unknown,
"100" => Analyze::Graph::Column::RaceColumn::Multiracial
}.freeze
end
end

View file

@ -0,0 +1,13 @@
module Analyze
module Group
class Ell
def name
"ELL"
end
def slug
"ell"
end
end
end
end

View file

@ -54,9 +54,27 @@ module Analyze
end
end
def ells
@ells ||= Ell.all.order(slug: :ASC)
end
def selected_ells
@selected_ells ||= begin
ell_params = params[:ells]
return ells unless ell_params
ell_params.split(",").map { |ell| Ell.find_by_slug ell }.compact
end
end
def graphs
@graphs ||= [Analyze::Graph::AllData.new, Analyze::Graph::StudentsAndTeachers.new, Analyze::Graph::StudentsByRace.new(races: selected_races),
Analyze::Graph::StudentsByGrade.new(grades: selected_grades), Analyze::Graph::StudentsByGender.new(genders: selected_genders), Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes)]
@graphs ||= [Analyze::Graph::AllData.new,
Analyze::Graph::StudentsAndTeachers.new,
Analyze::Graph::StudentsByRace.new(races: selected_races),
Analyze::Graph::StudentsByGrade.new(grades: selected_grades),
Analyze::Graph::StudentsByGender.new(genders: selected_genders),
Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes),
Analyze::Graph::StudentsByEll.new(ells: selected_ells)]
end
def graph
@ -88,7 +106,7 @@ module Analyze
end
def groups
@groups = [Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new,
@groups = [Analyze::Group::Ell.new, Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new,
Analyze::Group::Race.new]
end

View file

@ -1,12 +1,11 @@
require "fileutils"
class Cleaner
attr_reader :input_filepath, :output_filepath, :log_filepath, :disaggregation_filepath
attr_reader :input_filepath, :output_filepath, :log_filepath
def initialize(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:)
def initialize(input_filepath:, output_filepath:, log_filepath:)
@input_filepath = input_filepath
@output_filepath = output_filepath
@log_filepath = log_filepath
@disaggregation_filepath = disaggregation_filepath
initialize_directories
end
@ -14,7 +13,7 @@ class Cleaner
Dir.glob(Rails.root.join(input_filepath, "*.csv")).each do |filepath|
puts filepath
File.open(filepath) do |file|
processed_data = process_raw_file(file:, disaggregation_data:)
processed_data = process_raw_file(file:)
processed_data in [headers, clean_csv, log_csv, data]
return if data.empty?
@ -25,10 +24,6 @@ class Cleaner
end
end
def disaggregation_data
@disaggregation_data ||= DisaggregationLoader.new(path: disaggregation_filepath).load
end
def filename(headers:, data:)
survey_item_ids = headers.filter(&:present?).filter do |header|
header.start_with?("s-", "t-")
@ -43,17 +38,16 @@ class Cleaner
districts.join(".").to_s + "." + survey_type.to_s + "." + range + ".csv"
end
def process_raw_file(file:, disaggregation_data:)
def process_raw_file(file:)
clean_csv = []
log_csv = []
data = []
headers = (CSV.parse(file.first).first << "Raw Income") << "Income"
headers = CSV.parse(file.first).first.push("Raw Income").push("Income").push("Raw ELL").push("ELL")
filtered_headers = include_all_headers(headers:)
filtered_headers = remove_unwanted_headers(headers: filtered_headers)
log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?",
"Valid Standard Deviation?"]).flatten
clean_csv << filtered_headers
log_csv << log_headers
@ -62,7 +56,7 @@ class Cleaner
file.lazy.each_slice(1000) do |lines|
CSV.parse(lines.join, headers:).map do |row|
values = SurveyItemValues.new(row:, headers:, genders:,
survey_items: all_survey_items, schools:, disaggregation_data:)
survey_items: all_survey_items, schools:)
next unless values.valid_school?
data << values
@ -109,7 +103,7 @@ class Cleaner
end
def genders
@genders ||= Gender.gender_hash
@genders ||= Gender.by_qualtrics_code
end
def survey_items(headers:)

View file

@ -6,12 +6,13 @@ class DemographicLoader
process_race(row:)
process_gender(row:)
process_income(row:)
process_ell(row:)
end
end
def self.process_race(row:)
qualtrics_code = row['Race Qualtrics Code'].to_i
designation = row['Race/Ethnicity']
qualtrics_code = row["Race Qualtrics Code"].to_i
designation = row["Race/Ethnicity"]
return unless qualtrics_code && designation
if qualtrics_code.between?(6, 7)
@ -22,8 +23,8 @@ class DemographicLoader
end
def self.process_gender(row:)
qualtrics_code = row['Gender Qualtrics Code'].to_i
designation = row['Sex/Gender']
qualtrics_code = row["Gender Qualtrics Code"].to_i
designation = row["Sex/Gender"]
return unless qualtrics_code && designation
gender = ::Gender.find_or_create_by!(qualtrics_code:, designation:)
@ -31,11 +32,18 @@ class DemographicLoader
end
def self.process_income(row:)
designation = row['Income']
designation = row["Income"]
return unless designation
Income.find_or_create_by!(designation:)
end
def self.process_ell(row:)
designation = row["ELL"]
return unless designation
Ell.find_or_create_by!(designation:)
end
end
class KnownRace
@ -50,7 +58,7 @@ end
class UnknownRace
def initialize(qualtrics_code:, designation:)
unknown = Race.find_or_create_by!(qualtrics_code: 99)
unknown.designation = 'Race/Ethnicity Not Listed'
unknown.designation = "Race/Ethnicity Not Listed"
unknown.slug = designation.parameterize
unknown.save
end

View file

@ -14,7 +14,7 @@ class DisaggregationRow
@academic_year ||= value_from(pattern: /Academic\s*Year/i)
end
def income
def raw_income
@income ||= value_from(pattern: /Low\s*Income/i)
end
@ -22,6 +22,25 @@ class DisaggregationRow
@lasid ||= value_from(pattern: /LASID/i)
end
def raw_ell
@raw_ell ||= value_from(pattern: /EL Student First Year/i)
end
def ell
@ell ||= begin
value = value_from(pattern: /EL Student First Year/i).downcase
case value
when /lep student 1st year|LEP student not 1st year/i
"ELL"
when /Does not apply/i
"Not ELL"
else
"Unknown"
end
end
end
def value_from(pattern:)
output = nil
matches = headers.select do |header|

View file

@ -1,7 +1,7 @@
class SurveyItemValues
attr_reader :row, :headers, :genders, :survey_items, :schools, :disaggregation_data
attr_reader :row, :headers, :genders, :survey_items, :schools
def initialize(row:, headers:, genders:, survey_items:, schools:, disaggregation_data: nil)
def initialize(row:, headers:, genders:, survey_items:, schools:)
@row = row
# Remove any newlines in headers
headers = headers.map { |item| item.delete("\n") if item.present? }
@ -9,11 +9,12 @@ class SurveyItemValues
@genders = genders
@survey_items = survey_items
@schools = schools
@disaggregation_data = disaggregation_data
copy_likert_scores_from_variant_survey_items
row["Income"] = income
row["Raw Income"] = raw_income
row["Raw ELL"] = raw_ell
row["ELL"] = ell
copy_data_to_main_column(main: /Race/i, secondary: /Race Secondary|Race-1/i)
copy_data_to_main_column(main: /Gender/i, secondary: /Gender Secondary|Gender-1/i)
@ -134,20 +135,9 @@ class SurveyItemValues
def raw_income
@raw_income ||= value_from(pattern: /Low\s*Income|Raw\s*Income/i)
return @raw_income if @raw_income.present?
return "Unknown" unless disaggregation_data.present?
disaggregation = disaggregation_data[[lasid, district.name, academic_year.range]]
return "Unknown" unless disaggregation.present?
@raw_income ||= disaggregation.income
end
def income
@income ||= value_from(pattern: /^Income$/i)
return @income if @income.present?
@income ||= case raw_income
in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income/i
"Economically Disadvantaged - Y"
@ -158,6 +148,21 @@ class SurveyItemValues
end
end
def raw_ell
@raw_ell ||= value_from(pattern: /EL Student First Year|Raw\s*ELL/i)
end
def ell
@ell ||= case raw_ell
in /lep student 1st year|LEP student not 1st year|EL Student First Year/i
"ELL"
in /Does not apply/i
"Not ELL"
else
"Unknown"
end
end
def value_from(pattern:)
output = nil
matches = headers.select do |header|

View file

@ -1,31 +1,25 @@
# frozen_string_literal: true
class SurveyResponsesDataLoader
def self.load_data(filepath:, rules: [Rule::NoRule])
def load_data(filepath:, rules: [Rule::NoRule])
File.open(filepath) do |file|
headers = file.first
headers_array = CSV.parse(headers).first
genders = Gender.gender_hash
schools = School.school_hash
incomes = Income.by_designation
all_survey_items = survey_items(headers:)
file.lazy.each_slice(500) do |lines|
survey_item_responses = CSV.parse(lines.join, headers:).map do |row|
process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:),
rules:, incomes:)
rules:)
end
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500
end
end
end
def self.from_file(file:, rules: [])
def from_file(file:, rules: [])
headers = file.gets
headers_array = CSV.parse(headers).first
genders = Gender.gender_hash
schools = School.school_hash
incomes = Income.by_designation
all_survey_items = survey_items(headers:)
survey_item_responses = []
@ -36,7 +30,7 @@ class SurveyResponsesDataLoader
CSV.parse(line, headers:).map do |row|
survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:),
rules:, incomes:)
rules:)
end
row_count += 1
@ -52,7 +46,23 @@ class SurveyResponsesDataLoader
private
def self.process_row(row:, rules:, incomes:)
def schools
@schools = School.school_hash
end
def genders
@genders = Gender.by_qualtrics_code
end
def incomes
@incomes ||= Income.by_slug
end
def ells
@ells ||= Ell.by_designation
end
def process_row(row:, rules:)
return unless row.dese_id?
return unless row.school.present?
@ -60,10 +70,10 @@ class SurveyResponsesDataLoader
return if rule.new(row:).skip_row?
end
process_survey_items(row:, incomes:)
process_survey_items(row:)
end
def self.process_survey_items(row:, incomes:)
def process_survey_items(row:)
row.survey_items.map do |survey_item|
likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next
@ -72,38 +82,33 @@ class SurveyResponsesDataLoader
next
end
response = row.survey_item_response(survey_item:)
create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, incomes:)
create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:)
end.compact
end
def self.create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, incomes:)
def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:)
gender = row.gender
grade = row.grade
income = incomes[row.income]
income = incomes[row.income.parameterize]
ell = ells[row.ell]
if survey_item_response.present?
survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:)
survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:)
[]
else
SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:,
likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:)
likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:)
end
end
def self.survey_items(headers:)
def survey_items(headers:)
SurveyItem.where(survey_item_id: get_survey_item_ids_from_headers(headers:))
end
def self.get_survey_item_ids_from_headers(headers:)
def get_survey_item_ids_from_headers(headers:)
CSV.parse(headers).first
.filter(&:present?)
.filter { |header| header.start_with? "t-", "s-" }
end
private_class_method :process_row
private_class_method :process_survey_items
private_class_method :create_or_update_response
private_class_method :survey_items
private_class_method :get_survey_item_ids_from_headers
end
module StringMonkeyPatches

View file

@ -21,3 +21,7 @@
<% @presenter.incomes.each do |income| %>
<%= render(partial: "checkboxes", locals: {id: "income-#{income.slug}", item: income, selected_items: @presenter.selected_incomes, name: "income", label_text: income.label}) %>
<% end %>
<% @presenter.ells.each do |ell| %>
<%= render(partial: "checkboxes", locals: {id: "ell-#{ell.slug}", item: ell, selected_items: @presenter.selected_ells, name: "ell", label_text: ell.designation}) %>
<% end %>

View file

@ -13,7 +13,7 @@
<%= render partial: "school_years", locals: {available_academic_years: @presenter.academic_years, selected_academic_years: @presenter.selected_academic_years, district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory, measures: @presenter.measures, graph: @presenter.graph} %>
<%= render partial: "data_filters", locals: {district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory} %>
</div>
<% cache [@presenter.subcategory, @school, @presenter.selected_academic_years, @presenter.graph, @presenter.selected_races, @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders] do %>
<% cache [@presenter.subcategory, @school, @presenter.selected_academic_years, @presenter.graph, @presenter.selected_races, @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders, @presenter.selected_ells, @presenter.ells] do %>
<div class="bg-color-white flex-grow-1 col-9">
<% @presenter.measures.each do |measure| %>
<section class="mb-6">