diff --git a/app/javascript/controllers/analyze_controller.js b/app/javascript/controllers/analyze_controller.js index bf290c9b..bef3ccae 100644 --- a/app/javascript/controllers/analyze_controller.js +++ b/app/javascript/controllers/analyze_controller.js @@ -28,7 +28,9 @@ export default class extends Controller { "&incomes=" + this.selected_items("income").join(",") + "&grades=" + - this.selected_items("grade").join(","); + this.selected_items("grade").join(",") + + "&ells=" + + this.selected_items("ell").join(","); this.go_to(url); } @@ -126,7 +128,8 @@ export default class extends Controller { ['gender', 'students-by-gender'], ['grade', 'students-by-grade'], ['income', 'students-by-income'], - ['race', 'students-by-race'] + ['race', 'students-by-race'], + ['ell', 'students-by-ell'], ]) if (target.name === 'slice' || target.name === 'group') { diff --git a/app/models/ell.rb b/app/models/ell.rb new file mode 100644 index 00000000..6b121fb1 --- /dev/null +++ b/app/models/ell.rb @@ -0,0 +1,7 @@ +class Ell < ApplicationRecord + scope :by_designation, -> { all.map { |ell| [ell.designation, ell] }.to_h } + + include FriendlyId + + friendly_id :designation, use: [:slugged] +end diff --git a/app/models/gender.rb b/app/models/gender.rb index 5bfdc766..50f4d74c 100644 --- a/app/models/gender.rb +++ b/app/models/gender.rb @@ -1,5 +1,5 @@ class Gender < ApplicationRecord - scope :gender_hash, lambda { + scope :by_qualtrics_code, lambda { all.map { |gender| [gender.qualtrics_code, gender] }.to_h } end diff --git a/app/models/income.rb b/app/models/income.rb index f92b73b5..fd6d7845 100644 --- a/app/models/income.rb +++ b/app/models/income.rb @@ -1,5 +1,6 @@ class Income < ApplicationRecord scope :by_designation, -> { all.map { |income| [income.designation, income] }.to_h } + scope :by_slug, -> { all.map { |income| [income.slug, income] }.to_h } include FriendlyId diff --git a/app/models/survey_item_response.rb b/app/models/survey_item_response.rb index 21f05887..c767220f 100644 --- a/app/models/survey_item_response.rb +++ b/app/models/survey_item_response.rb @@ -10,6 +10,7 @@ class SurveyItemResponse < ActiveRecord::Base belongs_to :student, foreign_key: :student_id, optional: true belongs_to :gender belongs_to :income + belongs_to :ell has_one :measure, through: :survey_item @@ -32,6 +33,11 @@ class SurveyItemResponse < ActiveRecord::Base academic_year:, income:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score) } + scope :averages_for_ell, lambda { |survey_items, school, academic_year, ell| + SurveyItemResponse.where(survey_item: survey_items, school:, + academic_year:, ell:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score) + } + scope :averages_for_race, lambda { |school, academic_year, race| SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where( school:, academic_year:, grade: school.grades(academic_year:) diff --git a/app/presenters/analyze/graph/column/ell_column/ell.rb b/app/presenters/analyze/graph/column/ell_column/ell.rb new file mode 100644 index 00000000..cba07408 --- /dev/null +++ b/app/presenters/analyze/graph/column/ell_column/ell.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Analyze + module Graph + module Column + module EllColumn + class Ell < GroupedBarColumnPresenter + include Analyze::Graph::Column::EllColumn::ScoreForEll + include Analyze::Graph::Column::EllColumn::EllCount + def label + %w[ELL] + end + + def basis + "student" + end + + def show_irrelevancy_message? + false + end + + def show_insufficient_data_message? + false + end + + def ell + ::Ell.find_by_slug "ell" + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/column/ell_column/ell_count.rb b/app/presenters/analyze/graph/column/ell_column/ell_count.rb new file mode 100644 index 00000000..fa20518a --- /dev/null +++ b/app/presenters/analyze/graph/column/ell_column/ell_count.rb @@ -0,0 +1,18 @@ +module Analyze + module Graph + module Column + module EllColumn + module EllCount + def type + :student + end + + def n_size(year_index) + SurveyItemResponse.where(ell:, survey_item: measure.student_survey_items, school:, grade: grades(year_index), + academic_year: academic_years[year_index]).select(:response_id).distinct.count + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/column/ell_column/not_ell.rb b/app/presenters/analyze/graph/column/ell_column/not_ell.rb new file mode 100644 index 00000000..ba93d5f7 --- /dev/null +++ b/app/presenters/analyze/graph/column/ell_column/not_ell.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Analyze + module Graph + module Column + module EllColumn + class NotEll < GroupedBarColumnPresenter + include Analyze::Graph::Column::EllColumn::ScoreForEll + include Analyze::Graph::Column::EllColumn::EllCount + def label + %w[Not-ELL] + end + + def basis + "student" + end + + def show_irrelevancy_message? + false + end + + def show_insufficient_data_message? + false + end + + def ell + ::Ell.find_by_slug "not-ell" + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/column/ell_column/score_for_ell.rb b/app/presenters/analyze/graph/column/ell_column/score_for_ell.rb new file mode 100644 index 00000000..cd39cd0f --- /dev/null +++ b/app/presenters/analyze/graph/column/ell_column/score_for_ell.rb @@ -0,0 +1,42 @@ +module Analyze + module Graph + module Column + module EllColumn + module ScoreForEll + def score(year_index) + academic_year = academic_years[year_index] + meets_student_threshold = sufficient_student_responses?(academic_year:) + return Score::NIL_SCORE unless meets_student_threshold + + averages = SurveyItemResponse.averages_for_ell(measure.student_survey_items, school, academic_year, + ell) + average = bubble_up_averages(averages:).round(2) + + Score.new(average:, + meets_teacher_threshold: false, + meets_student_threshold:, + meets_admin_data_threshold: false) + end + + def bubble_up_averages(averages:) + measure.student_scales.map do |scale| + scale.survey_items.map do |survey_item| + averages[survey_item] + end.remove_blanks.average + end.remove_blanks.average + end + + def sufficient_student_responses?(academic_year:) + return false unless measure.subcategory.response_rate(school:, academic_year:).meets_student_threshold? + + yearly_counts = SurveyItemResponse.where(school:, academic_year:, + ell:, survey_item: measure.student_survey_items).group(:ell).select(:response_id).distinct(:response_id).count + yearly_counts.any? do |count| + count[1] >= 10 + end + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/column/ell_column/unknown.rb b/app/presenters/analyze/graph/column/ell_column/unknown.rb new file mode 100644 index 00000000..37f077ae --- /dev/null +++ b/app/presenters/analyze/graph/column/ell_column/unknown.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Analyze + module Graph + module Column + module EllColumn + class Unknown < GroupedBarColumnPresenter + include Analyze::Graph::Column::EllColumn::ScoreForEll + include Analyze::Graph::Column::EllColumn::EllCount + def label + %w[Unknown] + end + + def basis + "student" + end + + def show_irrelevancy_message? + false + end + + def show_insufficient_data_message? + false + end + + def ell + ::Ell.find_by_slug "unknown" + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/students_by_ell.rb b/app/presenters/analyze/graph/students_by_ell.rb new file mode 100644 index 00000000..ff96435b --- /dev/null +++ b/app/presenters/analyze/graph/students_by_ell.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +# +module Analyze + module Graph + class StudentsByEll + include Analyze::Graph::Column::GenderColumn + attr_reader :ells + + def initialize(ells:) + @ells = ells + end + + def to_s + "Students by Ell" + end + + def slug + "students-by-ell" + end + + def columns + [].tap do |array| + ells.each do |ell| + array << column_for_ell_code(code: ell.slug) + end + array.sort_by!(&:to_s) + array << Analyze::Graph::Column::AllStudent + end + end + + private + + def column_for_ell_code(code:) + CFR[code] + end + + CFR = { + "ell" => Analyze::Graph::Column::EllColumn::Ell, + "not-ell" => Analyze::Graph::Column::EllColumn::NotEll, + "unknown" => Analyze::Graph::Column::EllColumn::Unknown + }.freeze + end + end +end diff --git a/app/presenters/analyze/graph/students_by_gender.rb b/app/presenters/analyze/graph/students_by_gender.rb index 8e3fb50a..97d33634 100644 --- a/app/presenters/analyze/graph/students_by_gender.rb +++ b/app/presenters/analyze/graph/students_by_gender.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Analyze module Graph class StudentsByGender diff --git a/app/presenters/analyze/graph/students_by_grade.rb b/app/presenters/analyze/graph/students_by_grade.rb index 267a9037..3fb113d4 100644 --- a/app/presenters/analyze/graph/students_by_grade.rb +++ b/app/presenters/analyze/graph/students_by_grade.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Analyze module Graph class StudentsByGrade @@ -9,11 +11,11 @@ module Analyze end def to_s - 'Students by Grade' + "Students by Grade" end def slug - 'students-by-grade' + "students-by-grade" end def columns diff --git a/app/presenters/analyze/graph/students_by_income.rb b/app/presenters/analyze/graph/students_by_income.rb index 148b22c1..e72b6efb 100644 --- a/app/presenters/analyze/graph/students_by_income.rb +++ b/app/presenters/analyze/graph/students_by_income.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Analyze module Graph class StudentsByIncome diff --git a/app/presenters/analyze/graph/students_by_race.rb b/app/presenters/analyze/graph/students_by_race.rb index 0c4009ae..74a7cb8f 100644 --- a/app/presenters/analyze/graph/students_by_race.rb +++ b/app/presenters/analyze/graph/students_by_race.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Analyze module Graph class StudentsByRace @@ -8,11 +10,11 @@ module Analyze end def to_s - 'Students by Race' + "Students by Race" end def slug - 'students-by-race' + "students-by-race" end def columns @@ -31,14 +33,14 @@ module Analyze end CFR = { - '1' => Analyze::Graph::Column::RaceColumn::AmericanIndian, - '2' => Analyze::Graph::Column::RaceColumn::Asian, - '3' => Analyze::Graph::Column::RaceColumn::Black, - '4' => Analyze::Graph::Column::RaceColumn::Hispanic, - '5' => Analyze::Graph::Column::RaceColumn::White, - '8' => Analyze::Graph::Column::RaceColumn::MiddleEastern, - '99' => Analyze::Graph::Column::RaceColumn::Unknown, - '100' => Analyze::Graph::Column::RaceColumn::Multiracial + "1" => Analyze::Graph::Column::RaceColumn::AmericanIndian, + "2" => Analyze::Graph::Column::RaceColumn::Asian, + "3" => Analyze::Graph::Column::RaceColumn::Black, + "4" => Analyze::Graph::Column::RaceColumn::Hispanic, + "5" => Analyze::Graph::Column::RaceColumn::White, + "8" => Analyze::Graph::Column::RaceColumn::MiddleEastern, + "99" => Analyze::Graph::Column::RaceColumn::Unknown, + "100" => Analyze::Graph::Column::RaceColumn::Multiracial }.freeze end end diff --git a/app/presenters/analyze/group/ell.rb b/app/presenters/analyze/group/ell.rb new file mode 100644 index 00000000..512b3d5b --- /dev/null +++ b/app/presenters/analyze/group/ell.rb @@ -0,0 +1,13 @@ +module Analyze + module Group + class Ell + def name + "ELL" + end + + def slug + "ell" + end + end + end +end diff --git a/app/presenters/analyze/presenter.rb b/app/presenters/analyze/presenter.rb index 391b69a1..5118e1cd 100644 --- a/app/presenters/analyze/presenter.rb +++ b/app/presenters/analyze/presenter.rb @@ -54,9 +54,27 @@ module Analyze end end + def ells + @ells ||= Ell.all.order(slug: :ASC) + end + + def selected_ells + @selected_ells ||= begin + ell_params = params[:ells] + return ells unless ell_params + + ell_params.split(",").map { |ell| Ell.find_by_slug ell }.compact + end + end + def graphs - @graphs ||= [Analyze::Graph::AllData.new, Analyze::Graph::StudentsAndTeachers.new, Analyze::Graph::StudentsByRace.new(races: selected_races), - Analyze::Graph::StudentsByGrade.new(grades: selected_grades), Analyze::Graph::StudentsByGender.new(genders: selected_genders), Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes)] + @graphs ||= [Analyze::Graph::AllData.new, + Analyze::Graph::StudentsAndTeachers.new, + Analyze::Graph::StudentsByRace.new(races: selected_races), + Analyze::Graph::StudentsByGrade.new(grades: selected_grades), + Analyze::Graph::StudentsByGender.new(genders: selected_genders), + Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes), + Analyze::Graph::StudentsByEll.new(ells: selected_ells)] end def graph @@ -88,7 +106,7 @@ module Analyze end def groups - @groups = [Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new, + @groups = [Analyze::Group::Ell.new, Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new, Analyze::Group::Race.new] end diff --git a/app/services/cleaner.rb b/app/services/cleaner.rb index 307c177e..f4eba993 100644 --- a/app/services/cleaner.rb +++ b/app/services/cleaner.rb @@ -1,12 +1,11 @@ require "fileutils" class Cleaner - attr_reader :input_filepath, :output_filepath, :log_filepath, :disaggregation_filepath + attr_reader :input_filepath, :output_filepath, :log_filepath - def initialize(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:) + def initialize(input_filepath:, output_filepath:, log_filepath:) @input_filepath = input_filepath @output_filepath = output_filepath @log_filepath = log_filepath - @disaggregation_filepath = disaggregation_filepath initialize_directories end @@ -14,7 +13,7 @@ class Cleaner Dir.glob(Rails.root.join(input_filepath, "*.csv")).each do |filepath| puts filepath File.open(filepath) do |file| - processed_data = process_raw_file(file:, disaggregation_data:) + processed_data = process_raw_file(file:) processed_data in [headers, clean_csv, log_csv, data] return if data.empty? @@ -25,10 +24,6 @@ class Cleaner end end - def disaggregation_data - @disaggregation_data ||= DisaggregationLoader.new(path: disaggregation_filepath).load - end - def filename(headers:, data:) survey_item_ids = headers.filter(&:present?).filter do |header| header.start_with?("s-", "t-") @@ -43,17 +38,16 @@ class Cleaner districts.join(".").to_s + "." + survey_type.to_s + "." + range + ".csv" end - def process_raw_file(file:, disaggregation_data:) + def process_raw_file(file:) clean_csv = [] log_csv = [] data = [] - headers = (CSV.parse(file.first).first << "Raw Income") << "Income" + headers = CSV.parse(file.first).first.push("Raw Income").push("Income").push("Raw ELL").push("ELL") filtered_headers = include_all_headers(headers:) filtered_headers = remove_unwanted_headers(headers: filtered_headers) log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?", "Valid Standard Deviation?"]).flatten - clean_csv << filtered_headers log_csv << log_headers @@ -62,7 +56,7 @@ class Cleaner file.lazy.each_slice(1000) do |lines| CSV.parse(lines.join, headers:).map do |row| values = SurveyItemValues.new(row:, headers:, genders:, - survey_items: all_survey_items, schools:, disaggregation_data:) + survey_items: all_survey_items, schools:) next unless values.valid_school? data << values @@ -109,7 +103,7 @@ class Cleaner end def genders - @genders ||= Gender.gender_hash + @genders ||= Gender.by_qualtrics_code end def survey_items(headers:) diff --git a/app/services/demographic_loader.rb b/app/services/demographic_loader.rb index 345020e8..cb11a51c 100644 --- a/app/services/demographic_loader.rb +++ b/app/services/demographic_loader.rb @@ -6,12 +6,13 @@ class DemographicLoader process_race(row:) process_gender(row:) process_income(row:) + process_ell(row:) end end def self.process_race(row:) - qualtrics_code = row['Race Qualtrics Code'].to_i - designation = row['Race/Ethnicity'] + qualtrics_code = row["Race Qualtrics Code"].to_i + designation = row["Race/Ethnicity"] return unless qualtrics_code && designation if qualtrics_code.between?(6, 7) @@ -22,8 +23,8 @@ class DemographicLoader end def self.process_gender(row:) - qualtrics_code = row['Gender Qualtrics Code'].to_i - designation = row['Sex/Gender'] + qualtrics_code = row["Gender Qualtrics Code"].to_i + designation = row["Sex/Gender"] return unless qualtrics_code && designation gender = ::Gender.find_or_create_by!(qualtrics_code:, designation:) @@ -31,11 +32,18 @@ class DemographicLoader end def self.process_income(row:) - designation = row['Income'] + designation = row["Income"] return unless designation Income.find_or_create_by!(designation:) end + + def self.process_ell(row:) + designation = row["ELL"] + return unless designation + + Ell.find_or_create_by!(designation:) + end end class KnownRace @@ -50,7 +58,7 @@ end class UnknownRace def initialize(qualtrics_code:, designation:) unknown = Race.find_or_create_by!(qualtrics_code: 99) - unknown.designation = 'Race/Ethnicity Not Listed' + unknown.designation = "Race/Ethnicity Not Listed" unknown.slug = designation.parameterize unknown.save end diff --git a/app/services/disaggregation_row.rb b/app/services/disaggregation_row.rb index 0fefe53f..f251f8b2 100644 --- a/app/services/disaggregation_row.rb +++ b/app/services/disaggregation_row.rb @@ -14,7 +14,7 @@ class DisaggregationRow @academic_year ||= value_from(pattern: /Academic\s*Year/i) end - def income + def raw_income @income ||= value_from(pattern: /Low\s*Income/i) end @@ -22,6 +22,25 @@ class DisaggregationRow @lasid ||= value_from(pattern: /LASID/i) end + def raw_ell + @raw_ell ||= value_from(pattern: /EL Student First Year/i) + end + + def ell + @ell ||= begin + value = value_from(pattern: /EL Student First Year/i).downcase + + case value + when /lep student 1st year|LEP student not 1st year/i + "ELL" + when /Does not apply/i + "Not ELL" + else + "Unknown" + end + end + end + def value_from(pattern:) output = nil matches = headers.select do |header| diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index 481df58b..678c3fdc 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -1,7 +1,7 @@ class SurveyItemValues - attr_reader :row, :headers, :genders, :survey_items, :schools, :disaggregation_data + attr_reader :row, :headers, :genders, :survey_items, :schools - def initialize(row:, headers:, genders:, survey_items:, schools:, disaggregation_data: nil) + def initialize(row:, headers:, genders:, survey_items:, schools:) @row = row # Remove any newlines in headers headers = headers.map { |item| item.delete("\n") if item.present? } @@ -9,11 +9,12 @@ class SurveyItemValues @genders = genders @survey_items = survey_items @schools = schools - @disaggregation_data = disaggregation_data copy_likert_scores_from_variant_survey_items row["Income"] = income row["Raw Income"] = raw_income + row["Raw ELL"] = raw_ell + row["ELL"] = ell copy_data_to_main_column(main: /Race/i, secondary: /Race Secondary|Race-1/i) copy_data_to_main_column(main: /Gender/i, secondary: /Gender Secondary|Gender-1/i) @@ -134,20 +135,9 @@ class SurveyItemValues def raw_income @raw_income ||= value_from(pattern: /Low\s*Income|Raw\s*Income/i) - return @raw_income if @raw_income.present? - - return "Unknown" unless disaggregation_data.present? - - disaggregation = disaggregation_data[[lasid, district.name, academic_year.range]] - return "Unknown" unless disaggregation.present? - - @raw_income ||= disaggregation.income end def income - @income ||= value_from(pattern: /^Income$/i) - return @income if @income.present? - @income ||= case raw_income in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income/i "Economically Disadvantaged - Y" @@ -158,6 +148,21 @@ class SurveyItemValues end end + def raw_ell + @raw_ell ||= value_from(pattern: /EL Student First Year|Raw\s*ELL/i) + end + + def ell + @ell ||= case raw_ell + in /lep student 1st year|LEP student not 1st year|EL Student First Year/i + "ELL" + in /Does not apply/i + "Not ELL" + else + "Unknown" + end + end + def value_from(pattern:) output = nil matches = headers.select do |header| diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index 8f5f2a3a..723ceb86 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -1,31 +1,25 @@ # frozen_string_literal: true class SurveyResponsesDataLoader - def self.load_data(filepath:, rules: [Rule::NoRule]) + def load_data(filepath:, rules: [Rule::NoRule]) File.open(filepath) do |file| headers = file.first headers_array = CSV.parse(headers).first - genders = Gender.gender_hash - schools = School.school_hash - incomes = Income.by_designation all_survey_items = survey_items(headers:) file.lazy.each_slice(500) do |lines| survey_item_responses = CSV.parse(lines.join, headers:).map do |row| process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:), - rules:, incomes:) + rules:) end SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500 end end end - def self.from_file(file:, rules: []) + def from_file(file:, rules: []) headers = file.gets headers_array = CSV.parse(headers).first - genders = Gender.gender_hash - schools = School.school_hash - incomes = Income.by_designation all_survey_items = survey_items(headers:) survey_item_responses = [] @@ -36,7 +30,7 @@ class SurveyResponsesDataLoader CSV.parse(line, headers:).map do |row| survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:), - rules:, incomes:) + rules:) end row_count += 1 @@ -52,7 +46,23 @@ class SurveyResponsesDataLoader private - def self.process_row(row:, rules:, incomes:) + def schools + @schools = School.school_hash + end + + def genders + @genders = Gender.by_qualtrics_code + end + + def incomes + @incomes ||= Income.by_slug + end + + def ells + @ells ||= Ell.by_designation + end + + def process_row(row:, rules:) return unless row.dese_id? return unless row.school.present? @@ -60,10 +70,10 @@ class SurveyResponsesDataLoader return if rule.new(row:).skip_row? end - process_survey_items(row:, incomes:) + process_survey_items(row:) end - def self.process_survey_items(row:, incomes:) + def process_survey_items(row:) row.survey_items.map do |survey_item| likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next @@ -72,38 +82,33 @@ class SurveyResponsesDataLoader next end response = row.survey_item_response(survey_item:) - create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, incomes:) + create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:) end.compact end - def self.create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, incomes:) + def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:) gender = row.gender grade = row.grade - income = incomes[row.income] + income = incomes[row.income.parameterize] + ell = ells[row.ell] if survey_item_response.present? - survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:) + survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:) [] else SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:, - likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:) + likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:) end end - def self.survey_items(headers:) + def survey_items(headers:) SurveyItem.where(survey_item_id: get_survey_item_ids_from_headers(headers:)) end - def self.get_survey_item_ids_from_headers(headers:) + def get_survey_item_ids_from_headers(headers:) CSV.parse(headers).first .filter(&:present?) .filter { |header| header.start_with? "t-", "s-" } end - - private_class_method :process_row - private_class_method :process_survey_items - private_class_method :create_or_update_response - private_class_method :survey_items - private_class_method :get_survey_item_ids_from_headers end module StringMonkeyPatches diff --git a/app/views/analyze/_group_selectors.html.erb b/app/views/analyze/_group_selectors.html.erb index 61f21184..00d05062 100644 --- a/app/views/analyze/_group_selectors.html.erb +++ b/app/views/analyze/_group_selectors.html.erb @@ -21,3 +21,7 @@ <% @presenter.incomes.each do |income| %> <%= render(partial: "checkboxes", locals: {id: "income-#{income.slug}", item: income, selected_items: @presenter.selected_incomes, name: "income", label_text: income.label}) %> <% end %> + +<% @presenter.ells.each do |ell| %> + <%= render(partial: "checkboxes", locals: {id: "ell-#{ell.slug}", item: ell, selected_items: @presenter.selected_ells, name: "ell", label_text: ell.designation}) %> +<% end %> diff --git a/app/views/analyze/index.html.erb b/app/views/analyze/index.html.erb index d7704a5b..84737b19 100644 --- a/app/views/analyze/index.html.erb +++ b/app/views/analyze/index.html.erb @@ -13,7 +13,7 @@ <%= render partial: "school_years", locals: {available_academic_years: @presenter.academic_years, selected_academic_years: @presenter.selected_academic_years, district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory, measures: @presenter.measures, graph: @presenter.graph} %> <%= render partial: "data_filters", locals: {district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory} %> - <% cache [@presenter.subcategory, @school, @presenter.selected_academic_years, @presenter.graph, @presenter.selected_races, @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders] do %> + <% cache [@presenter.subcategory, @school, @presenter.selected_academic_years, @presenter.graph, @presenter.selected_races, @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders, @presenter.selected_ells, @presenter.ells] do %>
<% @presenter.measures.each do |measure| %>
diff --git a/data/demographics.csv b/data/demographics.csv index 3a09ce00..f95115e8 100644 --- a/data/demographics.csv +++ b/data/demographics.csv @@ -1,11 +1,11 @@ -Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income -1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged - N -2,Asian or Pacific Islander,1,Female,Economically Disadvantaged - Y -3,Black or African American,4,Non-Binary,Unknown -4,Hispanic or Latinx,99,Unknown, -5,White or Caucasian,,, -6,Prefer not to disclose,,, -7,Prefer to self-describe,,, -8,Middle Eastern,,, -99,Race/Ethnicity Not Listed,,, -100,Multiracial,,, +Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL +1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged - N,ELL +2,Asian or Pacific Islander,1,Female,Economically Disadvantaged - Y,Not ELL +3,Black or African American,4,Non-Binary,Unknown,Unknown +4,Hispanic or Latinx,99,Unknown,, +5,White or Caucasian,,,, +6,Prefer not to disclose,,,, +7,Prefer to self-describe,,,, +8,Middle Eastern,,,, +99,Race/Ethnicity Not Listed,,,, +100,Multiracial,,,, diff --git a/db/migrate/20230830213521_create_ell.rb b/db/migrate/20230830213521_create_ell.rb new file mode 100644 index 00000000..d4dca8ae --- /dev/null +++ b/db/migrate/20230830213521_create_ell.rb @@ -0,0 +1,12 @@ +class CreateEll < ActiveRecord::Migration[7.0] + def change + create_table :ells do |t| + t.string :designation + t.string :slug + + t.timestamps + end + + add_index :ells, :designation, unique: true + end +end diff --git a/db/migrate/20230912223701_add_ell_to_survey_item_response.rb b/db/migrate/20230912223701_add_ell_to_survey_item_response.rb new file mode 100644 index 00000000..026bcac9 --- /dev/null +++ b/db/migrate/20230912223701_add_ell_to_survey_item_response.rb @@ -0,0 +1,5 @@ +class AddEllToSurveyItemResponse < ActiveRecord::Migration[7.0] + def change + add_reference :survey_item_responses, :ell, foreign_key: true + end +end diff --git a/db/schema.rb b/db/schema.rb index e22679e8..43a30356 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_08_07_222503) do +ActiveRecord::Schema[7.0].define(version: 2023_09_12_223701) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -69,6 +69,14 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_07_222503) do t.datetime "updated_at", null: false end + create_table "ells", force: :cascade do |t| + t.string "designation" + t.string "slug" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["designation"], name: "index_ells_on_designation", unique: true + end + create_table "genders", force: :cascade do |t| t.integer "qualtrics_code" t.string "designation" @@ -439,7 +447,9 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_07_222503) do t.bigint "gender_id" t.bigint "income_id" t.datetime "recorded_date" + t.bigint "ell_id" t.index ["academic_year_id"], name: "index_survey_item_responses_on_academic_year_id" + t.index ["ell_id"], name: "index_survey_item_responses_on_ell_id" t.index ["gender_id"], name: "index_survey_item_responses_on_gender_id" t.index ["income_id"], name: "index_survey_item_responses_on_income_id" t.index ["response_id"], name: "index_survey_item_responses_on_response_id" @@ -491,6 +501,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_08_07_222503) do add_foreign_key "student_races", "students" add_foreign_key "subcategories", "categories" add_foreign_key "survey_item_responses", "academic_years" + add_foreign_key "survey_item_responses", "ells" add_foreign_key "survey_item_responses", "genders" add_foreign_key "survey_item_responses", "incomes" add_foreign_key "survey_item_responses", "schools" diff --git a/lib/tasks/clean.rake b/lib/tasks/clean.rake index 3011c02b..87223190 100644 --- a/lib/tasks/clean.rake +++ b/lib/tasks/clean.rake @@ -1,37 +1,33 @@ namespace :clean do # These tasks must be run in their respective project so the correct schools are in the database - desc 'clean ecp data' + desc "clean ecp data" task ecp: :environment do - input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'raw') - output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'clean') - log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'removed') - disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation') - Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean + input_filepath = Rails.root.join("tmp", "data", "ecp_data", "raw") + output_filepath = Rails.root.join("tmp", "data", "ecp_data", "clean") + log_filepath = Rails.root.join("tmp", "data", "ecp_data", "removed") + Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean end - desc 'clean prepped data' + desc "clean prepped data" task prepped: :environment do - input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped') - output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'clean') - log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'removed') - disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation') - Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean + input_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped") + output_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "clean") + log_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "removed") + Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean end - desc 'clean mciea data' + desc "clean mciea data" task mciea: :environment do - input_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'raw') - output_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'clean') - log_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'removed') - disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation') - Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean + input_filepath = Rails.root.join("tmp", "data", "mciea_data", "raw") + output_filepath = Rails.root.join("tmp", "data", "mciea_data", "clean") + log_filepath = Rails.root.join("tmp", "data", "mciea_data", "removed") + Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean end - desc 'clean rpp data' + desc "clean rpp data" task rpp: :environment do - input_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'raw') - output_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'clean') - log_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'removed') - disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation') - Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean + input_filepath = Rails.root.join("tmp", "data", "rpp_data", "raw") + output_filepath = Rails.root.join("tmp", "data", "rpp_data", "clean") + log_filepath = Rails.root.join("tmp", "data", "rpp_data", "removed") + Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean end end diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake index 6c5ed502..3bead793 100644 --- a/lib/tasks/data.rake +++ b/lib/tasks/data.rake @@ -5,7 +5,7 @@ namespace :data do student_count = Student.count path = "/data/survey_responses/clean/" Sftp::Directory.open(path:) do |file| - SurveyResponsesDataLoader.from_file(file:) + SurveyResponsesDataLoader.new.from_file(file:) end puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" diff --git a/lib/tasks/one_off.rake b/lib/tasks/one_off.rake index d1cce35c..c5ac715a 100644 --- a/lib/tasks/one_off.rake +++ b/lib/tasks/one_off.rake @@ -40,38 +40,6 @@ namespace :one_off do end end - desc 'load stoklosa results for 2022-23' - task load_stoklosa: :environment do - survey_item_response_count = SurveyItemResponse.count - school = School.find_by_dese_id(1_600_360) - academic_year = AcademicYear.find_by_range('2022-23') - - ['2022-23_stoklosa_student_survey_responses.csv', - '2022-23_stoklosa_teacher_survey_responses.csv'].each do |filepath| - filepath = Rails.root.join('data', 'survey_responses', filepath) - puts "=====================> Loading data from csv at path: #{filepath}" - SurveyResponsesDataLoader.load_data filepath: - end - puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" - - - Dir.glob(Rails.root.join('data', 'survey_responses', - '2022-23_stoklosa_student_survey_responses.csv')).each do |file| - puts "=====================> Loading student data from csv at path: #{file}" - StudentLoader.load_data filepath: file, rules: [Rule::SkipNonLowellSchools] - end - end - - desc 'load butler results for 2022-23' - task load_butler: :environment do - ['2022-23_butler_student_survey_responses.csv', - '2022-23_butler_teacher_survey_responses.csv'].each do |filepath| - filepath = Rails.root.join('data', 'survey_responses', filepath) - puts "=====================> Loading data from csv at path: #{filepath}" - SurveyResponsesDataLoader.load_data filepath: - end - end - desc 'list scales that have no survey responses' task list_scales_that_lack_survey_responses: :environment do output = AcademicYear.all.map do |academic_year| @@ -114,23 +82,6 @@ namespace :one_off do puts values end - desc 'load survey responses for lowell schools' - task load_survey_responses_for_lowell: :environment do - survey_item_response_count = SurveyItemResponse.count - student_count = Student.count - Sftp::Directory.open(path: '/test/survey_responses/') do |file| - SurveyResponsesDataLoader.from_file(file:) - end - puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" - - Sftp::Directory.open(path: '/test/survey_responses/') do |file| - StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools]) - end - puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - - Rails.cache.clear - end - desc 'delete 2022-23 survey responses' task delete_survey_responses_2022_23: :environment do response_count = SurveyItemResponse.all.count @@ -148,7 +99,7 @@ namespace :one_off do schools = District.find_by_slug('maynard-public-schools').schools Sftp::Directory.open(path:) do |file| - SurveyResponsesDataLoader.from_file(file:) + SurveyResponsesDataLoader.new.from_file(file:) end puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" diff --git a/spec/factories.rb b/spec/factories.rb index a96f0e5d..28b34ece 100644 --- a/spec/factories.rb +++ b/spec/factories.rb @@ -1,11 +1,15 @@ FactoryBot.define do factory :income do - designation { "MyString" } + designation { "DefaultIncome" } + end + + factory :ell do + designation { "DefaultEll" } end factory :gender do qualtrics_code { 1 } - designation { 'MyString' } + designation { "MyString" } end factory :race_score do @@ -68,22 +72,22 @@ FactoryBot.define do end factory :academic_year do - range { '2050-51' } + range { "2050-51" } initialize_with { AcademicYear.find_or_initialize_by(range:) } end - factory :category, class: 'Category' do + factory :category, class: "Category" do name { "A #{rand} category" } category_id { rand.to_s } - description { 'A description of a category' } + description { "A description of a category" } slug { name.parameterize } sort_index { 1 } end factory :subcategory do - name { 'A subcategory' } + name { "A subcategory" } subcategory_id { rand.to_s } - description { 'A description of a subcategory' } + description { "A description of a subcategory" } category factory :subcategory_with_measures do @@ -102,7 +106,7 @@ FactoryBot.define do factory :measure do measure_id { rand.to_s } - name { 'A Measure' } + name { "A Measure" } subcategory trait :with_student_survey_items do after(:create) do |measure| @@ -136,7 +140,7 @@ FactoryBot.define do factory :survey_item do scale - prompt { 'What do YOU think?' } + prompt { "What do YOU think?" } factory :teacher_survey_item do survey_item_id { "t-#{rand}" } watch_low_benchmark { 2.0 } diff --git a/spec/fixtures/sample_demographics.csv b/spec/fixtures/sample_demographics.csv index c60cb690..5444303e 100644 --- a/spec/fixtures/sample_demographics.csv +++ b/spec/fixtures/sample_demographics.csv @@ -1,11 +1,11 @@ -Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income -1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged – N -2,Asian or Pacific Islander,1,Female,Economically Disadvantaged – Y -3,Black or African American,4,Non-Binary,Unknown -4,Hispanic or Latinx,99,Unknown, -5,White or Caucasian,,, -6,Prefer not to disclose,,, -7,Prefer to self-describe,,, -8,Middle Eastern,,, -99,Race/Ethnicity Not Listed,,, -100,Multiracial,,, +Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL +1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged – N,ELL +2,Asian or Pacific Islander,1,Female,Economically Disadvantaged – Y,Not ELL +3,Black or African American,4,Non-Binary,Unknown,Unknown +4,Hispanic or Latinx,99,Unknown,, +5,White or Caucasian,,,, +6,Prefer not to disclose,,,, +7,Prefer to self-describe,,,, +8,Middle Eastern,,,, +99,Race/Ethnicity Not Listed,,,, +100,Multiracial,,,, diff --git a/spec/fixtures/test_2020-21_student_survey_responses.csv b/spec/fixtures/test_2020-21_student_survey_responses.csv index 56d58ce0..21cb3b2b 100644 --- a/spec/fixtures/test_2020-21_student_survey_responses.csv +++ b/spec/fixtures/test_2020-21_student_survey_responses.csv @@ -1,8 +1,8 @@ -Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,LASID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district,school,DESE ID,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,#N/A,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,#N/A,#N/A,s-cure-q3,s-cure-q4,#N/A,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,#N/A,#N/A,s-grmi-q4,s-appa-q1,s-appa-q2,#N/A,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q1-1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-acst-q4,s-acst-q5,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,race,What is your race/ethnicity?(Please select all that apply) - Selected Choice,grade,gender,Raw Income,Income -2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,1500025,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888,11th,1,Free Lunch,Economically Disadvantaged – Y -2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,"2,3,4",888,10,,Not Eligible,Economically Disadvantaged – N -2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1500505,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,6,888,8,2,Reduced Lunch,Economically Disadvantaged – Y -2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1500505,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888,8,3,,Unknown -2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888,7,4,Free Lunch,Economically Disadvantaged – Y -2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888,3,NA,Not Eligible,Economically Disadvantaged – N -2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,,,4,,Reduced Lunch,Economically Disadvantaged – Y +Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,LASID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district,school,DESE ID,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,#N/A,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,#N/A,#N/A,s-cure-q3,s-cure-q4,#N/A,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,#N/A,#N/A,s-grmi-q4,s-appa-q1,s-appa-q2,#N/A,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q1-1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-acst-q4,s-acst-q5,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,race,What is your race/ethnicity?(Please select all that apply) - Selected Choice,grade,gender,Raw Income,Income,Raw ELL,ELL +2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,1500025,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888,11th,1,Free Lunch,Economically Disadvantaged – Y,Does not apply,Not ELL +2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,"2,3,4",888,10,,Not Eligible,Economically Disadvantaged – N,,Unknown +2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1500505,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,6,888,8,2,Reduced Lunch,Economically Disadvantaged – Y,,Unknown +2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1500505,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888,8,3,,Unknown,LEP student not 1st year,ELL +2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888,7,4,Free Lunch,Economically Disadvantaged – Y,EL Student First Year,ELL +2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888,3,NA,Not Eligible,Economically Disadvantaged – N,Unknown,Unknown +2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,,,4,,Reduced Lunch,Economically Disadvantaged – Y,#N/A,Unknown diff --git a/spec/services/cleaner_spec.rb b/spec/services/cleaner_spec.rb index dec5e07e..31fc2a27 100644 --- a/spec/services/cleaner_spec.rb +++ b/spec/services/cleaner_spec.rb @@ -27,14 +27,6 @@ RSpec.describe Cleaner do Rails.root.join("tmp", "spec", "removed") end - let(:disaggregation_filepath) do - Rails.root.join("spec", "fixtures", "disaggregation") - end - - let(:path_to_sample_disaggregation_file) do - File.open(Rails.root.join("spec", "fixtures", "disaggregation", "sample_maynard_disaggregation_data.csv")) - end - let(:path_to_sample_raw_file) do File.open(Rails.root.join("spec", "fixtures", "raw", "sample_maynard_raw_student_survey.csv")) end @@ -99,21 +91,21 @@ RSpec.describe Cleaner do context "Creating a new Cleaner" do it "creates a directory for the clean data" do - Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean + Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean expect(output_filepath).to exist end it "creates a directory for the removed data" do - Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean + Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean expect(log_filepath).to exist end end context ".process_raw_file" do it "sorts data into valid and invalid csvs" do - cleaner = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:) + cleaner = Cleaner.new(input_filepath:, output_filepath:, log_filepath:) processed_data = cleaner.process_raw_file( - file: path_to_sample_raw_file, disaggregation_data: cleaner.disaggregation_data + file: path_to_sample_raw_file ) processed_data in [headers, clean_csv, log_csv, data] @@ -140,22 +132,6 @@ RSpec.describe Cleaner do csv_contains_the_correct_rows(log_csv, invalid_rows) invalid_rows_are_rejected_for_the_correct_reasons(data) end - - it "adds dissaggregation data to the cleaned file " do - cleaner = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:) - processed_data = cleaner.process_raw_file( - file: path_to_sample_raw_file, disaggregation_data: cleaner.disaggregation_data - ) - processed_data in [headers, clean_csv, log_csv, data] - index_of_income = clean_csv.first.index("Income") - expect(clean_csv.second[index_of_income]).to eq "Economically Disadvantaged - Y" - - one_thousand = data.find { |row| row.response_id == "1000" } - expect(one_thousand.income).to eq "Economically Disadvantaged - Y" - - one_thousand_one = data.find { |row| row.response_id == "1001" } - expect(one_thousand_one.income).to eq "Economically Disadvantaged - N" - end end context ".filename" do @@ -166,7 +142,7 @@ RSpec.describe Cleaner do data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: standard_survey_items, genders: nil, survey_items:, schools: School.school_hash)] - filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename( + filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( headers: standard_survey_items, data: ) expect(filename).to eq "maynard.standard.2022-23.csv" @@ -178,7 +154,7 @@ RSpec.describe Cleaner do data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: short_form_survey_items, genders: nil, survey_items:, schools: School.school_hash)] - filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename( + filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( headers: short_form_survey_items, data: ) expect(filename).to eq "maynard.short_form.2022-23.csv" @@ -191,7 +167,7 @@ RSpec.describe Cleaner do data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: early_education_survey_items, genders: nil, survey_items:, schools: School.school_hash)] - filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename( + filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( headers: early_education_survey_items, data: ) expect(filename).to eq "maynard.early_education.2022-23.csv" @@ -203,7 +179,7 @@ RSpec.describe Cleaner do data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: teacher_survey_items, genders: nil, survey_items:, schools: School.school_hash)] - filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename( + filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( headers: teacher_survey_items, data: ) expect(filename).to eq "maynard.teacher.2022-23.csv" @@ -217,7 +193,7 @@ RSpec.describe Cleaner do data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: teacher_survey_items, genders: nil, survey_items:, schools: School.school_hash), SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "222_222" }, headers: teacher_survey_items, genders: nil, survey_items:, schools: School.school_hash)] - filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename( + filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( headers: teacher_survey_items, data: ) expect(filename).to eq "maynard.district2.teacher.2022-23.csv" @@ -230,7 +206,7 @@ end def reads_headers_from_raw_csv(processed_data) processed_data in [headers, clean_csv, log_csv, data] - expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", + expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", "Finished", "RecordedDate", "ResponseId", "District", "School", "LASID", "Gender", "Race", "What grade are you in?", "s-emsa-q1", "s-emsa-q2", "s-emsa-q3", "s-tint-q1", "s-tint-q2", "s-tint-q3", "s-tint-q4", "s-tint-q5", "s-acpr-q1", "s-acpr-q2", @@ -243,7 +219,7 @@ def reads_headers_from_raw_csv(processed_data) "s-grit-q1", "s-grit-q2", "s-grit-q3", "s-grit-q4", "s-expa-q1", "s-poaf-q1", "s-poaf-q2", "s-poaf-q3", "s-poaf-q4", "s-tint-q1-1", "s-tint-q2-1", "s-tint-q3-1", "s-tint-q4-1", "s-tint-q5-1", "s-acpr-q1-1", "s-acpr-q2-1", "s-acpr-q3-1", "s-acpr-q4-1", "s-peff-q1-1", "s-peff-q2-1", "s-peff-q3-1", "s-peff-q4-1", - "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income"].to_set.sort + "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL"].to_set.sort end def invalid_rows_are_rejected_for_the_correct_reasons(data) diff --git a/spec/services/demographic_loader_spec.rb b/spec/services/demographic_loader_spec.rb index 1169e586..c314da6b 100644 --- a/spec/services/demographic_loader_spec.rb +++ b/spec/services/demographic_loader_spec.rb @@ -1,20 +1,24 @@ -require 'rails_helper' +require "rails_helper" describe DemographicLoader do - let(:filepath) { 'spec/fixtures/sample_demographics.csv' } + let(:filepath) { "spec/fixtures/sample_demographics.csv" } let(:race_codes) do - { 'American Indian or Alaskan Native' => 1, 'Asian or Pacific Islander' => 2, 'Black or African American' => 3, - 'Hispanic or Latinx' => 4, 'White or Caucasian' => 5, 'Race/Ethnicity Not Listed' => 99, 'Middle Eastern' => 8, 'Multiracial' => 100 } + { "American Indian or Alaskan Native" => 1, "Asian or Pacific Islander" => 2, "Black or African American" => 3, + "Hispanic or Latinx" => 4, "White or Caucasian" => 5, "Race/Ethnicity Not Listed" => 99, "Middle Eastern" => 8, "Multiracial" => 100 } end let(:gender_codes) do { - 'Female' => 1, 'Male' => 2, 'Non-Binary' => 4, 'Unknown' => 99 + "Female" => 1, "Male" => 2, "Non-Binary" => 4, "Unknown" => 99 } end let(:incomes) do - ['Economically Disadvantaged – N', 'Economically Disadvantaged – Y', 'Unknown'] + ["Economically Disadvantaged – N", "Economically Disadvantaged – Y", "Unknown"] + end + + let(:ells) do + ["ELL", "Not ELL", "Unknown"] end before :each do @@ -25,12 +29,12 @@ describe DemographicLoader do DatabaseCleaner.clean end - describe 'self.load_data' do - it 'does not load qualtrics categories for `prefer not to disclose` or `prefer to self-describe`' do + describe "self.load_data" do + it "does not load qualtrics categories for `prefer not to disclose` or `prefer to self-describe`" do expect(Race.find_by_qualtrics_code(6)).to be nil end - it 'loads all racial designations' do + it "loads all racial designations" do expect(Race.all.count).to eq 8 race_codes.each do |key, value| expect(Race.find_by_qualtrics_code(value)).not_to eq nil @@ -40,7 +44,7 @@ describe DemographicLoader do end end - it 'loads all gender designations' do + it "loads all gender designations" do expect(Gender.all.count).to eq 4 gender_codes.each do |key, value| @@ -51,11 +55,18 @@ describe DemographicLoader do end end - it 'loads all the income designations' do + it "loads all the income designations" do expect(Income.all.count).to eq 3 incomes.each do |income| expect(Income.find_by_designation(income).designation).to eq income end end + + it "loads all the ells designations" do + expect(Ell.all.count).to eq 3 + ells.each do |ell| + expect(Ell.find_by_designation(ell).designation).to eq ell + end + end end end diff --git a/spec/services/disaggregation_loader_spec.rb b/spec/services/disaggregation_loader_spec.rb index 6b0ee739..d8ff718e 100644 --- a/spec/services/disaggregation_loader_spec.rb +++ b/spec/services/disaggregation_loader_spec.rb @@ -1,33 +1,37 @@ -require 'rails_helper' -require 'fileutils' +require "rails_helper" +require "fileutils" RSpec.describe DisaggregationLoader do let(:path) do - Rails.root.join('spec', 'fixtures', 'disaggregation') + Rails.root.join("spec", "fixtures", "disaggregation") end - let(:academic_year) { create(:academic_year, range: '2022-23') } - let(:district) { create(:district, name: 'Maynard Public Schools') } - context '.load' do - it 'loads data from the file into a hash' do + let(:academic_year) { create(:academic_year, range: "2022-23") } + let(:district) { create(:district, name: "Maynard Public Schools") } + context ".load" do + it "loads data from the file into a hash" do data = DisaggregationLoader.new(path:).load - expect(data.values.first.lasid).to eq('1') - expect(data.values.first.academic_year).to eq('2022-23') - expect(data.values.first.district).to eq('Maynard Public Schools') - expect(data.values.first.income).to eq('Free Lunch') + expect(data.values.first.lasid).to eq("1") + expect(data.values.first.academic_year).to eq("2022-23") + expect(data.values.first.district).to eq("Maynard Public Schools") - expect(data.values.last.lasid).to eq('500') - expect(data.values.last.academic_year).to eq('2022-23') - expect(data.values.last.district).to eq('Maynard Public Schools') - expect(data.values.last.income).to eq('Not Eligible') + expect(data.values.last.lasid).to eq("500") + expect(data.values.last.academic_year).to eq("2022-23") + expect(data.values.last.district).to eq("Maynard Public Schools") + end + + it "loads income data" do + data = DisaggregationLoader.new(path:).load + expect(data.values.first.raw_income).to eq("Free Lunch") + expect(data.values.last.raw_income).to eq("Not Eligible") - expect(data[['1', 'Maynard Public Schools', '2022-23']].income).to eq('Free Lunch') - expect(data[['2', 'Maynard Public Schools', '2022-23']].income).to eq('Not Eligible') - expect(data[['3', 'Maynard Public Schools', '2022-23']].income).to eq('Reduced Lunch') + expect(data[["1", "Maynard Public Schools", "2022-23"]].raw_income).to eq("Free Lunch") + expect(data[["2", "Maynard Public Schools", "2022-23"]].raw_income).to eq("Not Eligible") + expect(data[["3", "Maynard Public Schools", "2022-23"]].raw_income).to eq("Reduced Lunch") end end - context 'Creating a new loader' do - it 'creates a directory for the loader file' do + context "Creating a new loader" do + it "creates a directory for the loader file" do DisaggregationLoader.new(path:) expect(path).to exist end diff --git a/spec/services/disaggregation_row_spec.rb b/spec/services/disaggregation_row_spec.rb index 0fdfa4b3..fbebf064 100644 --- a/spec/services/disaggregation_row_spec.rb +++ b/spec/services/disaggregation_row_spec.rb @@ -1,73 +1,107 @@ -require 'rails_helper' +require "rails_helper" RSpec.describe DisaggregationRow do let(:headers) do - ['District', 'Academic Year', 'LASID', 'HispanicLatino', 'Race', 'Gender', 'SpecialEdStatus', 'In 504 Plan', - 'LowIncome', 'EL Student First Year'] + ["District", "Academic Year", "LASID", "HispanicLatino", "Race", "Gender", "SpecialEdStatus", "In 504 Plan", + "LowIncome", "EL Student First Year"] end - context '.district' do - context 'when the column heading is any upper or lowercase variant of the word district' do - it 'returns the correct value for district' do - row = { 'District' => 'Maynard Public Schools' } - expect(DisaggregationRow.new(row:, headers:).district).to eq 'Maynard Public Schools' + context ".district" do + context "when the column heading is any upper or lowercase variant of the word district" do + it "returns the correct value for district" do + row = { "District" => "Maynard Public Schools" } + expect(DisaggregationRow.new(row:, headers:).district).to eq "Maynard Public Schools" - headers = ['dISTRICT'] + headers = ["dISTRICT"] headers in [district] - row = { district => 'Maynard Public Schools' } - expect(DisaggregationRow.new(row:, headers:).district).to eq 'Maynard Public Schools' + row = { district => "Maynard Public Schools" } + expect(DisaggregationRow.new(row:, headers:).district).to eq "Maynard Public Schools" end end end - context '.academic_year' do - context 'when the column heading is any upper or lowercase variant of the words academic year' do - it 'returns the correct value for district' do - row = { 'Academic Year' => '2022-23' } - expect(DisaggregationRow.new(row:, headers:).academic_year).to eq '2022-23' + context ".academic_year" do + context "when the column heading is any upper or lowercase variant of the words academic year" do + it "returns the correct value for district" do + row = { "Academic Year" => "2022-23" } + expect(DisaggregationRow.new(row:, headers:).academic_year).to eq "2022-23" - headers = ['aCADEMIC yEAR'] + headers = ["aCADEMIC yEAR"] headers in [academic_year] - row = { academic_year => '2022-23' } - expect(DisaggregationRow.new(row:, headers:).academic_year).to eq '2022-23' + row = { academic_year => "2022-23" } + expect(DisaggregationRow.new(row:, headers:).academic_year).to eq "2022-23" - headers = ['AcademicYear'] + headers = ["AcademicYear"] headers in [academic_year] - row = { academic_year => '2022-23' } - expect(DisaggregationRow.new(row:, headers:).academic_year).to eq '2022-23' + row = { academic_year => "2022-23" } + expect(DisaggregationRow.new(row:, headers:).academic_year).to eq "2022-23" end end end - context '.income' do - context 'when the column heading is any upper or lowercase variant of the words low income' do - it 'returns the correct value for low_income' do - row = { 'LowIncome' => 'Free Lunch' } - expect(DisaggregationRow.new(row:, headers:).income).to eq 'Free Lunch' + context ".raw_income" do + context "when the column heading is any upper or lowercase variant of the words low income" do + it "returns the correct value for low_income" do + row = { "LowIncome" => "Free Lunch" } + expect(DisaggregationRow.new(row:, headers:).raw_income).to eq "Free Lunch" - headers = ['Low income'] + headers = ["Low income"] headers in [income] - row = { income => 'Free Lunch' } - expect(DisaggregationRow.new(row:, headers:).income).to eq 'Free Lunch' + row = { income => "Free Lunch" } + expect(DisaggregationRow.new(row:, headers:).raw_income).to eq "Free Lunch" - headers = ['LoW InCOme'] + headers = ["LoW InCOme"] headers in [income] - row = { income => 'Free Lunch' } - expect(DisaggregationRow.new(row:, headers:).income).to eq 'Free Lunch' + row = { income => "Free Lunch" } + expect(DisaggregationRow.new(row:, headers:).raw_income).to eq "Free Lunch" end end end - context '.lasid' do - context 'when the column heading is any upper or lowercase variant of the words lasid' do - it 'returns the correct value for lasid' do - row = { 'LASID' => '2366' } - expect(DisaggregationRow.new(row:, headers:).lasid).to eq '2366' + context ".lasid" do + context "when the column heading is any upper or lowercase variant of the words lasid" do + it "returns the correct value for lasid" do + row = { "LASID" => "2366" } + expect(DisaggregationRow.new(row:, headers:).lasid).to eq "2366" - headers = ['LaSiD'] + headers = ["LaSiD"] headers in [lasid] - row = { lasid => '2366' } - expect(DisaggregationRow.new(row:, headers:).lasid).to eq '2366' + row = { lasid => "2366" } + expect(DisaggregationRow.new(row:, headers:).lasid).to eq "2366" + end + end + end + + context ".ell" do + context "when the column heading is any upper or lowercase variant of the words 'ELL' or 'El Student First Year'" do + it "returns the correct value for a student" do + row = { "EL Student First Year" => "LEP student 1st year" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "ELL" + + headers = ["EL Student First Year"] + headers in [ell] + row = { ell => "LEP student not 1st year" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "ELL" + + headers = ["EL Student First Year"] + headers in [ell] + row = { ell => "Does not apply" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "Not ELL" + + headers = ["EL Student First Year"] + headers in [ell] + row = { ell => "Unknown" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "Unknown" + + headers = ["EL Student First Year"] + headers in [ell] + row = { ell => "Any other text" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "Unknown" + + headers = ["EL Student First Year"] + headers in [ell] + row = { ell => "" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "Unknown" end end end diff --git a/spec/services/student_loader_spec.rb b/spec/services/student_loader_spec.rb index e78a35fb..e998ea7f 100644 --- a/spec/services/student_loader_spec.rb +++ b/spec/services/student_loader_spec.rb @@ -88,7 +88,7 @@ describe StudentLoader do describe "self.load_data" do context "load student data for all schools" do before :each do - SurveyResponsesDataLoader.load_data filepath: path_to_student_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses StudentLoader.load_data filepath: path_to_student_responses end @@ -102,7 +102,7 @@ describe StudentLoader do # TODO: get this test to run correctly. Since we are no longer seeding, we need to define schools, and districts; some Lowell, some not xcontext "When using the rule to skip non Lowell schools" do before :each do - SurveyResponsesDataLoader.load_data filepath: path_to_student_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses StudentLoader.load_data filepath: path_to_student_responses, rules: [Rule::SkipNonLowellSchools] end diff --git a/spec/services/survey_item_values_spec.rb b/spec/services/survey_item_values_spec.rb index bf80ad90..6b25a7a1 100644 --- a/spec/services/survey_item_values_spec.rb +++ b/spec/services/survey_item_values_spec.rb @@ -7,7 +7,7 @@ RSpec.describe SurveyItemValues, type: :model do end let(:genders) do create(:gender, qualtrics_code: 1) - Gender.gender_hash + Gender.by_qualtrics_code end let(:survey_items) { [] } let(:district) { create(:district, name: "Attleboro") } @@ -165,79 +165,90 @@ RSpec.describe SurveyItemValues, type: :model do end context ".income" do - context "when no disaggregation data is provided" do - it "returns an empty string " do - disaggregation_data = {} - values = SurveyItemValues.new(row: {}, headers:, genders:, survey_items:, schools:, disaggregation_data:) - expect(values.income).to eq "Unknown" - end + before :each do + attleboro + ay_2022_23 end - context "when disaggregation data is provided" do - before :each do - attleboro - ay_2022_23 - end + it "translates Free Lunch to Economically Disadvantaged - Y" do + headers = ["LowIncome"] + row = { "LowIncome" => "Free Lunch" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.income).to eq "Economically Disadvantaged - Y" + end - it "translates Free Lunch to Economically Disadvantaged - Y" do - headers = ["District", "Academic Year", "LASID", "LowIncome"] - row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Free Lunch" } - disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) } + it "translates Reduced Lunch to Economically Disadvantaged - Y" do + headers = ["LowIncome"] + row = { "LowIncome" => "Reduced Lunch" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.income).to eq "Economically Disadvantaged - Y" + end - headers = ["LASID", "Dese Id", "RecordedDate"] - row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" } - values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, - disaggregation_data:) - expect(values.income).to eq "Economically Disadvantaged - Y" - end + it "translates LowIncome to Economically Disadvantaged - Y" do + headers = ["LowIncome"] + row = { "LowIncome" => "LowIncome" } - it "translates Reduced Lunch to Economically Disadvantaged - Y" do - headers = ["District", "Academic Year", "LASID", "LowIncome"] - row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Reduced Lunch" } - disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.income).to eq "Economically Disadvantaged - Y" + end - headers = ["LASID", "Dese Id", "RecordedDate"] - row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" } - values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, - disaggregation_data:) - expect(values.income).to eq "Economically Disadvantaged - Y" - end + it "translates Not Eligible to Economically Disadvantaged - N" do + headers = ["LowIncome"] + row = { "LowIncome" => "Not Eligible" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.income).to eq "Economically Disadvantaged - N" + end - it "translates LowIncome to Economically Disadvantaged - Y" do - headers = ["District", "Academic Year", "LASID", "LowIncome"] - row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "LowIncome" } - disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) } + it "translates blanks to Unknown" do + headers = ["LowIncome"] + row = { "LowIncome" => "" } - headers = ["LASID", "Dese Id", "RecordedDate"] - row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" } - values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, - disaggregation_data:) - expect(values.income).to eq "Economically Disadvantaged - Y" - end + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.income).to eq "Unknown" + end + end - it "translates Not Eligible to Economically Disadvantaged - N" do - headers = ["District", "Academic Year", "LASID", "LowIncome"] - row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Not Eligible" } - disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) } + context ".ell" do + before :each do + attleboro + ay_2022_23 + end - headers = ["LASID", "Dese Id", "RecordedDate"] - row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" } - values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, - disaggregation_data:) - expect(values.income).to eq "Economically Disadvantaged - N" - end + it 'translates "LEP Student 1st Year" or "LEP Student Not 1st Year" into ELL' do + headers = ["Raw ELL"] + row = { "Raw ELL" => "LEP Student 1st Year" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "ELL" - it "translates blanks to Unknown" do - headers = ["District", "Academic Year", "LASID", "LowIncome"] - row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "" } - disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) } + row = { "Raw ELL" => "LEP Student Not 1st Year" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "ELL" - headers = ["LASID", "Dese Id", "RecordedDate"] - row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" } - values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, - disaggregation_data:) - expect(values.income).to eq "Unknown" - end + row = { "Raw ELL" => "LEP Student Not 1st Year" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "ELL" + end + + it 'translates "Does not Apply" into "Not ELL"' do + headers = ["Raw ELL"] + row = { "Raw ELL" => "Does not apply" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "Not ELL" + + row = { "Raw ELL" => "Does Not APPLY" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "Not ELL" + end + + it 'tranlsates blanks into "Unknown"' do + headers = ["Raw ELL"] + row = { "Raw ELL" => "" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "Unknown" + + row = { "Raw ELL" => "Anything else" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "Unknown" end end diff --git a/spec/services/survey_responses_data_loader_spec.rb b/spec/services/survey_responses_data_loader_spec.rb index fc52e69d..d0d70fae 100644 --- a/spec/services/survey_responses_data_loader_spec.rb +++ b/spec/services/survey_responses_data_loader_spec.rb @@ -54,6 +54,9 @@ describe SurveyResponsesDataLoader do let(:low_income) { create(:income, designation: "Economically Disadvantaged – Y") } let(:high_income) { create(:income, designation: "Economically Disadvantaged – N") } let(:unknown_income) { create(:income, designation: "Unknown") } + let(:yes_ell) { create(:ell, designation: "ELL") } + let(:not_ell) { create(:ell, designation: "Not ELL") } + let(:unknown_ell) { create(:ell, designation: "Unknown") } let(:setup) do ay_2020_21 @@ -92,6 +95,9 @@ describe SurveyResponsesDataLoader do low_income high_income unknown_income + yes_ell + not_ell + unknown_ell end before :each do @@ -100,7 +106,7 @@ describe SurveyResponsesDataLoader do describe "loading teacher survey responses" do before do - SurveyResponsesDataLoader.load_data filepath: path_to_teacher_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_teacher_responses end it "ensures teacher responses load correctly" do @@ -116,7 +122,7 @@ describe SurveyResponsesDataLoader do describe "student survey responses" do before do - SurveyResponsesDataLoader.load_data filepath: path_to_student_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses end it "ensures student responses load correctly" do @@ -129,13 +135,14 @@ describe SurveyResponsesDataLoader do assigns_grade_level_to_responses assigns_gender_to_responses assigns_income_to_responses + assigns_ell_to_responses is_idempotent_for_students end context "when updating student survey responses from another csv file" do before :each do - SurveyResponsesDataLoader.load_data filepath: Rails.root.join("spec", "fixtures", - "secondary_test_2020-21_student_survey_responses.csv") + SurveyResponsesDataLoader.new.load_data filepath: Rails.root.join("spec", "fixtures", + "secondary_test_2020-21_student_survey_responses.csv") end it "updates the likert score to the score on the new csv file" do s_emsa_q1 = SurveyItem.find_by_survey_item_id "s-emsa-q1" @@ -154,8 +161,8 @@ describe SurveyResponsesDataLoader do # figure out why this is failing describe "when using Lowell rules to skip rows in the csv file" do before :each do - SurveyResponsesDataLoader.load_data filepath: path_to_student_responses, - rules: [Rule::SkipNonLowellSchools] + SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses, + rules: [Rule::SkipNonLowellSchools] end it "rejects any non-lowell school" do @@ -172,8 +179,8 @@ describe SurveyResponsesDataLoader do context "when loading 22-23 butler survey responses" do before :each do - SurveyResponsesDataLoader.load_data filepath: path_to_butler_student_responses, - rules: [Rule::SkipNonLowellSchools] + SurveyResponsesDataLoader.new.load_data filepath: path_to_butler_student_responses, + rules: [Rule::SkipNonLowellSchools] end it "loads all the responses for Butler" do @@ -235,7 +242,7 @@ end def is_idempotent number_of_survey_item_responses = SurveyItemResponse.count - SurveyResponsesDataLoader.load_data filepath: path_to_teacher_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_teacher_responses expect(SurveyItemResponse.count).to eq number_of_survey_item_responses end @@ -281,7 +288,7 @@ end def is_idempotent_for_students number_of_survey_item_responses = SurveyItemResponse.count - SurveyResponsesDataLoader.load_data filepath: path_to_student_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses expect(SurveyItemResponse.count).to eq number_of_survey_item_responses end @@ -344,6 +351,21 @@ def assigns_income_to_responses "student_survey_response_7" => low_income } results.each do |key, value| - expect(SurveyItemResponse.where(response_id: key).first.income).to eq value + income = SurveyItemResponse.find_by_response_id(key).income + expect(income).to eq value + end +end + +def assigns_ell_to_responses + results = { "student_survey_response_1" => not_ell, + "student_survey_response_3" => unknown_ell, + "student_survey_response_4" => yes_ell, + "student_survey_response_5" => yes_ell, + "student_survey_response_6" => unknown_ell, + "student_survey_response_7" => unknown_ell } + + results.each do |key, value| + ell = SurveyItemResponse.find_by_response_id(key).ell + expect(ell).to eq value end end