diff --git a/app/javascript/controllers/analyze_controller.js b/app/javascript/controllers/analyze_controller.js index bf290c9b..bef3ccae 100644 --- a/app/javascript/controllers/analyze_controller.js +++ b/app/javascript/controllers/analyze_controller.js @@ -28,7 +28,9 @@ export default class extends Controller { "&incomes=" + this.selected_items("income").join(",") + "&grades=" + - this.selected_items("grade").join(","); + this.selected_items("grade").join(",") + + "&ells=" + + this.selected_items("ell").join(","); this.go_to(url); } @@ -126,7 +128,8 @@ export default class extends Controller { ['gender', 'students-by-gender'], ['grade', 'students-by-grade'], ['income', 'students-by-income'], - ['race', 'students-by-race'] + ['race', 'students-by-race'], + ['ell', 'students-by-ell'], ]) if (target.name === 'slice' || target.name === 'group') { diff --git a/app/models/district.rb b/app/models/district.rb index eefbb4ab..023fac3c 100644 --- a/app/models/district.rb +++ b/app/models/district.rb @@ -16,6 +16,11 @@ class District < ApplicationRecord end def self.boston - District.find_by_name('Boston') + District.find_by_name("Boston") + end + + def short_name + name.split(" ").first.downcase end end + diff --git a/app/models/ell.rb b/app/models/ell.rb new file mode 100644 index 00000000..6b121fb1 --- /dev/null +++ b/app/models/ell.rb @@ -0,0 +1,7 @@ +class Ell < ApplicationRecord + scope :by_designation, -> { all.map { |ell| [ell.designation, ell] }.to_h } + + include FriendlyId + + friendly_id :designation, use: [:slugged] +end diff --git a/app/models/gender.rb b/app/models/gender.rb index 5bfdc766..50f4d74c 100644 --- a/app/models/gender.rb +++ b/app/models/gender.rb @@ -1,5 +1,5 @@ class Gender < ApplicationRecord - scope :gender_hash, lambda { + scope :by_qualtrics_code, lambda { all.map { |gender| [gender.qualtrics_code, gender] }.to_h } end diff --git a/app/models/income.rb b/app/models/income.rb index 447f8465..f43be8d1 100644 --- a/app/models/income.rb +++ b/app/models/income.rb @@ -1,5 +1,6 @@ class Income < ApplicationRecord scope :by_designation, -> { all.map { |income| [income.designation, income] }.to_h } + scope :by_slug, -> { all.map { |income| [income.slug, income] }.to_h } include FriendlyId diff --git a/app/models/school.rb b/app/models/school.rb index 53fd88a9..98367218 100644 --- a/app/models/school.rb +++ b/app/models/school.rb @@ -19,4 +19,8 @@ class School < ApplicationRecord .where(districts: { qualtrics_code: district_code }) .find_by_qualtrics_code(school_code) end + + def grades(academic_year:) + Respondent.find_by(school: self, academic_year:)&.counts_by_grade&.keys || (-1..12).to_a + end end diff --git a/app/models/survey_item_response.rb b/app/models/survey_item_response.rb index 21f05887..e777239c 100644 --- a/app/models/survey_item_response.rb +++ b/app/models/survey_item_response.rb @@ -10,6 +10,7 @@ class SurveyItemResponse < ActiveRecord::Base belongs_to :student, foreign_key: :student_id, optional: true belongs_to :gender belongs_to :income + belongs_to :ell has_one :measure, through: :survey_item @@ -32,9 +33,15 @@ class SurveyItemResponse < ActiveRecord::Base academic_year:, income:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score) } + scope :averages_for_ell, lambda { |survey_items, school, academic_year, ell| + SurveyItemResponse.where(survey_item: survey_items, school:, + academic_year:, ell:, grade: school.grades(academic_year:)).group(:survey_item).having("count(*) >= 10").average(:likert_score) + } + scope :averages_for_race, lambda { |school, academic_year, race| SurveyItemResponse.joins("JOIN student_races on survey_item_responses.student_id = student_races.student_id JOIN students on students.id = student_races.student_id").where( school:, academic_year:, grade: school.grades(academic_year:) ).where("student_races.race_id": race.id).group(:survey_item_id).having("count(*) >= 10").average(:likert_score) } end + diff --git a/app/presenters/analyze/graph/column/ell_column/ell.rb b/app/presenters/analyze/graph/column/ell_column/ell.rb new file mode 100644 index 00000000..cba07408 --- /dev/null +++ b/app/presenters/analyze/graph/column/ell_column/ell.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Analyze + module Graph + module Column + module EllColumn + class Ell < GroupedBarColumnPresenter + include Analyze::Graph::Column::EllColumn::ScoreForEll + include Analyze::Graph::Column::EllColumn::EllCount + def label + %w[ELL] + end + + def basis + "student" + end + + def show_irrelevancy_message? + false + end + + def show_insufficient_data_message? + false + end + + def ell + ::Ell.find_by_slug "ell" + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/column/ell_column/ell_count.rb b/app/presenters/analyze/graph/column/ell_column/ell_count.rb new file mode 100644 index 00000000..fa20518a --- /dev/null +++ b/app/presenters/analyze/graph/column/ell_column/ell_count.rb @@ -0,0 +1,18 @@ +module Analyze + module Graph + module Column + module EllColumn + module EllCount + def type + :student + end + + def n_size(year_index) + SurveyItemResponse.where(ell:, survey_item: measure.student_survey_items, school:, grade: grades(year_index), + academic_year: academic_years[year_index]).select(:response_id).distinct.count + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/column/ell_column/not_ell.rb b/app/presenters/analyze/graph/column/ell_column/not_ell.rb new file mode 100644 index 00000000..ba93d5f7 --- /dev/null +++ b/app/presenters/analyze/graph/column/ell_column/not_ell.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Analyze + module Graph + module Column + module EllColumn + class NotEll < GroupedBarColumnPresenter + include Analyze::Graph::Column::EllColumn::ScoreForEll + include Analyze::Graph::Column::EllColumn::EllCount + def label + %w[Not-ELL] + end + + def basis + "student" + end + + def show_irrelevancy_message? + false + end + + def show_insufficient_data_message? + false + end + + def ell + ::Ell.find_by_slug "not-ell" + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/column/ell_column/score_for_ell.rb b/app/presenters/analyze/graph/column/ell_column/score_for_ell.rb new file mode 100644 index 00000000..cd39cd0f --- /dev/null +++ b/app/presenters/analyze/graph/column/ell_column/score_for_ell.rb @@ -0,0 +1,42 @@ +module Analyze + module Graph + module Column + module EllColumn + module ScoreForEll + def score(year_index) + academic_year = academic_years[year_index] + meets_student_threshold = sufficient_student_responses?(academic_year:) + return Score::NIL_SCORE unless meets_student_threshold + + averages = SurveyItemResponse.averages_for_ell(measure.student_survey_items, school, academic_year, + ell) + average = bubble_up_averages(averages:).round(2) + + Score.new(average:, + meets_teacher_threshold: false, + meets_student_threshold:, + meets_admin_data_threshold: false) + end + + def bubble_up_averages(averages:) + measure.student_scales.map do |scale| + scale.survey_items.map do |survey_item| + averages[survey_item] + end.remove_blanks.average + end.remove_blanks.average + end + + def sufficient_student_responses?(academic_year:) + return false unless measure.subcategory.response_rate(school:, academic_year:).meets_student_threshold? + + yearly_counts = SurveyItemResponse.where(school:, academic_year:, + ell:, survey_item: measure.student_survey_items).group(:ell).select(:response_id).distinct(:response_id).count + yearly_counts.any? do |count| + count[1] >= 10 + end + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/column/ell_column/unknown.rb b/app/presenters/analyze/graph/column/ell_column/unknown.rb new file mode 100644 index 00000000..37f077ae --- /dev/null +++ b/app/presenters/analyze/graph/column/ell_column/unknown.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Analyze + module Graph + module Column + module EllColumn + class Unknown < GroupedBarColumnPresenter + include Analyze::Graph::Column::EllColumn::ScoreForEll + include Analyze::Graph::Column::EllColumn::EllCount + def label + %w[Unknown] + end + + def basis + "student" + end + + def show_irrelevancy_message? + false + end + + def show_insufficient_data_message? + false + end + + def ell + ::Ell.find_by_slug "unknown" + end + end + end + end + end +end diff --git a/app/presenters/analyze/graph/students_by_ell.rb b/app/presenters/analyze/graph/students_by_ell.rb new file mode 100644 index 00000000..ff96435b --- /dev/null +++ b/app/presenters/analyze/graph/students_by_ell.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +# +module Analyze + module Graph + class StudentsByEll + include Analyze::Graph::Column::GenderColumn + attr_reader :ells + + def initialize(ells:) + @ells = ells + end + + def to_s + "Students by Ell" + end + + def slug + "students-by-ell" + end + + def columns + [].tap do |array| + ells.each do |ell| + array << column_for_ell_code(code: ell.slug) + end + array.sort_by!(&:to_s) + array << Analyze::Graph::Column::AllStudent + end + end + + private + + def column_for_ell_code(code:) + CFR[code] + end + + CFR = { + "ell" => Analyze::Graph::Column::EllColumn::Ell, + "not-ell" => Analyze::Graph::Column::EllColumn::NotEll, + "unknown" => Analyze::Graph::Column::EllColumn::Unknown + }.freeze + end + end +end diff --git a/app/presenters/analyze/graph/students_by_gender.rb b/app/presenters/analyze/graph/students_by_gender.rb index 8e3fb50a..97d33634 100644 --- a/app/presenters/analyze/graph/students_by_gender.rb +++ b/app/presenters/analyze/graph/students_by_gender.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Analyze module Graph class StudentsByGender diff --git a/app/presenters/analyze/graph/students_by_grade.rb b/app/presenters/analyze/graph/students_by_grade.rb index 267a9037..3fb113d4 100644 --- a/app/presenters/analyze/graph/students_by_grade.rb +++ b/app/presenters/analyze/graph/students_by_grade.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Analyze module Graph class StudentsByGrade @@ -9,11 +11,11 @@ module Analyze end def to_s - 'Students by Grade' + "Students by Grade" end def slug - 'students-by-grade' + "students-by-grade" end def columns diff --git a/app/presenters/analyze/graph/students_by_income.rb b/app/presenters/analyze/graph/students_by_income.rb index 148b22c1..e72b6efb 100644 --- a/app/presenters/analyze/graph/students_by_income.rb +++ b/app/presenters/analyze/graph/students_by_income.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Analyze module Graph class StudentsByIncome diff --git a/app/presenters/analyze/graph/students_by_race.rb b/app/presenters/analyze/graph/students_by_race.rb index 0c4009ae..74a7cb8f 100644 --- a/app/presenters/analyze/graph/students_by_race.rb +++ b/app/presenters/analyze/graph/students_by_race.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Analyze module Graph class StudentsByRace @@ -8,11 +10,11 @@ module Analyze end def to_s - 'Students by Race' + "Students by Race" end def slug - 'students-by-race' + "students-by-race" end def columns @@ -31,14 +33,14 @@ module Analyze end CFR = { - '1' => Analyze::Graph::Column::RaceColumn::AmericanIndian, - '2' => Analyze::Graph::Column::RaceColumn::Asian, - '3' => Analyze::Graph::Column::RaceColumn::Black, - '4' => Analyze::Graph::Column::RaceColumn::Hispanic, - '5' => Analyze::Graph::Column::RaceColumn::White, - '8' => Analyze::Graph::Column::RaceColumn::MiddleEastern, - '99' => Analyze::Graph::Column::RaceColumn::Unknown, - '100' => Analyze::Graph::Column::RaceColumn::Multiracial + "1" => Analyze::Graph::Column::RaceColumn::AmericanIndian, + "2" => Analyze::Graph::Column::RaceColumn::Asian, + "3" => Analyze::Graph::Column::RaceColumn::Black, + "4" => Analyze::Graph::Column::RaceColumn::Hispanic, + "5" => Analyze::Graph::Column::RaceColumn::White, + "8" => Analyze::Graph::Column::RaceColumn::MiddleEastern, + "99" => Analyze::Graph::Column::RaceColumn::Unknown, + "100" => Analyze::Graph::Column::RaceColumn::Multiracial }.freeze end end diff --git a/app/presenters/analyze/group/ell.rb b/app/presenters/analyze/group/ell.rb new file mode 100644 index 00000000..512b3d5b --- /dev/null +++ b/app/presenters/analyze/group/ell.rb @@ -0,0 +1,13 @@ +module Analyze + module Group + class Ell + def name + "ELL" + end + + def slug + "ell" + end + end + end +end diff --git a/app/presenters/analyze/presenter.rb b/app/presenters/analyze/presenter.rb index 9e5492d0..4353d905 100644 --- a/app/presenters/analyze/presenter.rb +++ b/app/presenters/analyze/presenter.rb @@ -54,9 +54,27 @@ module Analyze end end + def ells + @ells ||= Ell.all.order(slug: :ASC) + end + + def selected_ells + @selected_ells ||= begin + ell_params = params[:ells] + return ells unless ell_params + + ell_params.split(",").map { |ell| Ell.find_by_slug ell }.compact + end + end + def graphs - @graphs ||= [Analyze::Graph::AllData.new, Analyze::Graph::StudentsAndTeachers.new, Analyze::Graph::StudentsByRace.new(races: selected_races), - Analyze::Graph::StudentsByGrade.new(grades: selected_grades), Analyze::Graph::StudentsByGender.new(genders: selected_genders), Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes)] + @graphs ||= [Analyze::Graph::AllData.new, + Analyze::Graph::StudentsAndTeachers.new, + Analyze::Graph::StudentsByRace.new(races: selected_races), + Analyze::Graph::StudentsByGrade.new(grades: selected_grades), + Analyze::Graph::StudentsByGender.new(genders: selected_genders), + Analyze::Graph::StudentsByIncome.new(incomes: selected_incomes), + Analyze::Graph::StudentsByEll.new(ells: selected_ells)] end def graph @@ -88,7 +106,7 @@ module Analyze end def groups - @groups = [Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new, + @groups = [Analyze::Group::Ell.new, Analyze::Group::Gender.new, Analyze::Group::Grade.new, Analyze::Group::Income.new, Analyze::Group::Race.new] end diff --git a/app/services/cleaner.rb b/app/services/cleaner.rb index e457092d..39d8c38c 100644 --- a/app/services/cleaner.rb +++ b/app/services/cleaner.rb @@ -1,6 +1,6 @@ require "fileutils" class Cleaner - attr_reader :input_filepath, :output_filepath, :log_filepath, :clean_csv, :log_csv + attr_reader :input_filepath, :output_filepath, :log_filepath def initialize(input_filepath:, output_filepath:, log_filepath:) @input_filepath = input_filepath @@ -9,18 +9,13 @@ class Cleaner initialize_directories end - def initialize_directories - create_ouput_directory - create_log_directory - end - def clean Dir.glob(Rails.root.join(input_filepath, "*.csv")).each do |filepath| puts filepath - File.open(filepath) do |_file| - clean_csv = [] - log_csv = [] - data = [] + File.open(filepath) do |file| + processed_data = process_raw_file(file:) + processed_data in [headers, clean_csv, log_csv, data] + return if data.empty? filename = filename(headers:, data:) write_csv(data: clean_csv, output_filepath:, filename:) @@ -37,23 +32,22 @@ class Cleaner range = data.first.academic_year.range districts = data.map do |row| - row.district.name + row.district.short_name end.to_set.to_a districts.join(".").to_s + "." + survey_type.to_s + "." + range + ".csv" end - def process_raw_file(file:, disaggregation_data:) + def process_raw_file(file:) clean_csv = [] log_csv = [] data = [] - headers = (CSV.parse(file.first).first << "Raw Income") << "Income" + headers = CSV.parse(file.first).first.push("Raw Income").push("Income").push("Raw ELL").push("ELL").push("Raw SpEd").push("SpEd") filtered_headers = include_all_headers(headers:) filtered_headers = remove_unwanted_headers(headers: filtered_headers) log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?", "Valid Standard Deviation?"]).flatten - clean_csv << filtered_headers log_csv << log_headers @@ -62,7 +56,7 @@ class Cleaner file.lazy.each_slice(1000) do |lines| CSV.parse(lines.join, headers:).map do |row| values = SurveyItemValues.new(row:, headers:, genders:, - survey_items: all_survey_items, schools:, disaggregation_data:) + survey_items: all_survey_items, schools:) next unless values.valid_school? data << values @@ -104,24 +98,12 @@ class Cleaner File.write(output_filepath.join(prefix + filename), csv) end - def process_row(row:) - clean_csv << row.to_csv - log_csv << row.to_csv - end - def schools @schools ||= School.school_hash end def genders - @genders ||= begin - gender_hash = {} - - Gender.all.each do |gender| - gender_hash[gender.qualtrics_code] = gender - end - gender_hash - end + @genders ||= Gender.by_qualtrics_code end def survey_items(headers:) @@ -138,8 +120,5 @@ class Cleaner def create_log_directory FileUtils.mkdir_p log_filepath end - - def create_file(path:, filename:) - FileUtils.touch path.join(filename) - end end + diff --git a/app/services/demographic_loader.rb b/app/services/demographic_loader.rb index 345020e8..cb11a51c 100644 --- a/app/services/demographic_loader.rb +++ b/app/services/demographic_loader.rb @@ -6,12 +6,13 @@ class DemographicLoader process_race(row:) process_gender(row:) process_income(row:) + process_ell(row:) end end def self.process_race(row:) - qualtrics_code = row['Race Qualtrics Code'].to_i - designation = row['Race/Ethnicity'] + qualtrics_code = row["Race Qualtrics Code"].to_i + designation = row["Race/Ethnicity"] return unless qualtrics_code && designation if qualtrics_code.between?(6, 7) @@ -22,8 +23,8 @@ class DemographicLoader end def self.process_gender(row:) - qualtrics_code = row['Gender Qualtrics Code'].to_i - designation = row['Sex/Gender'] + qualtrics_code = row["Gender Qualtrics Code"].to_i + designation = row["Sex/Gender"] return unless qualtrics_code && designation gender = ::Gender.find_or_create_by!(qualtrics_code:, designation:) @@ -31,11 +32,18 @@ class DemographicLoader end def self.process_income(row:) - designation = row['Income'] + designation = row["Income"] return unless designation Income.find_or_create_by!(designation:) end + + def self.process_ell(row:) + designation = row["ELL"] + return unless designation + + Ell.find_or_create_by!(designation:) + end end class KnownRace @@ -50,7 +58,7 @@ end class UnknownRace def initialize(qualtrics_code:, designation:) unknown = Race.find_or_create_by!(qualtrics_code: 99) - unknown.designation = 'Race/Ethnicity Not Listed' + unknown.designation = "Race/Ethnicity Not Listed" unknown.slug = designation.parameterize unknown.save end diff --git a/app/services/disaggregation_row.rb b/app/services/disaggregation_row.rb new file mode 100644 index 00000000..f251f8b2 --- /dev/null +++ b/app/services/disaggregation_row.rb @@ -0,0 +1,54 @@ +class DisaggregationRow + attr_reader :row, :headers + + def initialize(row:, headers:) + @row = row + @headers = headers + end + + def district + @district ||= value_from(pattern: /District/i) + end + + def academic_year + @academic_year ||= value_from(pattern: /Academic\s*Year/i) + end + + def raw_income + @income ||= value_from(pattern: /Low\s*Income/i) + end + + def lasid + @lasid ||= value_from(pattern: /LASID/i) + end + + def raw_ell + @raw_ell ||= value_from(pattern: /EL Student First Year/i) + end + + def ell + @ell ||= begin + value = value_from(pattern: /EL Student First Year/i).downcase + + case value + when /lep student 1st year|LEP student not 1st year/i + "ELL" + when /Does not apply/i + "Not ELL" + else + "Unknown" + end + end + end + + def value_from(pattern:) + output = nil + matches = headers.select do |header| + pattern.match(header) + end.map { |item| item.delete("\n") } + matches.each do |match| + output ||= row[match] + end + output + end +end diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index 4fb9f1c1..e27f12dd 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -9,11 +9,12 @@ class SurveyItemValues @genders = genders @survey_items = survey_items @schools = schools - @disaggregation_data = disaggregation_data copy_likert_scores_from_variant_survey_items row["Income"] = income row["Raw Income"] = raw_income + row["Raw ELL"] = raw_ell + row["ELL"] = ell copy_data_to_main_column(main: /Race/i, secondary: /Race Secondary|Race-1/i) copy_data_to_main_column(main: /Gender/i, secondary: /Gender Secondary|Gender-1/i) @@ -134,20 +135,9 @@ class SurveyItemValues def raw_income @raw_income ||= value_from(pattern: /Low\s*Income|Raw\s*Income/i) - return @raw_income if @raw_income.present? - - return "Unknown" unless disaggregation_data.present? - - disaggregation = disaggregation_data[[lasid, district.name, academic_year.range]] - return "Unknown" unless disaggregation.present? - - @raw_income ||= disaggregation.income end def income - @income ||= value_from(pattern: /^Income$/i) - return @income if @income.present? - @income ||= case raw_income in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income/i "Economically Disadvantaged - Y" @@ -158,6 +148,21 @@ class SurveyItemValues end end + def raw_ell + @raw_ell ||= value_from(pattern: /EL Student First Year|Raw\s*ELL/i) + end + + def ell + @ell ||= case raw_ell + in /lep student 1st year|LEP student not 1st year|EL Student First Year/i + "ELL" + in /Does not apply/i + "Not ELL" + else + "Unknown" + end + end + def value_from(pattern:) output = nil matches = headers.select do |header| diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index 8f5f2a3a..723ceb86 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -1,31 +1,25 @@ # frozen_string_literal: true class SurveyResponsesDataLoader - def self.load_data(filepath:, rules: [Rule::NoRule]) + def load_data(filepath:, rules: [Rule::NoRule]) File.open(filepath) do |file| headers = file.first headers_array = CSV.parse(headers).first - genders = Gender.gender_hash - schools = School.school_hash - incomes = Income.by_designation all_survey_items = survey_items(headers:) file.lazy.each_slice(500) do |lines| survey_item_responses = CSV.parse(lines.join, headers:).map do |row| process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:), - rules:, incomes:) + rules:) end SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500 end end end - def self.from_file(file:, rules: []) + def from_file(file:, rules: []) headers = file.gets headers_array = CSV.parse(headers).first - genders = Gender.gender_hash - schools = School.school_hash - incomes = Income.by_designation all_survey_items = survey_items(headers:) survey_item_responses = [] @@ -36,7 +30,7 @@ class SurveyResponsesDataLoader CSV.parse(line, headers:).map do |row| survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, genders:, survey_items: all_survey_items, schools:), - rules:, incomes:) + rules:) end row_count += 1 @@ -52,7 +46,23 @@ class SurveyResponsesDataLoader private - def self.process_row(row:, rules:, incomes:) + def schools + @schools = School.school_hash + end + + def genders + @genders = Gender.by_qualtrics_code + end + + def incomes + @incomes ||= Income.by_slug + end + + def ells + @ells ||= Ell.by_designation + end + + def process_row(row:, rules:) return unless row.dese_id? return unless row.school.present? @@ -60,10 +70,10 @@ class SurveyResponsesDataLoader return if rule.new(row:).skip_row? end - process_survey_items(row:, incomes:) + process_survey_items(row:) end - def self.process_survey_items(row:, incomes:) + def process_survey_items(row:) row.survey_items.map do |survey_item| likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next @@ -72,38 +82,33 @@ class SurveyResponsesDataLoader next end response = row.survey_item_response(survey_item:) - create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, incomes:) + create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:) end.compact end - def self.create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, incomes:) + def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:) gender = row.gender grade = row.grade - income = incomes[row.income] + income = incomes[row.income.parameterize] + ell = ells[row.ell] if survey_item_response.present? - survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:) + survey_item_response.update!(likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:) [] else SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:, - likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:) + likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:) end end - def self.survey_items(headers:) + def survey_items(headers:) SurveyItem.where(survey_item_id: get_survey_item_ids_from_headers(headers:)) end - def self.get_survey_item_ids_from_headers(headers:) + def get_survey_item_ids_from_headers(headers:) CSV.parse(headers).first .filter(&:present?) .filter { |header| header.start_with? "t-", "s-" } end - - private_class_method :process_row - private_class_method :process_survey_items - private_class_method :create_or_update_response - private_class_method :survey_items - private_class_method :get_survey_item_ids_from_headers end module StringMonkeyPatches diff --git a/app/views/analyze/_group_selectors.html.erb b/app/views/analyze/_group_selectors.html.erb index 72bd8d38..c4e58603 100644 --- a/app/views/analyze/_group_selectors.html.erb +++ b/app/views/analyze/_group_selectors.html.erb @@ -21,3 +21,7 @@ <% @presenter.incomes.each do |income| %> <%= render(partial: "checkboxes", locals: {id: "income-#{income.slug}", item: income, selected_items: @presenter.selected_incomes, name: "income", label_text: income.designation}) %> <% end %> + +<% @presenter.ells.each do |ell| %> + <%= render(partial: "checkboxes", locals: {id: "ell-#{ell.slug}", item: ell, selected_items: @presenter.selected_ells, name: "ell", label_text: ell.designation}) %> +<% end %> diff --git a/app/views/analyze/index.html.erb b/app/views/analyze/index.html.erb index 63ec8a1f..84737b19 100644 --- a/app/views/analyze/index.html.erb +++ b/app/views/analyze/index.html.erb @@ -13,7 +13,7 @@ <%= render partial: "school_years", locals: {available_academic_years: @presenter.academic_years, selected_academic_years: @presenter.selected_academic_years, district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory, measures: @presenter.measures, graph: @presenter.graph} %> <%= render partial: "data_filters", locals: {district: @district, school: @school, academic_year: @academic_year, category: @presenter.category, subcategory: @presenter.subcategory} %> - <% cache [@presenter.subcategory, @school, @presenter.selected_academic_years, @presenter.graph, @presenter.selected_races, @presenter.race_score_timestamp, @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders] do %> + <% cache [@presenter.subcategory, @school, @presenter.selected_academic_years, @presenter.graph, @presenter.selected_races, @presenter.selected_grades, @presenter.grades, @presenter.selected_genders, @presenter.genders, @presenter.selected_ells, @presenter.ells] do %>
<% @presenter.measures.each do |measure| %>
diff --git a/data/demographics.csv b/data/demographics.csv index 3a09ce00..f95115e8 100644 --- a/data/demographics.csv +++ b/data/demographics.csv @@ -1,11 +1,11 @@ -Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income -1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged - N -2,Asian or Pacific Islander,1,Female,Economically Disadvantaged - Y -3,Black or African American,4,Non-Binary,Unknown -4,Hispanic or Latinx,99,Unknown, -5,White or Caucasian,,, -6,Prefer not to disclose,,, -7,Prefer to self-describe,,, -8,Middle Eastern,,, -99,Race/Ethnicity Not Listed,,, -100,Multiracial,,, +Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL +1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged - N,ELL +2,Asian or Pacific Islander,1,Female,Economically Disadvantaged - Y,Not ELL +3,Black or African American,4,Non-Binary,Unknown,Unknown +4,Hispanic or Latinx,99,Unknown,, +5,White or Caucasian,,,, +6,Prefer not to disclose,,,, +7,Prefer to self-describe,,,, +8,Middle Eastern,,,, +99,Race/Ethnicity Not Listed,,,, +100,Multiracial,,,, diff --git a/db/migrate/20230830213521_create_ell.rb b/db/migrate/20230830213521_create_ell.rb new file mode 100644 index 00000000..d4dca8ae --- /dev/null +++ b/db/migrate/20230830213521_create_ell.rb @@ -0,0 +1,12 @@ +class CreateEll < ActiveRecord::Migration[7.0] + def change + create_table :ells do |t| + t.string :designation + t.string :slug + + t.timestamps + end + + add_index :ells, :designation, unique: true + end +end diff --git a/db/migrate/20230912223701_add_ell_to_survey_item_response.rb b/db/migrate/20230912223701_add_ell_to_survey_item_response.rb new file mode 100644 index 00000000..026bcac9 --- /dev/null +++ b/db/migrate/20230912223701_add_ell_to_survey_item_response.rb @@ -0,0 +1,5 @@ +class AddEllToSurveyItemResponse < ActiveRecord::Migration[7.0] + def change + add_reference :survey_item_responses, :ell, foreign_key: true + end +end diff --git a/db/schema.rb b/db/schema.rb index c2da769f..71d8bf6c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 20_230_630_215_110) do +ActiveRecord::Schema[7.0].define(version: 2023_09_12_223701) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -69,6 +69,14 @@ ActiveRecord::Schema[7.0].define(version: 20_230_630_215_110) do t.datetime "updated_at", null: false end + create_table "ells", force: :cascade do |t| + t.string "designation" + t.string "slug" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["designation"], name: "index_ells_on_designation", unique: true + end + create_table "genders", force: :cascade do |t| t.integer "qualtrics_code" t.string "designation" @@ -352,7 +360,7 @@ ActiveRecord::Schema[7.0].define(version: 20_230_630_215_110) do t.integer "eleven" t.integer "twelve" t.index ["academic_year_id"], name: "index_respondents_on_academic_year_id" - t.index %w[school_id academic_year_id], name: "index_respondents_on_school_id_and_academic_year_id", unique: true + t.index ["school_id", "academic_year_id"], name: "index_respondents_on_school_id_and_academic_year_id", unique: true end create_table "response_rates", force: :cascade do |t| @@ -366,7 +374,7 @@ ActiveRecord::Schema[7.0].define(version: 20_230_630_215_110) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["academic_year_id"], name: "index_response_rates_on_academic_year_id" - t.index %w[school_id subcategory_id], name: "index_response_rates_on_school_id_and_subcategory_id" + t.index ["school_id", "subcategory_id"], name: "index_response_rates_on_school_id_and_subcategory_id" t.index ["school_id"], name: "index_response_rates_on_school_id" t.index ["subcategory_id"], name: "index_response_rates_on_subcategory_id" end @@ -419,7 +427,7 @@ ActiveRecord::Schema[7.0].define(version: 20_230_630_215_110) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["race_id"], name: "index_student_races_on_race_id" - t.index %w[student_id race_id], name: "index_student_races_on_student_id_and_race_id" + t.index ["student_id", "race_id"], name: "index_student_races_on_student_id_and_race_id" t.index ["student_id"], name: "index_student_races_on_student_id" end @@ -453,12 +461,16 @@ ActiveRecord::Schema[7.0].define(version: 20_230_630_215_110) do t.integer "grade" t.bigint "gender_id" t.datetime "recorded_date" + t.bigint "ell_id" + t.bigint "income_id" t.index ["academic_year_id"], name: "index_survey_item_responses_on_academic_year_id" + t.index ["ell_id"], name: "index_survey_item_responses_on_ell_id" t.index ["gender_id"], name: "index_survey_item_responses_on_gender_id" + t.index ["income_id"], name: "index_survey_item_responses_on_income_id" t.index ["response_id"], name: "index_survey_item_responses_on_response_id" - t.index %w[school_id academic_year_id survey_item_id], name: "by_school_year_and_survey_item" - t.index %w[school_id academic_year_id], name: "index_survey_item_responses_on_school_id_and_academic_year_id" - t.index %w[school_id survey_item_id academic_year_id grade], name: "index_survey_responses_on_grade" + t.index ["school_id", "academic_year_id", "survey_item_id"], name: "by_school_year_and_survey_item" + t.index ["school_id", "academic_year_id"], name: "index_survey_item_responses_on_school_id_and_academic_year_id" + t.index ["school_id", "survey_item_id", "academic_year_id", "grade"], name: "index_survey_responses_on_grade" t.index ["school_id"], name: "index_survey_item_responses_on_school_id" t.index ["student_id"], name: "index_survey_item_responses_on_student_id" t.index ["survey_item_id"], name: "index_survey_item_responses_on_survey_item_id" @@ -508,7 +520,9 @@ ActiveRecord::Schema[7.0].define(version: 20_230_630_215_110) do add_foreign_key "student_races", "students" add_foreign_key "subcategories", "categories" add_foreign_key "survey_item_responses", "academic_years" + add_foreign_key "survey_item_responses", "ells" add_foreign_key "survey_item_responses", "genders" + add_foreign_key "survey_item_responses", "incomes" add_foreign_key "survey_item_responses", "schools" add_foreign_key "survey_item_responses", "students" add_foreign_key "survey_item_responses", "survey_items" diff --git a/lib/tasks/clean.rake b/lib/tasks/clean.rake index ef8ab445..87223190 100644 --- a/lib/tasks/clean.rake +++ b/lib/tasks/clean.rake @@ -1,33 +1,33 @@ namespace :clean do # These tasks must be run in their respective project so the correct schools are in the database - desc 'clean ecp data' + desc "clean ecp data" task ecp: :environment do - input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'raw') - output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'clean') - log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'removed') + input_filepath = Rails.root.join("tmp", "data", "ecp_data", "raw") + output_filepath = Rails.root.join("tmp", "data", "ecp_data", "clean") + log_filepath = Rails.root.join("tmp", "data", "ecp_data", "removed") Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean end - desc 'clean prepped data' + desc "clean prepped data" task prepped: :environment do - input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped') - output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'clean') - log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'removed') + input_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped") + output_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "clean") + log_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "removed") Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean end - desc 'clean mciea data' + desc "clean mciea data" task mciea: :environment do - input_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'raw') - output_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'clean') - log_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'removed') + input_filepath = Rails.root.join("tmp", "data", "mciea_data", "raw") + output_filepath = Rails.root.join("tmp", "data", "mciea_data", "clean") + log_filepath = Rails.root.join("tmp", "data", "mciea_data", "removed") Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean end - desc 'clean rpp data' + desc "clean rpp data" task rpp: :environment do - input_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'raw') - output_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'clean') - log_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'removed') + input_filepath = Rails.root.join("tmp", "data", "rpp_data", "raw") + output_filepath = Rails.root.join("tmp", "data", "rpp_data", "clean") + log_filepath = Rails.root.join("tmp", "data", "rpp_data", "removed") Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean end end diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake index 1d88bc56..4075a32d 100644 --- a/lib/tasks/data.rake +++ b/lib/tasks/data.rake @@ -29,7 +29,7 @@ namespace :data do student_count = Student.count path = '/data/survey_responses/clean/' Sftp::Directory.open(path:) do |file| - SurveyResponsesDataLoader.from_file(file:) + SurveyResponsesDataLoader.new.from_file(file:) end puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" diff --git a/lib/tasks/one_off.rake b/lib/tasks/one_off.rake index 0d9da30d..8441efc7 100644 --- a/lib/tasks/one_off.rake +++ b/lib/tasks/one_off.rake @@ -82,27 +82,6 @@ namespace :one_off do puts values end - desc "load survey responses for lowell schools" - task load_survey_responses_for_lowell: :environment do - survey_item_response_count = SurveyItemResponse.count - student_count = Student.count - Sftp::Directory.open(path: "/test/survey_responses/") do |file| - SurveyResponsesDataLoader.from_file(file:) - end - puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" - - Sftp::Directory.open(path: "/test/survey_responses/") do |file| - StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools]) - end - puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students" - - puts "Resetting race scores" - RaceScoreLoader.reset(fast_processing: false) - puts "=====================> Completed loading #{RaceScore.count} race scores" - - Rails.cache.clear - end - desc "delete 2022-23 survey responses" task delete_survey_responses_2022_23: :environment do response_count = SurveyItemResponse.all.count @@ -127,7 +106,7 @@ namespace :one_off do path = "/data/survey_responses/2022-23/" Sftp::Directory.open(path:) do |file| - SurveyResponsesDataLoader.from_file(file:) + SurveyResponsesDataLoader.new.from_file(file:) end puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" diff --git a/spec/factories.rb b/spec/factories.rb index a96f0e5d..28b34ece 100644 --- a/spec/factories.rb +++ b/spec/factories.rb @@ -1,11 +1,15 @@ FactoryBot.define do factory :income do - designation { "MyString" } + designation { "DefaultIncome" } + end + + factory :ell do + designation { "DefaultEll" } end factory :gender do qualtrics_code { 1 } - designation { 'MyString' } + designation { "MyString" } end factory :race_score do @@ -68,22 +72,22 @@ FactoryBot.define do end factory :academic_year do - range { '2050-51' } + range { "2050-51" } initialize_with { AcademicYear.find_or_initialize_by(range:) } end - factory :category, class: 'Category' do + factory :category, class: "Category" do name { "A #{rand} category" } category_id { rand.to_s } - description { 'A description of a category' } + description { "A description of a category" } slug { name.parameterize } sort_index { 1 } end factory :subcategory do - name { 'A subcategory' } + name { "A subcategory" } subcategory_id { rand.to_s } - description { 'A description of a subcategory' } + description { "A description of a subcategory" } category factory :subcategory_with_measures do @@ -102,7 +106,7 @@ FactoryBot.define do factory :measure do measure_id { rand.to_s } - name { 'A Measure' } + name { "A Measure" } subcategory trait :with_student_survey_items do after(:create) do |measure| @@ -136,7 +140,7 @@ FactoryBot.define do factory :survey_item do scale - prompt { 'What do YOU think?' } + prompt { "What do YOU think?" } factory :teacher_survey_item do survey_item_id { "t-#{rand}" } watch_low_benchmark { 2.0 } diff --git a/spec/fixtures/sample_demographics.csv b/spec/fixtures/sample_demographics.csv index c60cb690..5444303e 100644 --- a/spec/fixtures/sample_demographics.csv +++ b/spec/fixtures/sample_demographics.csv @@ -1,11 +1,11 @@ -Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income -1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged – N -2,Asian or Pacific Islander,1,Female,Economically Disadvantaged – Y -3,Black or African American,4,Non-Binary,Unknown -4,Hispanic or Latinx,99,Unknown, -5,White or Caucasian,,, -6,Prefer not to disclose,,, -7,Prefer to self-describe,,, -8,Middle Eastern,,, -99,Race/Ethnicity Not Listed,,, -100,Multiracial,,, +Race Qualtrics Code,Race/Ethnicity,Gender Qualtrics Code,Sex/Gender,Income,ELL +1,American Indian or Alaskan Native,2,Male,Economically Disadvantaged – N,ELL +2,Asian or Pacific Islander,1,Female,Economically Disadvantaged – Y,Not ELL +3,Black or African American,4,Non-Binary,Unknown,Unknown +4,Hispanic or Latinx,99,Unknown,, +5,White or Caucasian,,,, +6,Prefer not to disclose,,,, +7,Prefer to self-describe,,,, +8,Middle Eastern,,,, +99,Race/Ethnicity Not Listed,,,, +100,Multiracial,,,, diff --git a/spec/fixtures/test_2020-21_student_survey_responses.csv b/spec/fixtures/test_2020-21_student_survey_responses.csv index 0ba7eb03..21cb3b2b 100644 --- a/spec/fixtures/test_2020-21_student_survey_responses.csv +++ b/spec/fixtures/test_2020-21_student_survey_responses.csv @@ -1,9 +1,8 @@ -Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,LASID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district,school,DESE ID,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,#N/A,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,#N/A,#N/A,s-cure-q3,s-cure-q4,#N/A,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,#N/A,#N/A,s-grmi-q4,s-appa-q1,s-appa-q2,#N/A,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q1-1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-acst-q4,s-acst-q5,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,race,What is your race/ethnicity?(Please select all that apply) - Selected Choice,grade,gender,Raw Income,Income -2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,1500025,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888,11th,1,Free Lunch,Economically Disadvantaged – Y -2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,"2,3,4",888,10,,Not Eligible,Economically Disadvantaged – N -2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1500505,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,6,888,8,2,Reduced Lunch,Economically Disadvantaged – Y -2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1500505,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888,8,3,,Unknown -2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888,7,4,Free Lunch,Economically Disadvantaged – Y -2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888,3,NA,Not Eligible,Economically Disadvantaged – N -2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,,,4,,Reduced Lunch,Economically Disadvantaged – Y -,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"1,2,3,4,5,8",,,,, +Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,LASID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district,school,DESE ID,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,#N/A,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,#N/A,#N/A,s-cure-q3,s-cure-q4,#N/A,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,#N/A,#N/A,s-grmi-q4,s-appa-q1,s-appa-q2,#N/A,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q1-1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-acst-q4,s-acst-q5,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,race,What is your race/ethnicity?(Please select all that apply) - Selected Choice,grade,gender,Raw Income,Income,Raw ELL,ELL +2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,1500025,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888,11th,1,Free Lunch,Economically Disadvantaged – Y,Does not apply,Not ELL +2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,"2,3,4",888,10,,Not Eligible,Economically Disadvantaged – N,,Unknown +2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1500505,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,6,888,8,2,Reduced Lunch,Economically Disadvantaged – Y,,Unknown +2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1500505,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888,8,3,,Unknown,LEP student not 1st year,ELL +2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888,7,4,Free Lunch,Economically Disadvantaged – Y,EL Student First Year,ELL +2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888,3,NA,Not Eligible,Economically Disadvantaged – N,Unknown,Unknown +2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,,,4,,Reduced Lunch,Economically Disadvantaged – Y,#N/A,Unknown diff --git a/spec/services/cleaner_spec.rb b/spec/services/cleaner_spec.rb index 3c358e45..24cd6658 100644 --- a/spec/services/cleaner_spec.rb +++ b/spec/services/cleaner_spec.rb @@ -2,13 +2,18 @@ require "rails_helper" require "fileutils" RSpec.describe Cleaner do - let(:district) { create(:district, name: "District1") } + let(:district) { create(:district, name: "Maynard Public Schools") } let(:second_district) { create(:district, name: "District2") } let(:school) { create(:school, dese_id: 1_740_505, district:) } - let(:second_school) { create(:school, dese_id: 222_222, district: second_district) } + let(:second_school) { create(:school, dese_id: 1_740_305, district:) } + let(:third_school) { create(:school, dese_id: 222_222, district: second_district) } let(:academic_year) { create(:academic_year, range: "2022-23") } - let(:respondents) { create(:respondent, school:, academic_year:, nine: 40, ten: 40, eleven: 40, twelve: 40) } + let(:respondents) do + create(:respondent, school:, academic_year:, one: 0, nine: 40, ten: 40, eleven: 40, twelve: 40) + create(:respondent, school: second_school, academic_year:, one: 0, four: 40, five: 40, six: 40, seven: 40, + eight: 40) + end let(:recorded_date) { "2023-04-01" } let(:input_filepath) do Rails.root.join("spec", "fixtures", "raw") @@ -22,6 +27,10 @@ RSpec.describe Cleaner do Rails.root.join("tmp", "spec", "removed") end + let(:path_to_sample_raw_file) do + File.open(Rails.root.join("spec", "fixtures", "raw", "sample_maynard_raw_student_survey.csv")) + end + let(:common_headers) do ["Recorded Date", "Dese ID", "ResponseID"] end @@ -71,6 +80,7 @@ RSpec.describe Cleaner do before :each do school second_school + third_school standard_survey_items short_form_survey_items early_education_survey_items @@ -93,9 +103,9 @@ RSpec.describe Cleaner do context ".process_raw_file" do it "sorts data into valid and invalid csvs" do - cleaner = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:) + cleaner = Cleaner.new(input_filepath:, output_filepath:, log_filepath:) processed_data = cleaner.process_raw_file( - file: path_to_sample_raw_file, disaggregation_data: cleaner.disaggregation_data + file: path_to_sample_raw_file ) processed_data in [headers, clean_csv, log_csv, data] @@ -122,22 +132,6 @@ RSpec.describe Cleaner do csv_contains_the_correct_rows(log_csv, invalid_rows) invalid_rows_are_rejected_for_the_correct_reasons(data) end - - it "adds dissaggregation data to the cleaned file " do - cleaner = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:) - processed_data = cleaner.process_raw_file( - file: path_to_sample_raw_file, disaggregation_data: cleaner.disaggregation_data - ) - processed_data in [headers, clean_csv, log_csv, data] - index_of_income = clean_csv.first.index("Income") - expect(clean_csv.second[index_of_income]).to eq "Economically Disadvantaged - Y" - - one_thousand = data.find { |row| row.response_id == "1000" } - expect(one_thousand.income).to eq "Economically Disadvantaged - Y" - - one_thousand_one = data.find { |row| row.response_id == "1001" } - expect(one_thousand_one.income).to eq "Economically Disadvantaged - N" - end end context ".filename" do @@ -151,7 +145,7 @@ RSpec.describe Cleaner do filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( headers: standard_survey_items, data: ) - expect(filename).to eq "District1.standard.2022-23.csv" + expect(filename).to eq "maynard.standard.2022-23.csv" end context "when the file is based on short form survey items" do @@ -163,7 +157,7 @@ RSpec.describe Cleaner do filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( headers: short_form_survey_items, data: ) - expect(filename).to eq "District1.short_form.2022-23.csv" + expect(filename).to eq "maynard.short_form.2022-23.csv" end end @@ -176,7 +170,7 @@ RSpec.describe Cleaner do filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( headers: early_education_survey_items, data: ) - expect(filename).to eq "District1.early_education.2022-23.csv" + expect(filename).to eq "maynard.early_education.2022-23.csv" end end context "when the file is based on teacher survey items" do @@ -188,7 +182,7 @@ RSpec.describe Cleaner do filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( headers: teacher_survey_items, data: ) - expect(filename).to eq "District1.teacher.2022-23.csv" + expect(filename).to eq "maynard.teacher.2022-23.csv" end end @@ -202,7 +196,7 @@ RSpec.describe Cleaner do filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( headers: teacher_survey_items, data: ) - expect(filename).to eq "District1.District2.teacher.2022-23.csv" + expect(filename).to eq "maynard.district2.teacher.2022-23.csv" end end end @@ -212,7 +206,7 @@ end def reads_headers_from_raw_csv(processed_data) processed_data in [headers, clean_csv, log_csv, data] - expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", + expect(headers.to_set.sort).to eq ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", "Finished", "RecordedDate", "ResponseId", "District", "School", "LASID", "Gender", "Race", "What grade are you in?", "s-emsa-q1", "s-emsa-q2", "s-emsa-q3", "s-tint-q1", "s-tint-q2", "s-tint-q3", "s-tint-q4", "s-tint-q5", "s-acpr-q1", "s-acpr-q2", @@ -225,7 +219,7 @@ def reads_headers_from_raw_csv(processed_data) "s-grit-q1", "s-grit-q2", "s-grit-q3", "s-grit-q4", "s-expa-q1", "s-poaf-q1", "s-poaf-q2", "s-poaf-q3", "s-poaf-q4", "s-tint-q1-1", "s-tint-q2-1", "s-tint-q3-1", "s-tint-q4-1", "s-tint-q5-1", "s-acpr-q1-1", "s-acpr-q2-1", "s-acpr-q3-1", "s-acpr-q4-1", "s-peff-q1-1", "s-peff-q2-1", "s-peff-q3-1", "s-peff-q4-1", - "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income"].to_set.sort + "s-peff-q5-1", "s-peff-q6-1", "Raw Income", "Income", "Raw ELL", "ELL", "Raw SpEd", "SpEd"].to_set.sort end def invalid_rows_are_rejected_for_the_correct_reasons(data) @@ -308,3 +302,4 @@ def csv_contains_the_correct_rows(csv, rows) expect(csv[index + 1][response_id]).to eq row end end + diff --git a/spec/services/demographic_loader_spec.rb b/spec/services/demographic_loader_spec.rb index 1169e586..c314da6b 100644 --- a/spec/services/demographic_loader_spec.rb +++ b/spec/services/demographic_loader_spec.rb @@ -1,20 +1,24 @@ -require 'rails_helper' +require "rails_helper" describe DemographicLoader do - let(:filepath) { 'spec/fixtures/sample_demographics.csv' } + let(:filepath) { "spec/fixtures/sample_demographics.csv" } let(:race_codes) do - { 'American Indian or Alaskan Native' => 1, 'Asian or Pacific Islander' => 2, 'Black or African American' => 3, - 'Hispanic or Latinx' => 4, 'White or Caucasian' => 5, 'Race/Ethnicity Not Listed' => 99, 'Middle Eastern' => 8, 'Multiracial' => 100 } + { "American Indian or Alaskan Native" => 1, "Asian or Pacific Islander" => 2, "Black or African American" => 3, + "Hispanic or Latinx" => 4, "White or Caucasian" => 5, "Race/Ethnicity Not Listed" => 99, "Middle Eastern" => 8, "Multiracial" => 100 } end let(:gender_codes) do { - 'Female' => 1, 'Male' => 2, 'Non-Binary' => 4, 'Unknown' => 99 + "Female" => 1, "Male" => 2, "Non-Binary" => 4, "Unknown" => 99 } end let(:incomes) do - ['Economically Disadvantaged – N', 'Economically Disadvantaged – Y', 'Unknown'] + ["Economically Disadvantaged – N", "Economically Disadvantaged – Y", "Unknown"] + end + + let(:ells) do + ["ELL", "Not ELL", "Unknown"] end before :each do @@ -25,12 +29,12 @@ describe DemographicLoader do DatabaseCleaner.clean end - describe 'self.load_data' do - it 'does not load qualtrics categories for `prefer not to disclose` or `prefer to self-describe`' do + describe "self.load_data" do + it "does not load qualtrics categories for `prefer not to disclose` or `prefer to self-describe`" do expect(Race.find_by_qualtrics_code(6)).to be nil end - it 'loads all racial designations' do + it "loads all racial designations" do expect(Race.all.count).to eq 8 race_codes.each do |key, value| expect(Race.find_by_qualtrics_code(value)).not_to eq nil @@ -40,7 +44,7 @@ describe DemographicLoader do end end - it 'loads all gender designations' do + it "loads all gender designations" do expect(Gender.all.count).to eq 4 gender_codes.each do |key, value| @@ -51,11 +55,18 @@ describe DemographicLoader do end end - it 'loads all the income designations' do + it "loads all the income designations" do expect(Income.all.count).to eq 3 incomes.each do |income| expect(Income.find_by_designation(income).designation).to eq income end end + + it "loads all the ells designations" do + expect(Ell.all.count).to eq 3 + ells.each do |ell| + expect(Ell.find_by_designation(ell).designation).to eq ell + end + end end end diff --git a/spec/services/disaggregation_loader_spec.rb b/spec/services/disaggregation_loader_spec.rb new file mode 100644 index 00000000..d8ff718e --- /dev/null +++ b/spec/services/disaggregation_loader_spec.rb @@ -0,0 +1,39 @@ +require "rails_helper" +require "fileutils" + +RSpec.describe DisaggregationLoader do + let(:path) do + Rails.root.join("spec", "fixtures", "disaggregation") + end + let(:academic_year) { create(:academic_year, range: "2022-23") } + let(:district) { create(:district, name: "Maynard Public Schools") } + context ".load" do + it "loads data from the file into a hash" do + data = DisaggregationLoader.new(path:).load + expect(data.values.first.lasid).to eq("1") + expect(data.values.first.academic_year).to eq("2022-23") + expect(data.values.first.district).to eq("Maynard Public Schools") + + expect(data.values.last.lasid).to eq("500") + expect(data.values.last.academic_year).to eq("2022-23") + expect(data.values.last.district).to eq("Maynard Public Schools") + end + + it "loads income data" do + data = DisaggregationLoader.new(path:).load + expect(data.values.first.raw_income).to eq("Free Lunch") + expect(data.values.last.raw_income).to eq("Not Eligible") + + expect(data[["1", "Maynard Public Schools", "2022-23"]].raw_income).to eq("Free Lunch") + expect(data[["2", "Maynard Public Schools", "2022-23"]].raw_income).to eq("Not Eligible") + expect(data[["3", "Maynard Public Schools", "2022-23"]].raw_income).to eq("Reduced Lunch") + end + end + + context "Creating a new loader" do + it "creates a directory for the loader file" do + DisaggregationLoader.new(path:) + expect(path).to exist + end + end +end diff --git a/spec/services/disaggregation_row_spec.rb b/spec/services/disaggregation_row_spec.rb new file mode 100644 index 00000000..fbebf064 --- /dev/null +++ b/spec/services/disaggregation_row_spec.rb @@ -0,0 +1,108 @@ +require "rails_helper" + +RSpec.describe DisaggregationRow do + let(:headers) do + ["District", "Academic Year", "LASID", "HispanicLatino", "Race", "Gender", "SpecialEdStatus", "In 504 Plan", + "LowIncome", "EL Student First Year"] + end + + context ".district" do + context "when the column heading is any upper or lowercase variant of the word district" do + it "returns the correct value for district" do + row = { "District" => "Maynard Public Schools" } + expect(DisaggregationRow.new(row:, headers:).district).to eq "Maynard Public Schools" + + headers = ["dISTRICT"] + headers in [district] + row = { district => "Maynard Public Schools" } + expect(DisaggregationRow.new(row:, headers:).district).to eq "Maynard Public Schools" + end + end + end + + context ".academic_year" do + context "when the column heading is any upper or lowercase variant of the words academic year" do + it "returns the correct value for district" do + row = { "Academic Year" => "2022-23" } + expect(DisaggregationRow.new(row:, headers:).academic_year).to eq "2022-23" + + headers = ["aCADEMIC yEAR"] + headers in [academic_year] + row = { academic_year => "2022-23" } + expect(DisaggregationRow.new(row:, headers:).academic_year).to eq "2022-23" + + headers = ["AcademicYear"] + headers in [academic_year] + row = { academic_year => "2022-23" } + expect(DisaggregationRow.new(row:, headers:).academic_year).to eq "2022-23" + end + end + end + + context ".raw_income" do + context "when the column heading is any upper or lowercase variant of the words low income" do + it "returns the correct value for low_income" do + row = { "LowIncome" => "Free Lunch" } + expect(DisaggregationRow.new(row:, headers:).raw_income).to eq "Free Lunch" + + headers = ["Low income"] + headers in [income] + row = { income => "Free Lunch" } + expect(DisaggregationRow.new(row:, headers:).raw_income).to eq "Free Lunch" + + headers = ["LoW InCOme"] + headers in [income] + row = { income => "Free Lunch" } + expect(DisaggregationRow.new(row:, headers:).raw_income).to eq "Free Lunch" + end + end + end + + context ".lasid" do + context "when the column heading is any upper or lowercase variant of the words lasid" do + it "returns the correct value for lasid" do + row = { "LASID" => "2366" } + expect(DisaggregationRow.new(row:, headers:).lasid).to eq "2366" + + headers = ["LaSiD"] + headers in [lasid] + row = { lasid => "2366" } + expect(DisaggregationRow.new(row:, headers:).lasid).to eq "2366" + end + end + end + + context ".ell" do + context "when the column heading is any upper or lowercase variant of the words 'ELL' or 'El Student First Year'" do + it "returns the correct value for a student" do + row = { "EL Student First Year" => "LEP student 1st year" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "ELL" + + headers = ["EL Student First Year"] + headers in [ell] + row = { ell => "LEP student not 1st year" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "ELL" + + headers = ["EL Student First Year"] + headers in [ell] + row = { ell => "Does not apply" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "Not ELL" + + headers = ["EL Student First Year"] + headers in [ell] + row = { ell => "Unknown" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "Unknown" + + headers = ["EL Student First Year"] + headers in [ell] + row = { ell => "Any other text" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "Unknown" + + headers = ["EL Student First Year"] + headers in [ell] + row = { ell => "" } + expect(DisaggregationRow.new(row:, headers:).ell).to eq "Unknown" + end + end + end +end diff --git a/spec/services/student_loader_spec.rb b/spec/services/student_loader_spec.rb index e78a35fb..e998ea7f 100644 --- a/spec/services/student_loader_spec.rb +++ b/spec/services/student_loader_spec.rb @@ -88,7 +88,7 @@ describe StudentLoader do describe "self.load_data" do context "load student data for all schools" do before :each do - SurveyResponsesDataLoader.load_data filepath: path_to_student_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses StudentLoader.load_data filepath: path_to_student_responses end @@ -102,7 +102,7 @@ describe StudentLoader do # TODO: get this test to run correctly. Since we are no longer seeding, we need to define schools, and districts; some Lowell, some not xcontext "When using the rule to skip non Lowell schools" do before :each do - SurveyResponsesDataLoader.load_data filepath: path_to_student_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses StudentLoader.load_data filepath: path_to_student_responses, rules: [Rule::SkipNonLowellSchools] end diff --git a/spec/services/survey_item_values_spec.rb b/spec/services/survey_item_values_spec.rb index 608260f9..29950026 100644 --- a/spec/services/survey_item_values_spec.rb +++ b/spec/services/survey_item_values_spec.rb @@ -7,12 +7,7 @@ RSpec.describe SurveyItemValues, type: :model do end let(:genders) do create(:gender, qualtrics_code: 1) - gender_hash = {} - - Gender.all.each do |gender| - gender_hash[gender.qualtrics_code] = gender - end - gender_hash + Gender.by_qualtrics_code end let(:survey_items) { [] } let(:district) { create(:district, name: "Attleboro") } @@ -169,79 +164,90 @@ RSpec.describe SurveyItemValues, type: :model do end context ".income" do - context "when no disaggregation data is provided" do - it "returns an empty string " do - disaggregation_data = {} - values = SurveyItemValues.new(row: {}, headers:, genders:, survey_items:, schools:, disaggregation_data:) - expect(values.income).to eq "Unknown" - end + before :each do + attleboro + ay_2022_23 end - context "when disaggregation data is provided" do - before :each do - attleboro - ay_2022_23 - end + it "translates Free Lunch to Economically Disadvantaged - Y" do + headers = ["LowIncome"] + row = { "LowIncome" => "Free Lunch" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.income).to eq "Economically Disadvantaged - Y" + end - it "translates Free Lunch to Economically Disadvantaged - Y" do - headers = ["District", "Academic Year", "LASID", "LowIncome"] - row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Free Lunch" } - disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) } + it "translates Reduced Lunch to Economically Disadvantaged - Y" do + headers = ["LowIncome"] + row = { "LowIncome" => "Reduced Lunch" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.income).to eq "Economically Disadvantaged - Y" + end - headers = ["LASID", "Dese Id", "RecordedDate"] - row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" } - values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, - disaggregation_data:) - expect(values.income).to eq "Economically Disadvantaged - Y" - end + it "translates LowIncome to Economically Disadvantaged - Y" do + headers = ["LowIncome"] + row = { "LowIncome" => "LowIncome" } - it "translates Reduced Lunch to Economically Disadvantaged - Y" do - headers = ["District", "Academic Year", "LASID", "LowIncome"] - row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Reduced Lunch" } - disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.income).to eq "Economically Disadvantaged - Y" + end - headers = ["LASID", "Dese Id", "RecordedDate"] - row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" } - values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, - disaggregation_data:) - expect(values.income).to eq "Economically Disadvantaged - Y" - end + it "translates Not Eligible to Economically Disadvantaged - N" do + headers = ["LowIncome"] + row = { "LowIncome" => "Not Eligible" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.income).to eq "Economically Disadvantaged - N" + end - it "translates LowIncome to Economically Disadvantaged - Y" do - headers = ["District", "Academic Year", "LASID", "LowIncome"] - row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "LowIncome" } - disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) } + it "translates blanks to Unknown" do + headers = ["LowIncome"] + row = { "LowIncome" => "" } - headers = ["LASID", "Dese Id", "RecordedDate"] - row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" } - values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, - disaggregation_data:) - expect(values.income).to eq "Economically Disadvantaged - Y" - end + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.income).to eq "Unknown" + end + end - it "translates Not Eligible to Economically Disadvantaged - N" do - headers = ["District", "Academic Year", "LASID", "LowIncome"] - row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Not Eligible" } - disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) } + context ".ell" do + before :each do + attleboro + ay_2022_23 + end - headers = ["LASID", "Dese Id", "RecordedDate"] - row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" } - values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, - disaggregation_data:) - expect(values.income).to eq "Economically Disadvantaged - N" - end + it 'translates "LEP Student 1st Year" or "LEP Student Not 1st Year" into ELL' do + headers = ["Raw ELL"] + row = { "Raw ELL" => "LEP Student 1st Year" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "ELL" - it "translates blanks to Unknown" do - headers = ["District", "Academic Year", "LASID", "LowIncome"] - row = { "District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "" } - disaggregation_data = { %w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:) } + row = { "Raw ELL" => "LEP Student Not 1st Year" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "ELL" - headers = ["LASID", "Dese Id", "RecordedDate"] - row = { "LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1" } - values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, - disaggregation_data:) - expect(values.income).to eq "Unknown" - end + row = { "Raw ELL" => "LEP Student Not 1st Year" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "ELL" + end + + it 'translates "Does not Apply" into "Not ELL"' do + headers = ["Raw ELL"] + row = { "Raw ELL" => "Does not apply" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "Not ELL" + + row = { "Raw ELL" => "Does Not APPLY" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "Not ELL" + end + + it 'tranlsates blanks into "Unknown"' do + headers = ["Raw ELL"] + row = { "Raw ELL" => "" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "Unknown" + + row = { "Raw ELL" => "Anything else" } + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) + expect(values.ell).to eq "Unknown" end end diff --git a/spec/services/survey_responses_data_loader_spec.rb b/spec/services/survey_responses_data_loader_spec.rb index 0558a6ce..d0d70fae 100644 --- a/spec/services/survey_responses_data_loader_spec.rb +++ b/spec/services/survey_responses_data_loader_spec.rb @@ -1,59 +1,62 @@ -require 'rails_helper' +require "rails_helper" describe SurveyResponsesDataLoader do - let(:path_to_teacher_responses) { Rails.root.join('spec', 'fixtures', 'test_2020-21_teacher_survey_responses.csv') } - let(:path_to_student_responses) { Rails.root.join('spec', 'fixtures', 'test_2020-21_student_survey_responses.csv') } + let(:path_to_teacher_responses) { Rails.root.join("spec", "fixtures", "test_2020-21_teacher_survey_responses.csv") } + let(:path_to_student_responses) { Rails.root.join("spec", "fixtures", "test_2020-21_student_survey_responses.csv") } let(:path_to_butler_student_responses) do - Rails.root.join('spec', 'fixtures', 'test_2022-23_butler_student_survey_responses.csv') + Rails.root.join("spec", "fixtures", "test_2022-23_butler_student_survey_responses.csv") end - let(:ay_2020_21) { create(:academic_year, range: '2020-21') } - let(:ay_2022_23) { create(:academic_year, range: '2022-23') } + let(:ay_2020_21) { create(:academic_year, range: "2020-21") } + let(:ay_2022_23) { create(:academic_year, range: "2022-23") } - let(:school) { create(:school, name: 'Lee Elementary School', slug: 'lee-elementary-school', dese_id: 1_500_025) } - let(:lowell) { create(:district, name: 'Lowell', slug: 'lowell') } + let(:school) { create(:school, name: "Lee Elementary School", slug: "lee-elementary-school", dese_id: 1_500_025) } + let(:lowell) { create(:district, name: "Lowell", slug: "lowell") } let(:second_school) do - create(:school, name: 'Lee Middle High School', slug: 'lee-middle-high-school', dese_id: 1_500_505, + create(:school, name: "Lee Middle High School", slug: "lee-middle-high-school", dese_id: 1_500_505, district: lowell) end let(:butler_school) do - create(:school, name: 'Butler Elementary School', slug: 'butler-elementary-school', dese_id: 1_600_310, + create(:school, name: "Butler Elementary School", slug: "butler-elementary-school", dese_id: 1_600_310, district: lowell) end - let(:t_pcom_q3) { create(:survey_item, survey_item_id: 't-pcom-q3') } - let(:t_pcom_q2) { create(:survey_item, survey_item_id: 't-pcom-q2') } - let(:t_coll_q1) { create(:survey_item, survey_item_id: 't-coll-q1') } - let(:t_coll_q2) { create(:survey_item, survey_item_id: 't-coll-q2') } - let(:t_coll_q3) { create(:survey_item, survey_item_id: 't-coll-q3') } - let(:t_sach_q1) { create(:survey_item, survey_item_id: 't-sach-q1') } - let(:t_sach_q2) { create(:survey_item, survey_item_id: 't-sach-q2') } - let(:t_sach_q3) { create(:survey_item, survey_item_id: 't-sach-q3') } - - let(:s_phys_q1) { create(:survey_item, survey_item_id: 's-phys-q1') } - let(:s_phys_q2) { create(:survey_item, survey_item_id: 's-phys-q2') } - let(:s_phys_q3) { create(:survey_item, survey_item_id: 's-phys-q3') } - let(:s_phys_q4) { create(:survey_item, survey_item_id: 's-phys-q4') } - let(:s_vale_q1) { create(:survey_item, survey_item_id: 's-phys-q1') } - let(:s_vale_q2) { create(:survey_item, survey_item_id: 's-phys-q2') } - let(:s_vale_q3) { create(:survey_item, survey_item_id: 's-phys-q3') } - let(:s_vale_q4) { create(:survey_item, survey_item_id: 's-phys-q4') } - let(:s_acst_q1) { create(:survey_item, survey_item_id: 's-acst-q1') } - let(:s_acst_q2) { create(:survey_item, survey_item_id: 's-acst-q2') } - let(:s_acst_q3) { create(:survey_item, survey_item_id: 's-acst-q3') } - let(:s_acst_q4) { create(:survey_item, survey_item_id: 's-acst-q4') } - let(:s_emsa_q1) { create(:survey_item, survey_item_id: 's-emsa-q1') } - let(:s_emsa_q2) { create(:survey_item, survey_item_id: 's-emsa-q2') } - let(:s_emsa_q3) { create(:survey_item, survey_item_id: 's-emsa-q3') } + let(:t_pcom_q3) { create(:survey_item, survey_item_id: "t-pcom-q3") } + let(:t_pcom_q2) { create(:survey_item, survey_item_id: "t-pcom-q2") } + let(:t_coll_q1) { create(:survey_item, survey_item_id: "t-coll-q1") } + let(:t_coll_q2) { create(:survey_item, survey_item_id: "t-coll-q2") } + let(:t_coll_q3) { create(:survey_item, survey_item_id: "t-coll-q3") } + let(:t_sach_q1) { create(:survey_item, survey_item_id: "t-sach-q1") } + let(:t_sach_q2) { create(:survey_item, survey_item_id: "t-sach-q2") } + let(:t_sach_q3) { create(:survey_item, survey_item_id: "t-sach-q3") } + + let(:s_phys_q1) { create(:survey_item, survey_item_id: "s-phys-q1") } + let(:s_phys_q2) { create(:survey_item, survey_item_id: "s-phys-q2") } + let(:s_phys_q3) { create(:survey_item, survey_item_id: "s-phys-q3") } + let(:s_phys_q4) { create(:survey_item, survey_item_id: "s-phys-q4") } + let(:s_vale_q1) { create(:survey_item, survey_item_id: "s-phys-q1") } + let(:s_vale_q2) { create(:survey_item, survey_item_id: "s-phys-q2") } + let(:s_vale_q3) { create(:survey_item, survey_item_id: "s-phys-q3") } + let(:s_vale_q4) { create(:survey_item, survey_item_id: "s-phys-q4") } + let(:s_acst_q1) { create(:survey_item, survey_item_id: "s-acst-q1") } + let(:s_acst_q2) { create(:survey_item, survey_item_id: "s-acst-q2") } + let(:s_acst_q3) { create(:survey_item, survey_item_id: "s-acst-q3") } + let(:s_acst_q4) { create(:survey_item, survey_item_id: "s-acst-q4") } + let(:s_emsa_q1) { create(:survey_item, survey_item_id: "s-emsa-q1") } + let(:s_emsa_q2) { create(:survey_item, survey_item_id: "s-emsa-q2") } + let(:s_emsa_q3) { create(:survey_item, survey_item_id: "s-emsa-q3") } let(:female) { create(:gender, qualtrics_code: 1) } let(:male) { create(:gender, qualtrics_code: 2) } let(:another_gender) { create(:gender, qualtrics_code: 3) } let(:non_binary) { create(:gender, qualtrics_code: 4) } let(:unknown_gender) { create(:gender, qualtrics_code: 99) } - let(:low_income) { create(:income, designation: 'Economically Disadvantaged – Y') } - let(:high_income) { create(:income, designation: 'Economically Disadvantaged – N') } - let(:unknown_income) { create(:income, designation: 'Unknown') } + let(:low_income) { create(:income, designation: "Economically Disadvantaged – Y") } + let(:high_income) { create(:income, designation: "Economically Disadvantaged – N") } + let(:unknown_income) { create(:income, designation: "Unknown") } + let(:yes_ell) { create(:ell, designation: "ELL") } + let(:not_ell) { create(:ell, designation: "Not ELL") } + let(:unknown_ell) { create(:ell, designation: "Unknown") } let(:setup) do ay_2020_21 @@ -92,18 +95,21 @@ describe SurveyResponsesDataLoader do low_income high_income unknown_income + yes_ell + not_ell + unknown_ell end before :each do setup end - describe 'loading teacher survey responses' do + describe "loading teacher survey responses" do before do - SurveyResponsesDataLoader.load_data filepath: path_to_teacher_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_teacher_responses end - it 'ensures teacher responses load correctly' do + it "ensures teacher responses load correctly" do assigns_academic_year_to_survey_item_responses assigns_school_to_the_survey_item_responses assigns_recorded_date_to_teacher_responses @@ -114,12 +120,12 @@ describe SurveyResponsesDataLoader do end end - describe 'student survey responses' do + describe "student survey responses" do before do - SurveyResponsesDataLoader.load_data filepath: path_to_student_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses end - it 'ensures student responses load correctly' do + it "ensures student responses load correctly" do assigns_academic_year_to_student_survey_item_responses assigns_school_to_student_survey_item_responses assigns_recorded_date_to_student_responses @@ -129,85 +135,86 @@ describe SurveyResponsesDataLoader do assigns_grade_level_to_responses assigns_gender_to_responses assigns_income_to_responses + assigns_ell_to_responses is_idempotent_for_students end - context 'when updating student survey responses from another csv file' do + context "when updating student survey responses from another csv file" do before :each do - SurveyResponsesDataLoader.load_data filepath: Rails.root.join('spec', 'fixtures', - 'secondary_test_2020-21_student_survey_responses.csv') + SurveyResponsesDataLoader.new.load_data filepath: Rails.root.join("spec", "fixtures", + "secondary_test_2020-21_student_survey_responses.csv") end - it 'updates the likert score to the score on the new csv file' do - s_emsa_q1 = SurveyItem.find_by_survey_item_id 's-emsa-q1' - expect(SurveyItemResponse.where(response_id: 'student_survey_response_3', + it "updates the likert score to the score on the new csv file" do + s_emsa_q1 = SurveyItem.find_by_survey_item_id "s-emsa-q1" + expect(SurveyItemResponse.where(response_id: "student_survey_response_3", survey_item: s_emsa_q1).first.likert_score).to eq 1 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_4', + expect(SurveyItemResponse.where(response_id: "student_survey_response_4", survey_item: s_emsa_q1).first.likert_score).to eq 1 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_5', + expect(SurveyItemResponse.where(response_id: "student_survey_response_5", survey_item: s_emsa_q1).first.likert_score).to eq 1 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_5', + expect(SurveyItemResponse.where(response_id: "student_survey_response_5", survey_item: s_acst_q3).first.likert_score).to eq 4 end end end # figure out why this is failing - describe 'when using Lowell rules to skip rows in the csv file' do + describe "when using Lowell rules to skip rows in the csv file" do before :each do - SurveyResponsesDataLoader.load_data filepath: path_to_student_responses, - rules: [Rule::SkipNonLowellSchools] + SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses, + rules: [Rule::SkipNonLowellSchools] end - it 'rejects any non-lowell school' do - expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').count).to eq 0 + it "rejects any non-lowell school" do + expect(SurveyItemResponse.where(response_id: "student_survey_response_1").count).to eq 0 expect(SurveyItemResponse.count).to eq 69 end - it 'loads the correct number of responses for lowell schools' do - expect(SurveyItemResponse.where(response_id: 'student_survey_response_2').count).to eq 0 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_3').count).to eq 12 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_4').count).to eq 15 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_5').count).to eq 14 + it "loads the correct number of responses for lowell schools" do + expect(SurveyItemResponse.where(response_id: "student_survey_response_2").count).to eq 0 + expect(SurveyItemResponse.where(response_id: "student_survey_response_3").count).to eq 12 + expect(SurveyItemResponse.where(response_id: "student_survey_response_4").count).to eq 15 + expect(SurveyItemResponse.where(response_id: "student_survey_response_5").count).to eq 14 end - context 'when loading 22-23 butler survey responses' do + context "when loading 22-23 butler survey responses" do before :each do - SurveyResponsesDataLoader.load_data filepath: path_to_butler_student_responses, - rules: [Rule::SkipNonLowellSchools] + SurveyResponsesDataLoader.new.load_data filepath: path_to_butler_student_responses, + rules: [Rule::SkipNonLowellSchools] end - it 'loads all the responses for Butler' do + it "loads all the responses for Butler" do expect(SurveyItemResponse.where(school: butler_school).count).to eq 56 end - it 'blank entries for grade get loaded as nils, not zero values' do - expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_1').first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_2').first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_3').first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_4').first.grade).to eq 5 - expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_5').first.grade).to eq 7 - expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_6').first.grade).to eq 6 - expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_7').first.grade).to eq nil - expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_8').first.grade).to eq 0 + it "blank entries for grade get loaded as nils, not zero values" do + expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_1").first.grade).to eq 7 + expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_2").first.grade).to eq 7 + expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_3").first.grade).to eq 7 + expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_4").first.grade).to eq 5 + expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_5").first.grade).to eq 7 + expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_6").first.grade).to eq 6 + expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_7").first.grade).to eq nil + expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_8").first.grade).to eq 0 end end end end def assigns_academic_year_to_survey_item_responses - expect(SurveyItemResponse.find_by_response_id('teacher_survey_response_1').academic_year).to eq ay_2020_21 + expect(SurveyItemResponse.find_by_response_id("teacher_survey_response_1").academic_year).to eq ay_2020_21 end def assigns_school_to_the_survey_item_responses - expect(SurveyItemResponse.find_by_response_id('teacher_survey_response_1').school).to eq school + expect(SurveyItemResponse.find_by_response_id("teacher_survey_response_1").school).to eq school end def loads_survey_item_responses_for_a_given_survey_response - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_1').count).to eq 5 - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_2').count).to eq 0 - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_3').count).to eq 8 - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_4').count).to eq 8 - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_5').count).to eq 8 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_1").count).to eq 5 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_2").count).to eq 0 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_3").count).to eq 8 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_4").count).to eq 8 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_5").count).to eq 8 end def loads_all_survey_item_responses_for_a_given_survey_item @@ -216,44 +223,44 @@ def loads_all_survey_item_responses_for_a_given_survey_item end def captures_likert_scores_for_survey_item_responses - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_1').where(survey_item: t_pcom_q2)).to be_empty - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_1').where(survey_item: t_pcom_q3).first.likert_score).to eq 3 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_1").where(survey_item: t_pcom_q2)).to be_empty + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_1").where(survey_item: t_pcom_q3).first.likert_score).to eq 3 - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_2').where(survey_item: t_pcom_q2)).to be_empty - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_2').where(survey_item: t_pcom_q3)).to be_empty + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_2").where(survey_item: t_pcom_q2)).to be_empty + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_2").where(survey_item: t_pcom_q3)).to be_empty - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_3').where(survey_item: t_pcom_q2).first.likert_score).to eq 5 - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_3').where(survey_item: t_pcom_q3).first.likert_score).to eq 5 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_3").where(survey_item: t_pcom_q2).first.likert_score).to eq 5 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_3").where(survey_item: t_pcom_q3).first.likert_score).to eq 5 - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_4').where(survey_item: t_pcom_q2).first.likert_score).to eq 4 - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_4').where(survey_item: t_pcom_q3).first.likert_score).to eq 4 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_4").where(survey_item: t_pcom_q2).first.likert_score).to eq 4 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_4").where(survey_item: t_pcom_q3).first.likert_score).to eq 4 - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_5').where(survey_item: t_pcom_q2).first.likert_score).to eq 2 - expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_5').where(survey_item: t_pcom_q3).first.likert_score).to eq 4 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_5").where(survey_item: t_pcom_q2).first.likert_score).to eq 2 + expect(SurveyItemResponse.where(response_id: "teacher_survey_response_5").where(survey_item: t_pcom_q3).first.likert_score).to eq 4 end def is_idempotent number_of_survey_item_responses = SurveyItemResponse.count - SurveyResponsesDataLoader.load_data filepath: path_to_teacher_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_teacher_responses expect(SurveyItemResponse.count).to eq number_of_survey_item_responses end def assigns_academic_year_to_student_survey_item_responses - expect(SurveyItemResponse.find_by_response_id('student_survey_response_3').academic_year).to eq ay_2020_21 + expect(SurveyItemResponse.find_by_response_id("student_survey_response_3").academic_year).to eq ay_2020_21 end def assigns_school_to_student_survey_item_responses - expect(SurveyItemResponse.find_by_response_id('student_survey_response_3').school).to eq second_school + expect(SurveyItemResponse.find_by_response_id("student_survey_response_3").school).to eq second_school end def loads_student_survey_item_response_values - expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').count).to eq 3 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_2').count).to eq 0 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_3').count).to eq 12 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_4').count).to eq 15 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_5').count).to eq 14 + expect(SurveyItemResponse.where(response_id: "student_survey_response_1").count).to eq 3 + expect(SurveyItemResponse.where(response_id: "student_survey_response_2").count).to eq 0 + expect(SurveyItemResponse.where(response_id: "student_survey_response_3").count).to eq 12 + expect(SurveyItemResponse.where(response_id: "student_survey_response_4").count).to eq 15 + expect(SurveyItemResponse.where(response_id: "student_survey_response_5").count).to eq 14 end def student_survey_item_response_count_matches_expected @@ -262,37 +269,37 @@ def student_survey_item_response_count_matches_expected end def captures_likert_scores_for_student_survey_item_responses - expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').where(survey_item: s_phys_q1).first.likert_score).to eq 3 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').where(survey_item: s_phys_q2)).to be_empty + expect(SurveyItemResponse.where(response_id: "student_survey_response_1").where(survey_item: s_phys_q1).first.likert_score).to eq 3 + expect(SurveyItemResponse.where(response_id: "student_survey_response_1").where(survey_item: s_phys_q2)).to be_empty - expect(SurveyItemResponse.where(response_id: 'student_survey_response_2').where(survey_item: s_phys_q1)).to be_empty - expect(SurveyItemResponse.where(response_id: 'student_survey_response_2').where(survey_item: s_phys_q2)).to be_empty + expect(SurveyItemResponse.where(response_id: "student_survey_response_2").where(survey_item: s_phys_q1)).to be_empty + expect(SurveyItemResponse.where(response_id: "student_survey_response_2").where(survey_item: s_phys_q2)).to be_empty - expect(SurveyItemResponse.where(response_id: 'student_survey_response_3').where(survey_item: s_phys_q1).first.likert_score).to eq 1 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_3').where(survey_item: s_phys_q2).first.likert_score).to eq 3 + expect(SurveyItemResponse.where(response_id: "student_survey_response_3").where(survey_item: s_phys_q1).first.likert_score).to eq 1 + expect(SurveyItemResponse.where(response_id: "student_survey_response_3").where(survey_item: s_phys_q2).first.likert_score).to eq 3 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_4').where(survey_item: s_phys_q1).first.likert_score).to eq 1 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_4').where(survey_item: s_phys_q2).first.likert_score).to eq 1 + expect(SurveyItemResponse.where(response_id: "student_survey_response_4").where(survey_item: s_phys_q1).first.likert_score).to eq 1 + expect(SurveyItemResponse.where(response_id: "student_survey_response_4").where(survey_item: s_phys_q2).first.likert_score).to eq 1 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_5').where(survey_item: s_phys_q1).first.likert_score).to eq 1 - expect(SurveyItemResponse.where(response_id: 'student_survey_response_5').where(survey_item: s_phys_q2).first.likert_score).to eq 2 + expect(SurveyItemResponse.where(response_id: "student_survey_response_5").where(survey_item: s_phys_q1).first.likert_score).to eq 1 + expect(SurveyItemResponse.where(response_id: "student_survey_response_5").where(survey_item: s_phys_q2).first.likert_score).to eq 2 end def is_idempotent_for_students number_of_survey_item_responses = SurveyItemResponse.count - SurveyResponsesDataLoader.load_data filepath: path_to_student_responses + SurveyResponsesDataLoader.new.load_data filepath: path_to_student_responses expect(SurveyItemResponse.count).to eq number_of_survey_item_responses end def assigns_grade_level_to_responses - results = { 'student_survey_response_1' => 11, - 'student_survey_response_3' => 8, - 'student_survey_response_4' => 8, - 'student_survey_response_5' => 7, - 'student_survey_response_6' => 3, - 'student_survey_response_7' => 4 } + results = { "student_survey_response_1" => 11, + "student_survey_response_3" => 8, + "student_survey_response_4" => 8, + "student_survey_response_5" => 7, + "student_survey_response_6" => 3, + "student_survey_response_7" => 4 } results.each do |key, value| expect(SurveyItemResponse.where(response_id: key).all? do |response| response.grade == value @@ -301,12 +308,12 @@ def assigns_grade_level_to_responses end def assigns_gender_to_responses - results = { 'student_survey_response_1' => female, - 'student_survey_response_3' => male, - 'student_survey_response_4' => non_binary, - 'student_survey_response_5' => non_binary, - 'student_survey_response_6' => unknown_gender, - 'student_survey_response_7' => unknown_gender } + results = { "student_survey_response_1" => female, + "student_survey_response_3" => male, + "student_survey_response_4" => non_binary, + "student_survey_response_5" => non_binary, + "student_survey_response_6" => unknown_gender, + "student_survey_response_7" => unknown_gender } results.each do |key, value| expect(SurveyItemResponse.where(response_id: key).first.gender).to eq value @@ -314,36 +321,51 @@ def assigns_gender_to_responses end def assigns_recorded_date_to_student_responses - results = { 'student_survey_response_1' => '2020-09-30T18:48:50', - 'student_survey_response_3' => '2021-03-31T09:59:02', - 'student_survey_response_4' => '2021-03-31T10:00:17', - 'student_survey_response_5' => '2021-03-31T10:01:36', - 'student_survey_response_6' => '2021-03-31T10:01:37', - 'student_survey_response_7' => '2021-03-31T10:01:38' } + results = { "student_survey_response_1" => "2020-09-30T18:48:50", + "student_survey_response_3" => "2021-03-31T09:59:02", + "student_survey_response_4" => "2021-03-31T10:00:17", + "student_survey_response_5" => "2021-03-31T10:01:36", + "student_survey_response_6" => "2021-03-31T10:01:37", + "student_survey_response_7" => "2021-03-31T10:01:38" } results.each do |key, value| expect(SurveyItemResponse.find_by_response_id(key).recorded_date).to eq Date.parse(value) end end def assigns_recorded_date_to_teacher_responses - results = { 'teacher_survey_response_1' => '2020-10-16 11:09:03', - 'teacher_survey_response_3' => '2020-12-06 8:36:52', - 'teacher_survey_response_4' => '2020-12-06 8:51:25', - 'teacher_survey_response_5' => '2020-12-06 8:55:58' } + results = { "teacher_survey_response_1" => "2020-10-16 11:09:03", + "teacher_survey_response_3" => "2020-12-06 8:36:52", + "teacher_survey_response_4" => "2020-12-06 8:51:25", + "teacher_survey_response_5" => "2020-12-06 8:55:58" } results.each do |key, value| expect(SurveyItemResponse.find_by_response_id(key).recorded_date).to eq Date.parse(value) end end def assigns_income_to_responses - results = { 'student_survey_response_1' => low_income, - 'student_survey_response_3' => low_income, - 'student_survey_response_4' => unknown_income, - 'student_survey_response_5' => low_income, - 'student_survey_response_6' => high_income, - 'student_survey_response_7' => low_income } + results = { "student_survey_response_1" => low_income, + "student_survey_response_3" => low_income, + "student_survey_response_4" => unknown_income, + "student_survey_response_5" => low_income, + "student_survey_response_6" => high_income, + "student_survey_response_7" => low_income } results.each do |key, value| - expect(SurveyItemResponse.where(response_id: key).first.income).to eq value + income = SurveyItemResponse.find_by_response_id(key).income + expect(income).to eq value + end +end + +def assigns_ell_to_responses + results = { "student_survey_response_1" => not_ell, + "student_survey_response_3" => unknown_ell, + "student_survey_response_4" => yes_ell, + "student_survey_response_5" => yes_ell, + "student_survey_response_6" => unknown_ell, + "student_survey_response_7" => unknown_ell } + + results.each do |key, value| + ell = SurveyItemResponse.find_by_response_id(key).ell + expect(ell).to eq value end end