diff --git a/app/services/cleaner.rb b/app/services/cleaner.rb index 29d51904..1b982e42 100644 --- a/app/services/cleaner.rb +++ b/app/services/cleaner.rb @@ -1,61 +1,38 @@ -require "fileutils" +require 'fileutils' class Cleaner - attr_reader :input_filepath, :output_filepath, :log_filepath, :clean_csv, :log_csv + attr_reader :input_filepath, :output_filepath, :log_filepath, :disaggregation_filepath - def initialize(input_filepath:, output_filepath:, log_filepath:) + def initialize(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:) @input_filepath = input_filepath @output_filepath = output_filepath @log_filepath = log_filepath + @disaggregation_filepath = disaggregation_filepath initialize_directories end - def initialize_directories - create_ouput_directory - create_log_directory - end - def clean - Dir.glob(Rails.root.join(input_filepath, "*.csv")).each do |filepath| + Dir.glob(Rails.root.join(input_filepath, '*.csv')).each do |filepath| puts filepath File.open(filepath) do |file| - clean_csv = [] - log_csv = [] - data = [] - - headers = CSV.parse(file.first).first - filtered_headers = remove_unwanted_headers(headers:) - log_headers = (filtered_headers + ["Valid Duration?", "Valid Progress?", "Valid Grade?", - "Valid Standard Deviation?"]).flatten - - clean_csv << filtered_headers - log_csv << log_headers - - all_survey_items = survey_items(headers:) - - file.lazy.each_slice(1000) do |lines| - CSV.parse(lines.join, headers:).map do |row| - values = SurveyItemValues.new(row:, headers:, genders:, - survey_items: all_survey_items, schools:) - next unless values.valid_school? - - data << values - values.valid? ? clean_csv << values.to_a : log_csv << (values.to_a << values.valid_duration?.to_s << values.valid_progress?.to_s << values.valid_grade?.to_s << values.valid_sd?.to_s) - end - end - - unless data.empty? - filename = filename(headers:, data:) - write_csv(data: clean_csv, output_filepath:, filename:) - write_csv(data: log_csv, output_filepath: log_filepath, prefix: "removed.", filename:) - end + processed_data = process_raw_file(file:, disaggregation_data:) + processed_data in [headers, clean_csv, log_csv, data] + return if data.empty? + + filename = filename(headers:, data:) + write_csv(data: clean_csv, output_filepath:, filename:) + write_csv(data: log_csv, output_filepath: log_filepath, prefix: 'removed.', filename:) end end end + def disaggregation_data + @disaggregation_data ||= DisaggregationLoader.new(path: disaggregation_filepath).load + end + def filename(headers:, data:) survey_item_ids = headers.filter(&:present?).filter do |header| - header.start_with?("s-", "t-") - end.reject { |item| item.end_with? "-1" } + header.start_with?('s-', 't-') + end.reject { |item| item.end_with? '-1' } survey_type = SurveyItem.survey_type(survey_item_ids:) range = data.first.academic_year.range @@ -63,16 +40,51 @@ class Cleaner row.district.short_name end.to_set.to_a - districts.join(".").to_s + "." + survey_type.to_s + "." + range + ".csv" + districts.join('.').to_s + '.' + survey_type.to_s + '.' + range + '.csv' + end + + def process_raw_file(file:, disaggregation_data:) + clean_csv = [] + log_csv = [] + data = [] + + headers = (CSV.parse(file.first).first << 'Raw Income') << 'Income' + filtered_headers = remove_unwanted_headers(headers:) + log_headers = (filtered_headers + ['Valid Duration?', 'Valid Progress?', 'Valid Grade?', + 'Valid Standard Deviation?']).flatten + + clean_csv << filtered_headers + log_csv << log_headers + + all_survey_items = survey_items(headers:) + + file.lazy.each_slice(1000) do |lines| + CSV.parse(lines.join, headers:).map do |row| + values = SurveyItemValues.new(row:, headers:, genders:, + survey_items: all_survey_items, schools:, disaggregation_data:) + next unless values.valid_school? + + data << values + values.valid? ? clean_csv << values.to_a : log_csv << (values.to_a << values.valid_duration?.to_s << values.valid_progress?.to_s << values.valid_grade?.to_s << values.valid_sd?.to_s) + end + end + [headers, clean_csv, log_csv, data] + end + + private + + def initialize_directories + create_ouput_directory + create_log_directory end def remove_unwanted_headers(headers:) headers.to_set.to_a.compact.reject do |item| - item.start_with? "Q" - end.reject { |item| item.end_with? "-1" } + item.start_with? 'Q' + end.reject { |item| item.end_with? '-1' } end - def write_csv(data:, output_filepath:, filename:, prefix: "") + def write_csv(data:, output_filepath:, filename:, prefix: '') csv = CSV.generate do |csv| data.each do |row| csv << row @@ -81,34 +93,19 @@ class Cleaner File.write(output_filepath.join(prefix + filename), csv) end - def process_row(row:) - clean_csv << row.to_csv - log_csv << row.to_csv - end - def schools @schools ||= School.school_hash end def genders - @genders ||= begin - gender_hash = {} - - Gender.all.each do |gender| - gender_hash[gender.qualtrics_code] = gender - end - gender_hash - end + @genders ||= Gender.gender_hash end def survey_items(headers:) - @survey_items ||= SurveyItem.where(survey_item_id: get_survey_item_ids_from_headers(headers:)) - end - - def get_survey_item_ids_from_headers(headers:) - headers - .filter(&:present?) - .filter { |header| header.start_with? "t-", "s-" } + survey_item_ids = headers + .filter(&:present?) + .filter { |header| header.start_with? 't-', 's-' } + @survey_items ||= SurveyItem.where(survey_item_id: survey_item_ids) end def create_ouput_directory @@ -118,8 +115,4 @@ class Cleaner def create_log_directory FileUtils.mkdir_p log_filepath end - - def create_file(path:, filename:) - FileUtils.touch path.join(filename) - end end diff --git a/app/services/disaggregation_loader.rb b/app/services/disaggregation_loader.rb new file mode 100644 index 00000000..a4ac7e42 --- /dev/null +++ b/app/services/disaggregation_loader.rb @@ -0,0 +1,30 @@ +class DisaggregationLoader + attr_reader :path + + def initialize(path:) + @path = path + initialize_directory + end + + def load + data = {} + Dir.glob(Rails.root.join(path, '*.csv')).each do |filepath| + puts filepath + File.open(filepath) do |file| + headers = CSV.parse(file.first).first + + file.lazy.each_slice(1000) do |lines| + CSV.parse(lines.join, headers:).map do |row| + values = DisaggregationRow.new(row:, headers:) + data[[values.lasid, values.district, values.academic_year]] = values + end + end + end + end + data + end + + def initialize_directory + FileUtils.mkdir_p(path) + end +end diff --git a/app/services/disaggregation_row.rb b/app/services/disaggregation_row.rb new file mode 100644 index 00000000..0fefe53f --- /dev/null +++ b/app/services/disaggregation_row.rb @@ -0,0 +1,35 @@ +class DisaggregationRow + attr_reader :row, :headers + + def initialize(row:, headers:) + @row = row + @headers = headers + end + + def district + @district ||= value_from(pattern: /District/i) + end + + def academic_year + @academic_year ||= value_from(pattern: /Academic\s*Year/i) + end + + def income + @income ||= value_from(pattern: /Low\s*Income/i) + end + + def lasid + @lasid ||= value_from(pattern: /LASID/i) + end + + def value_from(pattern:) + output = nil + matches = headers.select do |header| + pattern.match(header) + end.map { |item| item.delete("\n") } + matches.each do |match| + output ||= row[match] + end + output + end +end diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index 99bbd96e..57913704 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -1,12 +1,13 @@ class SurveyItemValues - attr_reader :row, :headers, :genders, :survey_items, :schools + attr_reader :row, :headers, :genders, :survey_items, :schools, :disaggregation_data - def initialize(row:, headers:, genders:, survey_items:, schools:) + def initialize(row:, headers:, genders:, survey_items:, schools:, disaggregation_data: nil) @row = row @headers = headers @genders = genders @survey_items = survey_items @schools = schools + @disaggregation_data = disaggregation_data end def dese_id? @@ -51,7 +52,7 @@ class SurveyItemValues def dese_id @dese_id ||= begin dese_id = nil - dese_headers = ["DESE ID", "Dese ID", "DeseId", "DeseID", "School", "school"] + dese_headers = ['DESE ID', 'Dese ID', 'DeseId', 'DeseID', 'School', 'school'] school_headers = headers.select { |header| /School-\s\w/.match(header) } dese_headers << school_headers dese_headers.flatten.each do |header| @@ -93,6 +94,31 @@ class SurveyItemValues genders[gender_code] end + def lasid + @lasid ||= value_from(pattern: /LASID/i) + end + + def raw_income + @raw_income ||= value_from(pattern: /Income|Low\s*Income/i) + return 'Unknown' unless disaggregation_data.present? + + disaggregation = disaggregation_data[[lasid, district.name, academic_year.range]] + return 'Unknown' unless disaggregation.present? + + @raw_income ||= disaggregation.income + end + + def income + @income ||= case raw_income + in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income/i + 'Economically Disadvantaged - Y' + in /Not\s*Eligible/i + 'Economically Disadvantaged - N' + else + 'Unknown' + end + end + def value_from(pattern:) output = nil matches = headers.select do |header| @@ -106,10 +132,12 @@ class SurveyItemValues def to_a copy_likert_scores_from_variant_survey_items + row['Income'] = income + row['Raw Income'] = raw_income headers.select(&:present?) - .reject { |key, _value| key.start_with? "Q" } - .reject { |key, _value| key.end_with? "-1" } - .map { |header| row[header] } + .reject { |key, _value| key.start_with? 'Q' } + .reject { |key, _value| key.end_with? '-1' } + .map { |header| row[header] } end def duration @@ -122,23 +150,23 @@ class SurveyItemValues def respondent_type return :teacher if headers - .filter(&:present?) - .filter { |header| header.start_with? "t-" }.count > 0 + .filter(&:present?) + .filter { |header| header.start_with? 't-' }.count > 0 :student end def survey_type survey_item_ids = headers - .filter(&:present?) - .reject { |header| header.end_with?("-1") } - .filter { |header| header.start_with?("t-", "s-") } + .filter(&:present?) + .reject { |header| header.end_with?('-1') } + .filter { |header| header.start_with?('t-', 's-') } SurveyItem.survey_type(survey_item_ids:) end def valid_duration? - return true if duration.nil? || duration == "" || duration.downcase == "n/a" || duration.downcase == "na" + return true if duration.nil? || duration == '' || duration.downcase == 'n/a' || duration.downcase == 'na' span_in_seconds = duration.to_i return span_in_seconds >= 300 if survey_type == :teacher @@ -149,8 +177,8 @@ class SurveyItemValues end def valid_progress? - progress = row["Progress"] - return true if progress.nil? || progress == "" || progress.downcase == "n/a" || progress.downcase == "na" + progress = row['Progress'] + return true if progress.nil? || progress == '' || progress.downcase == 'n/a' || progress.downcase == 'na' progress = progress.to_i progress.to_i >= 25 @@ -175,7 +203,7 @@ class SurveyItemValues def valid_sd? return true if survey_type == :early_education - survey_item_headers = headers.filter(&:present?).filter { |header| header.start_with?("s-", "t-") } + survey_item_headers = headers.filter(&:present?).filter { |header| header.start_with?('s-', 't-') } likert_scores = [] survey_item_headers.each do |header| likert_scores << likert_score(survey_item_id: header).to_i @@ -193,9 +221,9 @@ class SurveyItemValues private def copy_likert_scores_from_variant_survey_items - headers.filter(&:present?).filter { |header| header.end_with? "-1" }.each do |header| + headers.filter(&:present?).filter { |header| header.end_with? '-1' }.each do |header| likert_score = row[header] - main_item = header.gsub("-1", "") + main_item = header.gsub('-1', '') row[main_item] = likert_score if likert_score.present? end end diff --git a/lib/tasks/clean.rake b/lib/tasks/clean.rake index ef8ab445..3011c02b 100644 --- a/lib/tasks/clean.rake +++ b/lib/tasks/clean.rake @@ -5,7 +5,8 @@ namespace :clean do input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'raw') output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'clean') log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'removed') - Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean + disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation') + Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean end desc 'clean prepped data' @@ -13,14 +14,16 @@ namespace :clean do input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped') output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'clean') log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'removed') - Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean + disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation') + Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean end desc 'clean mciea data' task mciea: :environment do input_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'raw') output_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'clean') log_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'removed') - Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean + disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation') + Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean end desc 'clean rpp data' @@ -28,6 +31,7 @@ namespace :clean do input_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'raw') output_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'clean') log_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'removed') - Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean + disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation') + Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean end end diff --git a/spec/fixtures/disaggregation/sample_maynard_disaggregation_data.csv b/spec/fixtures/disaggregation/sample_maynard_disaggregation_data.csv new file mode 100644 index 00000000..02b7fae1 --- /dev/null +++ b/spec/fixtures/disaggregation/sample_maynard_disaggregation_data.csv @@ -0,0 +1,501 @@ +District,Academic Year,LASID,HispanicLatino,Race,Gender,SpecialEdStatus,In 504 Plan,LowIncome,EL Student First Year +Maynard Public Schools,2022-23,1,TRUE,Caucasian,M,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,2,TRUE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,3,FALSE,Black,M,,,Reduced Lunch, +Maynard Public Schools,2022-23,4,FALSE,Caucasian,F,,Not 504,,LEP student not 1st year +Maynard Public Schools,2022-23,5,FALSE,Caucasian,M,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,6,TRUE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,7,FALSE,Caucasian,F,,Not 504,Reduced Lunch,Does not apply +Maynard Public Schools,2022-23,8,FALSE,Caucasian,F,,Not 504,,Does not apply +Maynard Public Schools,2022-23,9,FALSE,"Asian,Native American",F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,10,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,11,FALSE,Caucasian,M,,Not 504,Reduced Lunch,Does not apply +Maynard Public Schools,2022-23,12,FALSE,Caucasian,F,,Not 504,,Does not apply +Maynard Public Schools,2022-23,13,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,14,TRUE,Caucasian,F,,,Free Lunch, +Maynard Public Schools,2022-23,15,FALSE,Caucasian,F,,,Free Lunch, +Maynard Public Schools,2022-23,16,FALSE,Caucasian,M,,,Free Lunch, +Maynard Public Schools,2022-23,17,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,18,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,19,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,20,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,21,FALSE,Asian,F,,Not 504,Reduced Lunch,Does not apply +Maynard Public Schools,2022-23,22,FALSE,Asian,F,,Not 504,Reduced Lunch,Does not apply +Maynard Public Schools,2022-23,23,TRUE,Caucasian,F,,,Not Eligible,LEP student not 1st year +Maynard Public Schools,2022-23,24,TRUE,Caucasian,F,,Not 504,Free Lunch, +Maynard Public Schools,2022-23,25,TRUE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,26,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,27,TRUE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,28,TRUE,Caucasian,F,,Not 504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,29,TRUE,Native American,F,,,Free Lunch,LEP student 1st year +Maynard Public Schools,2022-23,30,FALSE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,31,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,32,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,33,FALSE,Caucasian,F,,,Free Lunch, +Maynard Public Schools,2022-23,34,FALSE,Caucasian,M,Active,Not 504,Free Lunch, +Maynard Public Schools,2022-23,35,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,36,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,37,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,38,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,39,TRUE,"Black, Caucasian",M,Exited,Not 504,Reduced Lunch,Does not apply +Maynard Public Schools,2022-23,40,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,41,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,42,FALSE,,M,,,Not Eligible, +Maynard Public Schools,2022-23,43,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,44,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,45,FALSE,Caucasian,F,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,46,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,47,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,48,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,49,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,50,TRUE,Native American,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,51,TRUE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,52,TRUE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,53,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,54,FALSE,"Caucasian,Pacific Island",M,Active,Not 504,Reduced Lunch,Does not apply +Maynard Public Schools,2022-23,55,FALSE,"Caucasian,Pacific Island",F,,Not 504,Reduced Lunch,Does not apply +Maynard Public Schools,2022-23,56,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,57,TRUE,Caucasian,M,Active,,Free Lunch, +Maynard Public Schools,2022-23,58,TRUE,Caucasian,M,,,Free Lunch,LEP student 1st year +Maynard Public Schools,2022-23,59,TRUE,Caucasian,M,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,60,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,61,TRUE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,62,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,63,FALSE,,M,,,Not Eligible, +Maynard Public Schools,2022-23,64,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,65,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,66,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,67,FALSE,"Asian,Caucasian",M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,68,FALSE,"Asian,Caucasian",M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,69,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,70,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,71,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,72,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,73,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,74,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,75,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,76,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,77,TRUE,Caucasian,M,,504,Reduced Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,78,TRUE,Caucasian,M,,504,Reduced Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,79,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,80,TRUE,Caucasian,F,,,Free Lunch,LEP student 1st year +Maynard Public Schools,2022-23,81,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,82,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,83,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,84,TRUE,Caucasian,M,,Not 504,Not Eligible,LEP student not 1st year +Maynard Public Schools,2022-23,85,FALSE,Caucasian,F,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,86,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,87,TRUE,Black,F,Active,504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,88,TRUE,Black,F,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,89,TRUE,Black,F,,,Free Lunch, +Maynard Public Schools,2022-23,90,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,91,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,92,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,93,FALSE,Caucasian,F,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,94,FALSE,Caucasian,M,,,Free Lunch, +Maynard Public Schools,2022-23,95,FALSE,Caucasian,F,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,96,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,97,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,98,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,99,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,100,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,101,FALSE,Caucasian,F,,,Free Lunch, +Maynard Public Schools,2022-23,102,FALSE,Caucasian,F,,,Free Lunch, +Maynard Public Schools,2022-23,103,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,104,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,105,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,106,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,107,FALSE,Caucasian,M,Active,Not 504,Not Eligible, +Maynard Public Schools,2022-23,108,FALSE,Caucasian,M,Eligible,Not 504,Not Eligible, +Maynard Public Schools,2022-23,109,FALSE,Caucasian,F,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,110,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,111,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,112,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,113,TRUE,Caucasian,F,,,Free Lunch, +Maynard Public Schools,2022-23,114,TRUE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,115,TRUE,Caucasian,F,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,116,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,117,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,118,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,119,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,120,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,121,FALSE,"Black, Native American",F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,122,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,123,FALSE,Caucasian,F,,,Reduced Lunch, +Maynard Public Schools,2022-23,124,FALSE,Caucasian,M,,,Reduced Lunch, +Maynard Public Schools,2022-23,125,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,126,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,127,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,128,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,129,FALSE,"Caucasian, Native American",F,,,Not Eligible, +Maynard Public Schools,2022-23,130,TRUE,Caucasian,F,,Not 504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,131,TRUE,Caucasian,M,,Not 504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,132,TRUE,Caucasian,F,Active,Not 504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,133,TRUE,Caucasian,F,,Not 504,Not Eligible,LEP student not 1st year +Maynard Public Schools,2022-23,134,TRUE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,135,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,136,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,137,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,138,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,139,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,140,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,141,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,142,TRUE,Caucasian,M,,,Not Eligible,LEP student 1st year +Maynard Public Schools,2022-23,143,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,144,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,145,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,146,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,147,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,148,FALSE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,149,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,150,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,151,FALSE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,152,TRUE,Caucasian,M,Active,Not 504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,153,TRUE,Caucasian,F,,Not 504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,154,FALSE,Caucasian,F,,,Not Eligible,Does not apply +Maynard Public Schools,2022-23,155,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,156,FALSE,Black,F,,504,Not Eligible, +Maynard Public Schools,2022-23,157,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,158,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,159,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,160,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,161,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,162,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,163,TRUE,Caucasian,M,,,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,164,TRUE,Caucasian,M,,,Free Lunch,LEP student 1st year +Maynard Public Schools,2022-23,165,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,166,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,167,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,168,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,169,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,170,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,171,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,172,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,173,FALSE,Caucasian,F,,504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,174,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,175,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,176,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,177,FALSE,Caucasian,M,Exited,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,178,TRUE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,179,FALSE,Caucasian,M,,,Free Lunch, +Maynard Public Schools,2022-23,180,TRUE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,181,TRUE,Caucasian,F,,Not 504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,182,TRUE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,183,TRUE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,184,TRUE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,185,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,186,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,187,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,188,FALSE,"Asian,Caucasian",M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,189,FALSE,"Asian,Caucasian",M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,190,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,191,FALSE,"Asian,Caucasian",F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,192,FALSE,Black,F,Active,Not 504,Free Lunch, +Maynard Public Schools,2022-23,193,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,194,FALSE,Asian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,195,FALSE,Asian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,196,TRUE,Caucasian,F,,,Not Eligible,LEP student not 1st year +Maynard Public Schools,2022-23,197,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,198,FALSE,Caucasian,F,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,199,FALSE,"Asian,Caucasian",M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,200,FALSE,"Asian,Caucasian",F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,201,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,202,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,203,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,204,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,205,TRUE,Caucasian,M,,,Free Lunch, +Maynard Public Schools,2022-23,206,FALSE,Asian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,207,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,208,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,209,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,210,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,211,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,212,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,213,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,214,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,215,FALSE,Caucasian,F,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,216,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,217,FALSE,Caucasian,F,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,218,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,219,FALSE,Caucasian,M,Active,Not 504,Reduced Lunch,Does not apply +Maynard Public Schools,2022-23,220,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,221,TRUE,"Black, Caucasian",M,Active,,Not Eligible, +Maynard Public Schools,2022-23,222,TRUE,Caucasian,F,,,Free Lunch,LEP student 1st year +Maynard Public Schools,2022-23,223,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,224,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,225,FALSE,Caucasian,F,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,226,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,227,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,228,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,229,FALSE,Caucasian,M,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,230,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,231,TRUE,Pacific Island,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,232,TRUE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,233,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,234,FALSE,Caucasian,M,,,Free Lunch, +Maynard Public Schools,2022-23,235,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,236,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,237,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,238,TRUE,"Black, Caucasian, Native American",F,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,239,TRUE,Caucasian,M,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,240,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,241,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,242,TRUE,Caucasian,M,,Not 504,Reduced Lunch,Does not apply +Maynard Public Schools,2022-23,243,TRUE,Caucasian,F,,,Reduced Lunch,LEP student 1st year +Maynard Public Schools,2022-23,244,TRUE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,245,FALSE,Caucasian,M,Active,Not 504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,246,TRUE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,247,FALSE,"Caucasian,Native American",M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,248,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,249,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,250,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,251,FALSE,"Black, Caucasian",M,Exited,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,252,FALSE,"Black, Caucasian",F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,253,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,254,FALSE,Caucasian,M,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,255,FALSE,Caucasian,F,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,256,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,257,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,258,FALSE,Caucasian,M,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,259,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,260,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,261,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,262,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,263,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,264,TRUE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,265,TRUE,Caucasian,F,,,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,266,TRUE,Caucasian,M,,,Free Lunch,LEP student 1st year +Maynard Public Schools,2022-23,267,FALSE,Caucasian,F,,Not 504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,268,TRUE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,269,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,270,TRUE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,271,TRUE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,272,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,273,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,274,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,275,FALSE,Black,M,,,Free Lunch, +Maynard Public Schools,2022-23,276,FALSE,Black,F,,,Free Lunch, +Maynard Public Schools,2022-23,277,FALSE,Black,F,,,Free Lunch, +Maynard Public Schools,2022-23,278,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,279,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,280,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,281,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,282,TRUE,Caucasian,M,Active,Not 504,Not Eligible,LEP student not 1st year +Maynard Public Schools,2022-23,283,TRUE,Caucasian,F,,Not 504,Not Eligible,LEP student not 1st year +Maynard Public Schools,2022-23,284,TRUE,Caucasian,F,,Not 504,Not Eligible,LEP student not 1st year +Maynard Public Schools,2022-23,285,TRUE,Caucasian,F,,,Free Lunch, +Maynard Public Schools,2022-23,286,TRUE,Caucasian,M,,504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,287,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,288,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,289,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,290,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,291,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,292,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,293,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,294,FALSE,Caucasian,M,Active,Not 504,Free Lunch, +Maynard Public Schools,2022-23,295,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,296,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,297,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,298,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,299,FALSE,Caucasian,M,Active,504,Not Eligible, +Maynard Public Schools,2022-23,300,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,301,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,302,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,303,FALSE,"Asian, Black",M,,,Not Eligible, +Maynard Public Schools,2022-23,304,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,305,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,306,FALSE,Caucasian,F,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,307,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,308,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,309,TRUE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,310,TRUE,Caucasian,F,,,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,311,TRUE,Caucasian,F,,,Free Lunch,LEP student 1st year +Maynard Public Schools,2022-23,312,TRUE,Caucasian,M,,504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,313,TRUE,Caucasian,M,Active,Not 504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,314,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,315,FALSE,Caucasian,M,Active,Not 504,Not Eligible, +Maynard Public Schools,2022-23,316,FALSE,Caucasian,M,Exited,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,317,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,318,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,319,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,320,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,321,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,322,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,323,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,324,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,325,FALSE,Caucasian,F,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,326,FALSE,Caucasian,F,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,327,TRUE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,328,TRUE,Caucasian,M,,,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,329,TRUE,Caucasian,F,,,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,330,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,331,FALSE,,M,,,Not Eligible, +Maynard Public Schools,2022-23,332,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,333,FALSE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,334,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,335,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,336,FALSE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,337,FALSE,Native American,M,Active,,Not Eligible, +Maynard Public Schools,2022-23,338,TRUE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,339,FALSE,"Black, Caucasian",F,,,Not Eligible, +Maynard Public Schools,2022-23,340,FALSE,"Black, Caucasian",F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,341,TRUE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,342,TRUE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,343,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,344,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,345,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,346,FALSE,Caucasian,F,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,347,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,348,FALSE,Caucasian,F,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,349,TRUE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,350,TRUE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,351,TRUE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,352,FALSE,Caucasian,M,,,Free Lunch, +Maynard Public Schools,2022-23,353,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,354,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,355,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,356,FALSE,Caucasian,F,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,357,FALSE,Caucasian,M,,504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,358,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,359,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,360,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,361,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,362,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,363,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,364,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,365,FALSE,Black,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,366,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,367,FALSE,Caucasian,M,,504,Not Eligible, +Maynard Public Schools,2022-23,368,FALSE,Caucasian,M,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,369,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,370,FALSE,Caucasian,M,,,Free Lunch, +Maynard Public Schools,2022-23,371,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,372,FALSE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,373,FALSE,"Asian,Caucasian",F,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,374,FALSE,Caucasian,M,,,Free Lunch, +Maynard Public Schools,2022-23,375,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,376,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,377,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,378,FALSE,Caucasian,F,Active,Not 504,Not Eligible, +Maynard Public Schools,2022-23,379,FALSE,Caucasian,F,Exited,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,380,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,381,FALSE,Caucasian,F,Active,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,382,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,383,TRUE,Black,F,Active,Not 504,Free Lunch,LEP student not 1st year +Maynard Public Schools,2022-23,384,TRUE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,385,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,386,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,387,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,388,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,389,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,390,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,391,FALSE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,392,FALSE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,393,FALSE,"Caucasian,Native American",F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,394,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,395,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,396,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,397,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,398,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,399,FALSE,"Caucasian,Native American",M,,504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,400,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,401,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,402,FALSE,Caucasian,F,,Not 504,Reduced Lunch,Does not apply +Maynard Public Schools,2022-23,403,FALSE,Caucasian,M,,Not 504,Reduced Lunch,Does not apply +Maynard Public Schools,2022-23,404,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,405,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,406,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,407,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,408,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,409,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,410,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,411,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,412,TRUE,Native American,F,,,Not Eligible, +Maynard Public Schools,2022-23,413,TRUE,Native American,F,,504,Not Eligible, +Maynard Public Schools,2022-23,414,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,415,FALSE,Asian,M,,Not 504,Not Eligible, +Maynard Public Schools,2022-23,416,TRUE,Native American,F,,,Not Eligible, +Maynard Public Schools,2022-23,417,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,418,TRUE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,419,FALSE,Asian,M,Exited,Not 504,Not Eligible,LEP student not 1st year +Maynard Public Schools,2022-23,420,FALSE,Asian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,421,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,422,FALSE,"Asian,Caucasian",F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,423,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,424,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,425,FALSE,Caucasian,M,Active,Not 504,Free Lunch, +Maynard Public Schools,2022-23,426,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,427,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,428,FALSE,Caucasian,F,,,Free Lunch,Does not apply +Maynard Public Schools,2022-23,429,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,430,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,431,TRUE,Caucasian,M,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,432,TRUE,Caucasian,M,,,Free Lunch,LEP student 1st year +Maynard Public Schools,2022-23,433,TRUE,Native American,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,434,TRUE,Native American,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,435,TRUE,Caucasian,F,,,Free Lunch, +Maynard Public Schools,2022-23,436,TRUE,Caucasian,M,Active,Not 504,Free Lunch, +Maynard Public Schools,2022-23,437,TRUE,Caucasian,M,,,Not Eligible,Does not apply +Maynard Public Schools,2022-23,438,TRUE,Caucasian,M,,Not 504,Not Eligible,LEP student not 1st year +Maynard Public Schools,2022-23,439,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,440,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,441,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,442,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,443,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,444,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,445,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,446,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,447,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,448,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,449,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,450,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,451,TRUE,Caucasian,F,,,Free Lunch,LEP student 1st year +Maynard Public Schools,2022-23,452,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,453,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,454,FALSE,Caucasian,M,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,455,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,456,FALSE,"Caucasian,Pacific Island",F,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,457,FALSE,Black,M,,,Free Lunch, +Maynard Public Schools,2022-23,458,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,459,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,460,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,461,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,462,TRUE,Caucasian,F,,,Free Lunch, +Maynard Public Schools,2022-23,463,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,464,FALSE,Caucasian,F,,Not 504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,465,FALSE,Caucasian,F,,,Not Eligible,Does not apply +Maynard Public Schools,2022-23,466,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,467,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,468,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,469,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,470,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,471,FALSE,Caucasian,N,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,472,FALSE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,473,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,474,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,475,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,476,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,477,TRUE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,478,TRUE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,479,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,480,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,481,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,482,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,483,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,484,FALSE,Caucasian,M,Active,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,485,FALSE,Caucasian,M,,,Not Eligible, +Maynard Public Schools,2022-23,486,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,487,FALSE,Caucasian,F,,504,Free Lunch,Does not apply +Maynard Public Schools,2022-23,488,TRUE,Caucasian,M,,,Not Eligible,LEP student not 1st year +Maynard Public Schools,2022-23,489,FALSE,Caucasian,F,Exited,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,490,FALSE,Caucasian,M,Active,Not 504,Free Lunch, +Maynard Public Schools,2022-23,491,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,492,FALSE,Caucasian,F,,504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,493,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,494,TRUE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,495,TRUE,Caucasian,M,Active,Not 504,Free Lunch, +Maynard Public Schools,2022-23,496,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,497,TRUE,Caucasian,F,,,Not Eligible, +Maynard Public Schools,2022-23,498,FALSE,Caucasian,M,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,499,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply +Maynard Public Schools,2022-23,500,FALSE,Caucasian,F,,Not 504,Not Eligible,Does not apply diff --git a/spec/fixtures/raw/prepped_2022_23_student_survey_responses.csv b/spec/fixtures/raw/bak/prepped_2022_23_student_survey_responses.csv similarity index 100% rename from spec/fixtures/raw/prepped_2022_23_student_survey_responses.csv rename to spec/fixtures/raw/bak/prepped_2022_23_student_survey_responses.csv diff --git a/spec/fixtures/raw/prepped_2022_23_teacher_survey_responses.csv b/spec/fixtures/raw/bak/prepped_2022_23_teacher_survey_responses.csv similarity index 100% rename from spec/fixtures/raw/prepped_2022_23_teacher_survey_responses.csv rename to spec/fixtures/raw/bak/prepped_2022_23_teacher_survey_responses.csv diff --git a/spec/fixtures/raw/sample_maynard_raw_student_survey.csv b/spec/fixtures/raw/sample_maynard_raw_student_survey.csv new file mode 100644 index 00000000..1a2b72e6 --- /dev/null +++ b/spec/fixtures/raw/sample_maynard_raw_student_survey.csv @@ -0,0 +1,36 @@ +StartDate,EndDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,District,School,LASID,Gender,Race,What grade are you in?,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,s-tint-q3,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,s-cure-q1,s-cure-q2,s-cure-q3,s-cure-q4,s-sten-q1,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,s-grmi-q2,s-grmi-q3,s-grmi-q4,s-appa-q1,s-appa-q2,s-appa-q3,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-sust-q1,s-sust-q2,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,s-expa-q1,s-poaf-q1,s-poaf-q2,s-poaf-q3,s-poaf-q4,s-tint-q1-1,s-tint-q2-1,s-tint-q3-1,s-tint-q4-1,s-tint-q5-1,s-acpr-q1-1,s-acpr-q2-1,s-acpr-q3-1,s-acpr-q4-1,s-peff-q1-1,s-peff-q2-1,s-peff-q3-1,s-peff-q4-1,s-peff-q5-1,s-peff-q6-1 +2023-03-17 7:57:47,2023-03-17 8:09:15,0,71.174.81.214,100,1000,1,2023-03-17T8:9:15,1000,2,1740505,1,2,4,9,3,5,5,,,,,,,,,,,,,,,,,,,,,,,,,4,4,3,5,,,,,,,,,,4,4,2,3,2,5,5,5,5,4,2,2,4,3,2,3,3,5,4,4,3,5,2,3,3,4,4,4,1,2,5,5,,,,,4,4,4,3,4,5 +2023-03-17 8:02:15,2023-03-17 8:08:02,0,71.174.81.214,25,1000,1,2023-03-17T8:8:3,1001,2,1740505,2,1,5,10,,,,,,,,,,,,,,,,,,,,5,4,4,4,,,,,,,,,2,3,2,,,,,,,4,3,2,4,3,5,5,4,4,4,4,3,5,3,4,3,2,4,3,4,3,3,1,2,2,2,3,,,,,,5,4,4,5,4,4,5,3,3,4 +2023-03-17 8:00:05,2023-03-17 8:07:39,0,71.174.81.214,24,1000,1,2023-03-17T8:7:39,1002,2,1740505,3,,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,4,4,5,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +2023-03-17 8:03:35,2023-03-17 8:15:38,0,71.174.81.214,0,1000,1,2023-03-17T8:15:38,1003,2,1740505,4,1,"1,4",10,4,5,5,,,,,,,,,,3,4,5,5,4,4,3,5,4,3,4,4,3,3,3,2,1,3,2,3,1,1,,,,,,,,,,,,,,,,,,,,,,,5,4,3,5,3,4,3,3,3,5,4,5,4,4,5,5,5,5,5,5,,,,,, +2023-03-17 7:57:09,2023-03-17 8:12:26,0,71.174.81.214,,1000,1,2023-03-17T8:12:27,1004,2,1740505,5,1,"5,4",9,4,4,5,,,,,,,,,,,,,,,,,,,,,,,,,4,4,2,4,,,,,,,,,,3,3,3,3,3,5,5,4,5,5,3,3,4,2,2,1,3,2,4,5,4,5,4,3,1,3,4,3,1,3,3,1,,,,,4,4,3,4,2,4 +2023-03-17 8:01:50,2023-03-17 8:17:51,0,71.174.81.214,100,240,1,2023-03-17T8:17:52,1005,2,1740505,6,1,"5,4",9,4,3,4,,,,,,,,,,4,4,3,3,3,4,4,4,5,2,5,4,4,4,4,4,4,5,3,4,3,5,,,,,,,4,4,4,5,2,,,,,,,,,,,,,,,,,,,,,,,4,1,4,4,4,5,4,4,5,4,5,5,5,5,4 +2023-03-17 8:01:45,2023-03-17 8:07:59,0,71.174.81.214,100,239,1,2023-03-17T8:8:0,1006,2,1740505,7,1,5,10,,,,,,,,,,,,,5,3,3,5,2,3,3,,,,,4,4,4,4,,,,,,,,,,,,,,4,5,3,4,3,5,5,4,5,5,4,4,4,2,1,1,4,5,4,4,3,4,2,2,3,2,3,,,,,,,,,,4,4,5,4,4,5 +2023-03-17 9:07:09,2023-03-17 9:20:10,0,71.174.81.214,100,0,1,2023-03-17T9:20:11,1007,2,1740305,8,2,5,7,,,,,,,,,,,,,4,5,5,4,,4,3,,,,,5,4,4,5,,,,,,,,,,,,,,5,5,4,5,5,5,5,5,5,4,5,5,5,3,2,3,3,3,5,4,4,4,3,3,4,,4,,,,,,,,,,4,5,5,5,5,5 +2023-03-17 8:02:11,2023-03-17 8:29:53,0,71.174.81.214,100,,1,2023-03-17T8:29:53,1008,2,1740505,9,1,"5,4",10,,,,,,,,,,,,,,,,,,,,3,3,2,3,,,,,,,,,2,3,3,,,,,,,3,4,3,3,3,5,5,5,5,4,4,2,3,2,3,2,2,5,2,4,3,3,1,3,3,3,4,,,,,,4,4,4,4,4,4,4,4,2,4 +2023-03-17 8:00:42,2023-03-17 8:12:00,0,71.174.81.214,100,1000,1,2023-03-17T8:12:0,1009,2,1740505,10,2,5,1,,,,,,,,,,,,,1,4,3,2,1,3,2,,,,,3,3,4,4,,,,,,,,,,,,,,5,5,2,4,5,4,4,5,4,2,2,1,2,2,4,3,5,5,4,4,1,4,1,2,1,3,3,,,,,,,,,,3,4,3,1,1,1 +2023-03-17 8:03:09,2023-03-17 8:13:27,0,71.174.81.214,100,1000,1,2023-03-17T8:13:28,1010,2,1740505,11,1,5,2,,,,,,,,,,,,,5,3,2,4,2,2,3,,,,,4,5,5,5,,,,,,,,,,,,,,4,4,3,4,4,4,4,4,5,4,3,5,4,2,1,2,4,4,5,4,3,5,4,1,3,3,3,,,,,,,,,,4,3,4,4,4,4 +2023-03-17 8:23:20,2023-03-17 8:34:00,0,71.174.81.214,100,1000,1,2023-03-17T8:34:0,1011,2,1740505,12,2,3,3,1,2,2,2,3,2,4,2,5,5,3,5,3,3,3,2,3,4,4,2,4,3,5,4,4,4,3,4,3,3,4,3,1,3,,,,,,,,,,,,,,,,,,,,,,,1,2,4,4,3,4,3,2,2,4,4,,,,,,,,,,,,,,, +2023-03-17 8:36:36,2023-03-17 8:47:33,0,71.174.81.214,100,1000,1,2023-03-17T8:47:34,1012,2,1740505,13,1,3,4,4,5,4,,,,,,,,,,4,2,3,2,2,3,4,4,5,3,4,2,4,2,3,3,4,3,3,2,1,1,,,,,,,,,,,,5,5,2,4,2,3,3,4,5,4,5,,,,,,,,,,,,4,2,3,3,2,4,4,3,3,,,,,, +2023-03-17 8:01:10,2023-03-17 8:09:17,0,71.174.81.214,100,1000,1,2023-03-17T8:9:18,1013,2,1740505,14,1,4,5,4,3,5,,,,,,,,,,3,4,3,4,3,4,4,2,4,4,2,3,1,2,2,4,2,3,5,2,2,1,,,,,,,4,4,3,3,4,,,,,,,,,,,,,,,,,,,,,,,2,1,3,2,5,4,5,2,2,2,3,4,3,3,4 +2023-03-17 10:06:07,2023-03-17 10:12:54,0,71.174.81.214,100,1000,1,2023-03-17T10:12:56,1014,2,1740505,15,1,5,6,,,,,,,,,,,,,3,3,4,2,1,2,4,,,,,2,2,2,2,,,,,,,,,,,,,,1,2,1,2,3,4,5,3,5,3,1,2,3,2,2,1,2,4,3,2,3,2,4,2,3,2,2,,,,,,,,,,4,4,4,4,4,5 +2023-03-17 7:57:13,2023-03-17 8:05:02,0,71.174.81.214,100,1000,1,2023-03-17T8:5:2,1015,2,1740505,16,4,5,7,,,,,,,,,,,,,,,,,,,,3,3,3,3,,,,,,,,,3,5,2,,,,,,,2,1,3,2,4,4,4,3,3,2,2,3,4,3,5,5,5,4,3,4,2,4,5,3,1,3,2,,,,,,4,4,3,4,4,5,4,5,5,5 +2023-03-17 7:57:50,2023-03-17 8:02:53,0,71.174.81.214,100,1000,1,2023-03-17T8:2:54,1016,2,1740505,17,2,5,8,1,1,1,,,,,,,,,,,,,,,,,,,,,,,,,5,2,1,1,,,,,,,,,,5,4,3,3,3,2,4,3,3,5,1,1,1,1,3,5,1,1,4,5,4,1,3,1,3,2,1,1,1,1,1,1,,,,,1,1,1,1,1,2 +2023-03-17 8:40:22,2023-03-17 8:53:19,0,71.174.81.214,100,1000,0,2023-03-18T8:53:20,1017,2,1740505,18,2,5,9,,,,,,,,,,,,,,,,,3,3,4,,,,,1,1,1,2,4,3,2,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,2,2,1,3,2,3,2,4,,,,,, +2023-03-17 8:37:13,2023-03-17 8:43:34,0,71.174.81.214,100,1000,1,2023-03-17T8:43:35,1018,2,1740505,19,2,2,10,,,,,,,,,,,,,,,,,,,,3,3,2,3,,,,,,,,,2,2,1,,,,,,,4,4,4,4,4,4,5,4,5,3,4,3,3,4,5,4,,,3,2,3,4,1,4,2,3,4,,,,,,4,5,5,4,4,4,5,4,3,4 +2023-03-17 8:36:27,2023-03-17 8:44:07,0,71.174.81.214,100,1000,1,2023-03-17T8:44:8,1019,2,1740505,20,1,2,11,3,4,3,,,,,,,,,,2,3,3,2,4,5,4,3,4,3,5,3,4,4,5,4,5,3,4,5,4,4,,,,,,,3,4,3,4,3,,,,,,,,,,,,,,,,,,,,,,,4,2,,2,4,4,5,2,4,5,5,4,3,3,4 +2023-03-17 8:33:55,2023-03-17 8:43:13,0,71.174.81.214,100,1000,1,2023-03-17T8:43:14,1020,2,1740505,21,1,5,12,3,1,3,,,,,,,,,,,,,,2,3,3,,4,,4,,4,,4,4,3,4,3,1,5,2,,,,,,,3,4,2,3,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,4,5,5,5,4,,4,4,4 +2023-03-17 8:50:49,2023-03-17 9:13:18,0,71.174.81.214,100,1000,1,2023-03-17T9:13:19,1021,2,1740305,22,2,"5,2",1,,,,,,,,,,,,,2,3,3,3,2,3,3,,,,,3,4,4,4,,,,,,,,,,,,,,2,2,1,3,2,5,5,4,5,4,2,2,2,1,1,1,1,3,3,4,3,4,4,1,1,1,3,,,,,,,,,,4,4,4,4,3,4 +2023-03-17 7:57:37,2023-03-17 8:04:25,0,71.174.81.214,100,1000,1,2023-03-17T8:4:25,1022,2,1740305,23,1,5,2,,,,,,,,,,,,,,,,,,,,4,4,4,5,,,,,,,,,4,4,5,,,,,,,3,3,4,4,4,4,5,4,5,4,3,3,3,2,2,2,3,3,3,2,3,4,4,3,3,2,4,,,,,,3,2,4,3,3,3,3,4,2,2 +2023-03-17 8:01:47,2023-03-17 8:08:39,0,71.174.81.214,100,1000,1,2023-03-17T8:8:39,1023,2,1740305,24,1,"2,4",3,4,3,5,,,,,,,,,,2,2,2,2,1,2,2,4,3,2,3,2,3,3,4,3,4,2,3,4,3,2,,,,,,,,,,,,5,4,3,5,3,1,2,2,2,2,4,,,,,,,,,,,,1,2,4,2,2,5,4,1,4,,,,,, +2023-03-17 8:37:21,2023-03-17 8:58:16,0,71.174.81.214,100,1000,1,2023-03-17T8:58:16,1024,2,1740305,25,1,5,4,3,3,3,,,,,,,,,,,,,,,,,,,,,,,,,4,4,2,3,,,,,,,,,,3,4,4,3,2,5,5,4,5,3,3,2,3,4,3,3,2,4,4,3,3,3,2,2,3,3,4,3,3,3,3,3,,,,,4,3,2,3,3,3 +2023-03-17 8:02:25,2023-03-17 8:11:16,0,71.174.81.214,100,1000,0,2023-03-18T8:11:21,1025,2,1740305,26,2,5,5,,,,,,,,,,,,,,,,,3,3,3,4,4,4,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,1,3,,,,,,4,3,3,3,3,,,,,,,,,, +2023-03-17 9:33:54,2023-03-17 9:57:21,0,71.174.81.214,100,1000,1,2023-03-17T9:57:22,1026,2,1740305,27,2,5,6,,,,,,,,,5,5,4,5,,,,,,,,1,2,2,2,,,,,,,,,1,1,2,5,4,4,4,3,5,5,4,,4,2,5,4,4,3,5,3,2,2,1,1,5,3,2,2,3,1,1,1,2,1,3,3,,,,,,,,,,,,,,, +2023-03-17 9:48:38,2023-03-17 9:58:45,0,71.174.81.214,100,1000,1,2023-03-17T9:58:45,1027,2,1740305,28,1,5,7,,,,,,,,,2,4,4,2,,,,,,,,3,3,3,5,,,,,,,,,2,3,3,4,3,2,2,3,2,1,3,2,3,2,3,3,3,2,4,3,4,2,2,1,1,5,5,2,3,2,3,5,4,3,2,2,,,,,,,,,,,,,,, +2023-03-17 8:36:40,2023-03-17 8:43:21,0,71.174.81.214,100,1000,1,2023-03-17T8:43:22,1028,2,1740305,29,2,5,8,,,,,,,,,,,,,,,,,,,,3,3,2,3,,,,,,,,,4,3,4,,,,,,,3,3,2,4,3,5,4,4,5,4,4,2,3,4,2,5,4,4,3,3,3,3,2,2,3,1,4,,,,,,3,4,4,4,2,4,5,4,3,4 +2023-03-17 9:40:56,2023-03-17 9:52:52,0,71.174.81.214,100,1000,1,2023-03-17T9:52:52,1029,2,1740305,30,2,5,9,3,4,3,5,3,5,5,5,5,4,4,5,4,4,4,4,2,3,2,4,4,3,4,3,4,3,5,5,5,4,3,5,4,4,,,,,,,,,,,,1,5,4,4,5,4,3,4,3,1,4,,,,,,,,,,,,,,,,,,,,,,,,,, +2023-03-17 9:33:58,2023-03-17 9:48:33,0,71.174.81.214,100,1000,1,2023-03-17T9:48:33,1030,2,1740305,31,2,5,10,,,,,,,,,4,3,5,2,,,,,,,,5,4,4,5,,,,,,,,,5,2,4,4,4,2,2,3,4,2,2,1,1,2,1,2,2,5,5,5,4,4,3,1,1,1,1,3,2,3,3,3,4,4,4,2,,,,,,,,,,,,,,, +2023-03-17 8:03:04,2023-03-17 8:23:33,0,71.174.81.214,100,1000,1,2023-03-17T8:23:33,1031,2,1740305,32,1,5,11,1,1,1,3,4,2,4,3,,,,,,,,,,,,,,,,,,,,5,5,5,5,,,,4,4,3,2,3,3,2,3,3,1,3,3,5,3,4,5,3,3,5,1,2,2,1,1,4,5,3,5,4,2,4,2,3,,,,,,,,,,,,,,, +2023-03-17 8:33:14,2023-03-17 8:41:01,0,71.174.81.214,100,1000,1,2023-03-17T8:41:2,1032,2,1740305,33,1,5,12,,,,,,,,,,,,,2,1,1,1,2,2,3,,,,,2,1,3,2,,,,,,,,,,,,,,1,1,1,1,1,4,3,4,5,4,1,1,2,3,2,3,4,2,2,3,3,2,2,2,1,2,3,,,,,,,,,,3,2,2,1,2,2 +2023-03-17 7:57:06,2023-03-17 8:08:35,0,71.174.81.214,100,1000,1,2023-03-17T8:8:35,1033,2,1740505,34,2,5,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,2,2,2,2,2,2,2,2,2 +2023-03-17 7:58:38,2023-03-17 8:12:04,0,71.174.81.214,100,1000,1,2023-03-17T8:12:5,1034,2,1740505,35,2,"5,4",12,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/spec/services/cleaner_spec.rb b/spec/services/cleaner_spec.rb index b01fb4e9..b572572a 100644 --- a/spec/services/cleaner_spec.rb +++ b/spec/services/cleaner_spec.rb @@ -2,13 +2,18 @@ require 'rails_helper' require 'fileutils' RSpec.describe Cleaner do - let(:district) { create(:district, name: 'District1') } + let(:district) { create(:district, name: 'Maynard Public Schools') } let(:second_district) { create(:district, name: 'District2') } let(:school) { create(:school, dese_id: 1_740_505, district:) } - let(:second_school) { create(:school, dese_id: 222_222, district: second_district) } + let(:second_school) { create(:school, dese_id: 1_740_305, district:) } + let(:third_school) { create(:school, dese_id: 222_222, district: second_district) } let(:academic_year) { create(:academic_year, range: '2022-23') } - let(:respondents) { create(:respondent, school:, academic_year:, nine: 40, ten: 40, eleven: 40, twelve: 40) } + let(:respondents) do + create(:respondent, school:, academic_year:, one: 0, nine: 40, ten: 40, eleven: 40, twelve: 40) + create(:respondent, school: second_school, academic_year:, one: 0, four: 40, five: 40, six: 40, seven: 40, + eight: 40) + end let(:recorded_date) { '2023-04-01' } let(:input_filepath) do Rails.root.join('spec', 'fixtures', 'raw') @@ -22,6 +27,18 @@ RSpec.describe Cleaner do Rails.root.join('tmp', 'spec', 'removed') end + let(:disaggregation_filepath) do + Rails.root.join('spec', 'fixtures', 'disaggregation') + end + + let(:path_to_sample_disaggregation_file) do + File.open(Rails.root.join('spec', 'fixtures', 'disaggregation', 'sample_maynard_disaggregation_data.csv')) + end + + let(:path_to_sample_raw_file) do + File.open(Rails.root.join('spec', 'fixtures', 'raw', 'sample_maynard_raw_student_survey.csv')) + end + let(:common_headers) do ['Recorded Date', 'DeseID', 'ResponseID'] end @@ -71,6 +88,7 @@ RSpec.describe Cleaner do before :each do school second_school + third_school standard_survey_items short_form_survey_items early_education_survey_items @@ -81,16 +99,64 @@ RSpec.describe Cleaner do context 'Creating a new Cleaner' do it 'creates a directory for the clean data' do - Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean + Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean expect(output_filepath).to exist end it 'creates a directory for the removed data' do - Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean + Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean expect(log_filepath).to exist end end + context '.process_raw_file' do + it 'sorts data into valid and invalid csvs' do + cleaner = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:) + processed_data = cleaner.process_raw_file( + file: path_to_sample_raw_file, disaggregation_data: cleaner.disaggregation_data + ) + processed_data in [headers, clean_csv, log_csv, data] + + reads_headers_from_raw_csv(processed_data) + + valid_rows = %w[1000 1001 1004 1005 1008 1017 1018 1019 1020 1024 1025 1026 + 1027 1028] + valid_rows.each do |response_id| + valid_row = data.find { |row| row.response_id == response_id } + expect(valid_row.valid?).to eq true + end + + invalid_rows = %w[1002 1003 1006 1007 1009 1010 1011 1012 1013 1014 1015 1016 1021 1022 1023 1029 1030 1031 1032 + 1033 1034] + invalid_rows.each do |response_id| + invalid_row = data.find { |row| row.response_id == response_id } + expect(invalid_row.valid?).to eq false + end + + expect(clean_csv.length).to eq valid_rows.length + 1 # headers + rows + expect(log_csv.length).to eq invalid_rows.length + 1 # headers + rows + + csv_contains_the_correct_rows(clean_csv, valid_rows) + csv_contains_the_correct_rows(log_csv, invalid_rows) + invalid_rows_are_rejected_for_the_correct_reasons(data) + end + + it 'adds dissaggregation data to the cleaned file ' do + cleaner = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:) + processed_data = cleaner.process_raw_file( + file: path_to_sample_raw_file, disaggregation_data: cleaner.disaggregation_data + ) + processed_data in [headers, clean_csv, log_csv, data] + expect(clean_csv.second.last).to eq 'Economically Disadvantaged - Y' + + one_thousand = data.find { |row| row.response_id == '1000' } + expect(one_thousand.income).to eq 'Economically Disadvantaged - Y' + + one_thousand_one = data.find { |row| row.response_id == '1001' } + expect(one_thousand_one.income).to eq 'Economically Disadvantaged - N' + end + end + context '.filename' do context 'defines a filename in the format: [district].[early_ed/short_form/standard/teacher].[year as 2022-23]' do context 'when the file is based on standard survey items' do @@ -99,10 +165,10 @@ RSpec.describe Cleaner do data = [SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '1_740_505' }, headers: standard_survey_items, genders: nil, survey_items:, schools: School.school_hash)] - filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( + filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename( headers: standard_survey_items, data: ) - expect(filename).to eq 'district1.standard.2022-23.csv' + expect(filename).to eq 'maynard.standard.2022-23.csv' end context 'when the file is based on short form survey items' do @@ -111,10 +177,10 @@ RSpec.describe Cleaner do data = [SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '1_740_505' }, headers: short_form_survey_items, genders: nil, survey_items:, schools: School.school_hash)] - filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( + filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename( headers: short_form_survey_items, data: ) - expect(filename).to eq 'district1.short_form.2022-23.csv' + expect(filename).to eq 'maynard.short_form.2022-23.csv' end end @@ -124,10 +190,10 @@ RSpec.describe Cleaner do data = [SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '1_740_505' }, headers: early_education_survey_items, genders: nil, survey_items:, schools: School.school_hash)] - filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( + filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename( headers: early_education_survey_items, data: ) - expect(filename).to eq 'district1.early_education.2022-23.csv' + expect(filename).to eq 'maynard.early_education.2022-23.csv' end end context 'when the file is based on teacher survey items' do @@ -136,10 +202,10 @@ RSpec.describe Cleaner do data = [SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '1_740_505' }, headers: teacher_survey_items, genders: nil, survey_items:, schools: School.school_hash)] - filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( + filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename( headers: teacher_survey_items, data: ) - expect(filename).to eq 'district1.teacher.2022-23.csv' + expect(filename).to eq 'maynard.teacher.2022-23.csv' end end @@ -150,13 +216,112 @@ RSpec.describe Cleaner do data = [SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '1_740_505' }, headers: teacher_survey_items, genders: nil, survey_items:, schools: School.school_hash), SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '222_222' }, headers: teacher_survey_items, genders: nil, survey_items:, schools: School.school_hash)] - filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename( + filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).filename( headers: teacher_survey_items, data: ) - expect(filename).to eq 'district1.district2.teacher.2022-23.csv' + expect(filename).to eq 'maynard.district2.teacher.2022-23.csv' end end end end end end + +def reads_headers_from_raw_csv(processed_data) + processed_data in [headers, clean_csv, log_csv, data] + expect(headers).to eq ['StartDate', 'EndDate', 'Status', 'IPAddress', 'Progress', 'Duration (in seconds)', + 'Finished', 'RecordedDate', 'ResponseId', 'District', 'School', + 'LASID', 'Gender', 'Race', 'What grade are you in?', 's-emsa-q1', 's-emsa-q2', 's-emsa-q3', 's-tint-q1', + 's-tint-q2', 's-tint-q3', 's-tint-q4', 's-tint-q5', 's-acpr-q1', 's-acpr-q2', + 's-acpr-q3', 's-acpr-q4', 's-cure-q1', 's-cure-q2', 's-cure-q3', 's-cure-q4', 's-sten-q1', 's-sten-q2', + 's-sten-q3', 's-sper-q1', 's-sper-q2', 's-sper-q3', 's-sper-q4', 's-civp-q1', 's-civp-q2', 's-civp-q3', + 's-civp-q4', 's-grmi-q1', 's-grmi-q2', 's-grmi-q3', 's-grmi-q4', 's-appa-q1', 's-appa-q2', 's-appa-q3', + 's-peff-q1', 's-peff-q2', 's-peff-q3', 's-peff-q4', 's-peff-q5', 's-peff-q6', 's-sbel-q1', 's-sbel-q2', + 's-sbel-q3', 's-sbel-q4', 's-sbel-q5', 's-phys-q1', 's-phys-q2', 's-phys-q3', 's-phys-q4', 's-vale-q1', + 's-vale-q2', 's-vale-q3', 's-vale-q4', 's-acst-q1', 's-acst-q2', 's-acst-q3', 's-sust-q1', 's-sust-q2', + 's-grit-q1', 's-grit-q2', 's-grit-q3', 's-grit-q4', 's-expa-q1', 's-poaf-q1', 's-poaf-q2', 's-poaf-q3', + 's-poaf-q4', 's-tint-q1-1', 's-tint-q2-1', 's-tint-q3-1', 's-tint-q4-1', 's-tint-q5-1', 's-acpr-q1-1', + 's-acpr-q2-1', 's-acpr-q3-1', 's-acpr-q4-1', 's-peff-q1-1', 's-peff-q2-1', 's-peff-q3-1', 's-peff-q4-1', + 's-peff-q5-1', 's-peff-q6-1', 'Raw Income', 'Income'] +end + +def invalid_rows_are_rejected_for_the_correct_reasons(data) + one_thousand_two = data.find { |row| row.response_id == '1002' } + expect(one_thousand_two.valid_progress?).to eq false + expect(one_thousand_two.valid_duration?).to eq true + expect(one_thousand_two.valid_grade?).to eq true + expect(one_thousand_two.valid_sd?).to eq true + + one_thousand_three = data.find { |row| row.response_id == '1003' } + expect(one_thousand_three.valid_progress?).to eq false + expect(one_thousand_three.valid_duration?).to eq true + expect(one_thousand_three.valid_grade?).to eq true + expect(one_thousand_three.valid_sd?).to eq true + + one_thousand_six = data.find { |row| row.response_id == '1006' } + expect(one_thousand_six.valid_progress?).to eq true + expect(one_thousand_six.valid_duration?).to eq false + expect(one_thousand_six.valid_grade?).to eq true + expect(one_thousand_six.valid_sd?).to eq true + + one_thousand_seven = data.find { |row| row.response_id == '1007' } + expect(one_thousand_seven.valid_progress?).to eq true + expect(one_thousand_seven.valid_duration?).to eq false + expect(one_thousand_seven.valid_grade?).to eq true + expect(one_thousand_seven.valid_sd?).to eq true + + one_thousand_seven = data.find { |row| row.response_id == '1007' } + expect(one_thousand_seven.valid_progress?).to eq true + expect(one_thousand_seven.valid_duration?).to eq false + expect(one_thousand_seven.valid_grade?).to eq true + expect(one_thousand_seven.valid_sd?).to eq true + + one_thousand_nine = data.find { |row| row.response_id == '1009' } + expect(one_thousand_nine.valid_progress?).to eq true + expect(one_thousand_nine.valid_duration?).to eq true + expect(one_thousand_nine.valid_grade?).to eq false + expect(one_thousand_nine.valid_sd?).to eq true + + one_thousand_ten = data.find { |row| row.response_id == '1010' } + expect(one_thousand_ten.valid_progress?).to eq true + expect(one_thousand_ten.valid_duration?).to eq true + expect(one_thousand_ten.valid_grade?).to eq false + expect(one_thousand_ten.valid_sd?).to eq true + + one_thousand_eleven = data.find { |row| row.response_id == '1011' } + expect(one_thousand_eleven.valid_progress?).to eq true + expect(one_thousand_eleven.valid_duration?).to eq true + expect(one_thousand_eleven.valid_grade?).to eq false + expect(one_thousand_eleven.valid_sd?).to eq true + + one_thousand_twenty_two = data.find { |row| row.response_id == '1022' } + expect(one_thousand_twenty_two.valid_progress?).to eq true + expect(one_thousand_twenty_two.valid_duration?).to eq true + expect(one_thousand_twenty_two.valid_grade?).to eq false + expect(one_thousand_twenty_two.valid_sd?).to eq true + + one_thousand_twenty_three = data.find { |row| row.response_id == '1023' } + expect(one_thousand_twenty_three.valid_progress?).to eq true + expect(one_thousand_twenty_three.valid_duration?).to eq true + expect(one_thousand_twenty_three.valid_grade?).to eq false + expect(one_thousand_twenty_three.valid_sd?).to eq true + + one_thousand_thirty_three = data.find { |row| row.response_id == '1033' } + expect(one_thousand_thirty_three.valid_progress?).to eq true + expect(one_thousand_thirty_three.valid_duration?).to eq true + expect(one_thousand_thirty_three.valid_grade?).to eq true + expect(one_thousand_thirty_three.valid_sd?).to eq false + + one_thousand_thirty_four = data.find { |row| row.response_id == '1034' } + expect(one_thousand_thirty_four.valid_progress?).to eq true + expect(one_thousand_thirty_four.valid_duration?).to eq true + expect(one_thousand_thirty_four.valid_grade?).to eq true + expect(one_thousand_thirty_four.valid_sd?).to eq false +end + +def csv_contains_the_correct_rows(csv, rows) + rows.each_with_index do |row, index| + response_id = 8 # eigth column + expect(csv[index + 1][response_id]).to eq row + end +end diff --git a/spec/services/disaggregation_loader_spec.rb b/spec/services/disaggregation_loader_spec.rb new file mode 100644 index 00000000..6b0ee739 --- /dev/null +++ b/spec/services/disaggregation_loader_spec.rb @@ -0,0 +1,35 @@ +require 'rails_helper' +require 'fileutils' + +RSpec.describe DisaggregationLoader do + let(:path) do + Rails.root.join('spec', 'fixtures', 'disaggregation') + end + let(:academic_year) { create(:academic_year, range: '2022-23') } + let(:district) { create(:district, name: 'Maynard Public Schools') } + context '.load' do + it 'loads data from the file into a hash' do + data = DisaggregationLoader.new(path:).load + expect(data.values.first.lasid).to eq('1') + expect(data.values.first.academic_year).to eq('2022-23') + expect(data.values.first.district).to eq('Maynard Public Schools') + expect(data.values.first.income).to eq('Free Lunch') + + expect(data.values.last.lasid).to eq('500') + expect(data.values.last.academic_year).to eq('2022-23') + expect(data.values.last.district).to eq('Maynard Public Schools') + expect(data.values.last.income).to eq('Not Eligible') + + expect(data[['1', 'Maynard Public Schools', '2022-23']].income).to eq('Free Lunch') + expect(data[['2', 'Maynard Public Schools', '2022-23']].income).to eq('Not Eligible') + expect(data[['3', 'Maynard Public Schools', '2022-23']].income).to eq('Reduced Lunch') + end + end + + context 'Creating a new loader' do + it 'creates a directory for the loader file' do + DisaggregationLoader.new(path:) + expect(path).to exist + end + end +end diff --git a/spec/services/disaggregation_row_spec.rb b/spec/services/disaggregation_row_spec.rb new file mode 100644 index 00000000..0fdfa4b3 --- /dev/null +++ b/spec/services/disaggregation_row_spec.rb @@ -0,0 +1,74 @@ +require 'rails_helper' + +RSpec.describe DisaggregationRow do + let(:headers) do + ['District', 'Academic Year', 'LASID', 'HispanicLatino', 'Race', 'Gender', 'SpecialEdStatus', 'In 504 Plan', + 'LowIncome', 'EL Student First Year'] + end + + context '.district' do + context 'when the column heading is any upper or lowercase variant of the word district' do + it 'returns the correct value for district' do + row = { 'District' => 'Maynard Public Schools' } + expect(DisaggregationRow.new(row:, headers:).district).to eq 'Maynard Public Schools' + + headers = ['dISTRICT'] + headers in [district] + row = { district => 'Maynard Public Schools' } + expect(DisaggregationRow.new(row:, headers:).district).to eq 'Maynard Public Schools' + end + end + end + + context '.academic_year' do + context 'when the column heading is any upper or lowercase variant of the words academic year' do + it 'returns the correct value for district' do + row = { 'Academic Year' => '2022-23' } + expect(DisaggregationRow.new(row:, headers:).academic_year).to eq '2022-23' + + headers = ['aCADEMIC yEAR'] + headers in [academic_year] + row = { academic_year => '2022-23' } + expect(DisaggregationRow.new(row:, headers:).academic_year).to eq '2022-23' + + headers = ['AcademicYear'] + headers in [academic_year] + row = { academic_year => '2022-23' } + expect(DisaggregationRow.new(row:, headers:).academic_year).to eq '2022-23' + end + end + end + + context '.income' do + context 'when the column heading is any upper or lowercase variant of the words low income' do + it 'returns the correct value for low_income' do + row = { 'LowIncome' => 'Free Lunch' } + expect(DisaggregationRow.new(row:, headers:).income).to eq 'Free Lunch' + + headers = ['Low income'] + headers in [income] + row = { income => 'Free Lunch' } + expect(DisaggregationRow.new(row:, headers:).income).to eq 'Free Lunch' + + headers = ['LoW InCOme'] + headers in [income] + row = { income => 'Free Lunch' } + expect(DisaggregationRow.new(row:, headers:).income).to eq 'Free Lunch' + end + end + end + + context '.lasid' do + context 'when the column heading is any upper or lowercase variant of the words lasid' do + it 'returns the correct value for lasid' do + row = { 'LASID' => '2366' } + expect(DisaggregationRow.new(row:, headers:).lasid).to eq '2366' + + headers = ['LaSiD'] + headers in [lasid] + row = { lasid => '2366' } + expect(DisaggregationRow.new(row:, headers:).lasid).to eq '2366' + end + end + end +end diff --git a/spec/services/survey_item_values_spec.rb b/spec/services/survey_item_values_spec.rb index 9a6ab7ef..6f3e5d01 100644 --- a/spec/services/survey_item_values_spec.rb +++ b/spec/services/survey_item_values_spec.rb @@ -1,44 +1,40 @@ -require 'rails_helper' +require "rails_helper" RSpec.describe SurveyItemValues, type: :model do let(:headers) do - ['StartDate', 'EndDate', 'Status', 'IPAddress', 'Progress', 'Duration (in seconds)', 'Finished', 'RecordedDate', - 'ResponseId', 'RecipientLastName', 'RecipientFirstName', 'RecipientEmail', 'ExternalReference', 'LocationLatitude', 'LocationLongitude', 'DistributionChannel', 'UserLanguage', 'District', 'School- Lee', 'School- Maynard', 'LASID', 'Grade', 's-emsa-q1', 's-emsa-q2', 's-emsa-q3', 's-tint-q1', 's-tint-q2', 's-tint-q3', 's-tint-q4', 's-tint-q5', 's-acpr-q1', 's-acpr-q2', 's-acpr-q3', 's-acpr-q4', 's-cure-q1', 's-cure-q2', 's-cure-q3', 's-cure-q4', 's-sten-q1', 's-sten-q2', 's-sten-q3', 's-sper-q1', 's-sper-q2', 's-sper-q3', 's-sper-q4', 's-civp-q1', 's-civp-q2', 's-civp-q3', 's-civp-q4', 's-grmi-q1', 's-grmi-q2', 's-grmi-q3', 's-grmi-q4', 's-appa-q1', 's-appa-q2', 's-appa-q3', 's-peff-q1', 's-peff-q2', 's-peff-q3', 's-peff-q4', 's-peff-q5', 's-peff-q6', 's-sbel-q1', 's-sbel-q2', 's-sbel-q3', 's-sbel-q4', 's-sbel-q5', 's-phys-q1', 's-phys-q2', 's-phys-q3', 's-phys-q4', 's-vale-q1', 's-vale-q2', 's-vale-q3', 's-vale-q4', 's-acst-q1', 's-acst-q2', 's-acst-q3', 's-sust-q1', 's-sust-q2', 's-grit-q1', 's-grit-q2', 's-grit-q3', 's-grit-q4', 's-expa-q1', 's-poaf-q1', 's-poaf-q2', 's-poaf-q3', 's-poaf-q4', 's-tint-q1-1', 's-tint-q2-1', 's-tint-q3-1', 's-tint-q4-1', 's-tint-q5-1', 's-acpr-q1-1', 's-acpr-q2-1', 's-acpr-q3-1', 's-acpr-q4-1', 's-peff-q1-1', 's-peff-q2-1', 's-peff-q3-1', 's-peff-q4-1', 's-peff-q5-1', 's-peff-q6-1', 'Gender', 'Race'] + ["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", "Finished", "RecordedDate", + "ResponseId", "RecipientLastName", "RecipientFirstName", "RecipientEmail", "ExternalReference", "LocationLatitude", "LocationLongitude", "DistributionChannel", "UserLanguage", "District", "School- Lee", "School- Maynard", "LASID", "Grade", "s-emsa-q1", "s-emsa-q2", "s-emsa-q3", "s-tint-q1", "s-tint-q2", "s-tint-q3", "s-tint-q4", "s-tint-q5", "s-acpr-q1", "s-acpr-q2", "s-acpr-q3", "s-acpr-q4", "s-cure-q1", "s-cure-q2", "s-cure-q3", "s-cure-q4", "s-sten-q1", "s-sten-q2", "s-sten-q3", "s-sper-q1", "s-sper-q2", "s-sper-q3", "s-sper-q4", "s-civp-q1", "s-civp-q2", "s-civp-q3", "s-civp-q4", "s-grmi-q1", "s-grmi-q2", "s-grmi-q3", "s-grmi-q4", "s-appa-q1", "s-appa-q2", "s-appa-q3", "s-peff-q1", "s-peff-q2", "s-peff-q3", "s-peff-q4", "s-peff-q5", "s-peff-q6", "s-sbel-q1", "s-sbel-q2", "s-sbel-q3", "s-sbel-q4", "s-sbel-q5", "s-phys-q1", "s-phys-q2", "s-phys-q3", "s-phys-q4", "s-vale-q1", "s-vale-q2", "s-vale-q3", "s-vale-q4", "s-acst-q1", "s-acst-q2", "s-acst-q3", "s-sust-q1", "s-sust-q2", "s-grit-q1", "s-grit-q2", "s-grit-q3", "s-grit-q4", "s-expa-q1", "s-poaf-q1", "s-poaf-q2", "s-poaf-q3", "s-poaf-q4", "s-tint-q1-1", "s-tint-q2-1", "s-tint-q3-1", "s-tint-q4-1", "s-tint-q5-1", "s-acpr-q1-1", "s-acpr-q2-1", "s-acpr-q3-1", "s-acpr-q4-1", "s-peff-q1-1", "s-peff-q2-1", "s-peff-q3-1", "s-peff-q4-1", "s-peff-q5-1", "s-peff-q6-1", "Gender", "Race"] end let(:genders) do create(:gender, qualtrics_code: 1) - gender_hash = {} - - Gender.all.each do |gender| - gender_hash[gender.qualtrics_code] = gender - end - gender_hash + Gender.gender_hash end let(:survey_items) { [] } + let(:district) { create(:district, name: "Attleboro") } let(:attleboro) do - create(:school, name: 'Attleboro', dese_id: 1234) + create(:school, name: "Attleboro", dese_id: 1234, district:) end let(:attleboro_respondents) do create(:respondent, school: attleboro, academic_year: ay_2022_23, nine: 40, ten: 40, eleven: 40, twelve: 40) end let(:schools) { School.school_hash } - let(:recorded_date) { '2023-04-01' } + let(:recorded_date) { "2023-04-01" } let(:ay_2022_23) do - create(:academic_year, range: '2022-23') + create(:academic_year, range: "2022-23") end let(:common_headers) do - ['Recorded Date', 'DeseID', 'ResponseID', 'Duration (in seconds)', 'Gender', 'Grade'] + ["Recorded Date", "DeseID", "ResponseID", "Duration (in seconds)", "Gender", "Grade"] end let(:standard_survey_items) do survey_item_ids = %w[s-peff-q1 s-peff-q2 s-peff-q3 s-peff-q4 s-peff-q5 s-peff-q6 s-phys-q1 s-phys-q2 s-phys-q3 s-phys-q4 - s-emsa-q1 s-emsa-q2 s-emsa-q3 s-sbel-q1 s-sbel-q2 s-sbel-q3 s-sbel-q4 s-sbel-q5 s-tint-q1 s-tint-q2 - s-tint-q3 s-tint-q4 s-tint-q5 s-vale-q1 s-vale-q2 s-vale-q3 s-vale-q4 s-acpr-q1 s-acpr-q2 s-acpr-q3 - s-acpr-q4 s-sust-q1 s-sust-q2 s-cure-q1 s-cure-q2 s-cure-q3 s-cure-q4 s-sten-q1 s-sten-q2 s-sten-q3 - s-sper-q1 s-sper-q2 s-sper-q3 s-sper-q4 s-civp-q1 s-civp-q2 s-civp-q3 s-civp-q4 s-grit-q1 s-grit-q2 - s-grit-q3 s-grit-q4 s-grmi-q1 s-grmi-q2 s-grmi-q3 s-grmi-q4 s-expa-q1 s-appa-q1 s-appa-q2 s-appa-q3 - s-acst-q1 s-acst-q2 s-acst-q3 s-poaf-q1 s-poaf-q2 s-poaf-q3 s-poaf-q4] + s-emsa-q1 s-emsa-q2 s-emsa-q3 s-sbel-q1 s-sbel-q2 s-sbel-q3 s-sbel-q4 s-sbel-q5 s-tint-q1 s-tint-q2 + s-tint-q3 s-tint-q4 s-tint-q5 s-vale-q1 s-vale-q2 s-vale-q3 s-vale-q4 s-acpr-q1 s-acpr-q2 s-acpr-q3 + s-acpr-q4 s-sust-q1 s-sust-q2 s-cure-q1 s-cure-q2 s-cure-q3 s-cure-q4 s-sten-q1 s-sten-q2 s-sten-q3 + s-sper-q1 s-sper-q2 s-sper-q3 s-sper-q4 s-civp-q1 s-civp-q2 s-civp-q3 s-civp-q4 s-grit-q1 s-grit-q2 + s-grit-q3 s-grit-q4 s-grmi-q1 s-grmi-q2 s-grmi-q3 s-grmi-q4 s-expa-q1 s-appa-q1 s-appa-q2 s-appa-q3 + s-acst-q1 s-acst-q2 s-acst-q3 s-poaf-q1 s-poaf-q2 s-poaf-q3 s-poaf-q4] survey_item_ids.map do |survey_item_id| create(:survey_item, survey_item_id:) end @@ -46,10 +42,10 @@ RSpec.describe SurveyItemValues, type: :model do end let(:short_form_survey_items) do - survey_item_ids = [create(:survey_item, survey_item_id: 's-phys-q1', on_short_form: true), - create(:survey_item, survey_item_id: 's-phys-q2', on_short_form: true), - create(:survey_item, survey_item_id: 's-phys-q3', - on_short_form: true)].map(&:survey_item_id) + survey_item_ids = [create(:survey_item, survey_item_id: "s-phys-q1", on_short_form: true), + create(:survey_item, survey_item_id: "s-phys-q2", on_short_form: true), + create(:survey_item, survey_item_id: "s-phys-q3", + on_short_form: true)].map(&:survey_item_id) survey_item_ids.map do |survey_item_id| create(:survey_item, survey_item_id:) end @@ -57,9 +53,9 @@ RSpec.describe SurveyItemValues, type: :model do end let(:early_education_survey_items) do - survey_item_ids = [create(:survey_item, survey_item_id: 's-emsa-es1'), - create(:survey_item, survey_item_id: 's-emsa-es2'), - create(:survey_item, survey_item_id: 's-emsa-es3')].map(&:survey_item_id) + survey_item_ids = [create(:survey_item, survey_item_id: "s-emsa-es1"), + create(:survey_item, survey_item_id: "s-emsa-es2"), + create(:survey_item, survey_item_id: "s-emsa-es3")].map(&:survey_item_id) survey_item_ids.map do |survey_item_id| create(:survey_item, survey_item_id:) end @@ -68,12 +64,12 @@ RSpec.describe SurveyItemValues, type: :model do let(:teacher_survey_items) do survey_item_ids = %w[t-prep-q1 t-prep-q2 t-prep-q3 t-ieff-q1 t-ieff-q2 t-ieff-q3 t-ieff-q4 t-pcom-q1 t-pcom-q2 t-pcom-q3 - t-pcom-q4 t-pcom-q5 t-inle-q1 t-inle-q2 t-inle-q3 t-prtr-q1 t-prtr-q2 t-prtr-q3 t-coll-q1 t-coll-q2 - t-coll-q3 t-qupd-q1 t-qupd-q2 t-qupd-q3 t-qupd-q4 t-pvic-q1 t-pvic-q2 t-pvic-q3 t-psup-q1 t-psup-q2 - t-psup-q3 t-psup-q4 t-acch-q1 t-acch-q2 t-acch-q3 t-reso-q1 t-reso-q2 t-reso-q3 t-reso-q4 t-reso-q5 - t-sust-q1 t-sust-q2 t-sust-q3 t-sust-q4 t-curv-q1 t-curv-q2 t-curv-q3 t-curv-q4 t-cure-q1 t-cure-q2 - t-cure-q3 t-cure-q4 t-peng-q1 t-peng-q2 t-peng-q3 t-peng-q4 t-ceng-q1 t-ceng-q2 t-ceng-q3 t-ceng-q4 - t-sach-q1 t-sach-q2 t-sach-q3 t-psol-q1 t-psol-q2 t-psol-q3 t-expa-q2 t-expa-q3 t-phya-q2 t-phya-q3] + t-pcom-q4 t-pcom-q5 t-inle-q1 t-inle-q2 t-inle-q3 t-prtr-q1 t-prtr-q2 t-prtr-q3 t-coll-q1 t-coll-q2 + t-coll-q3 t-qupd-q1 t-qupd-q2 t-qupd-q3 t-qupd-q4 t-pvic-q1 t-pvic-q2 t-pvic-q3 t-psup-q1 t-psup-q2 + t-psup-q3 t-psup-q4 t-acch-q1 t-acch-q2 t-acch-q3 t-reso-q1 t-reso-q2 t-reso-q3 t-reso-q4 t-reso-q5 + t-sust-q1 t-sust-q2 t-sust-q3 t-sust-q4 t-curv-q1 t-curv-q2 t-curv-q3 t-curv-q4 t-cure-q1 t-cure-q2 + t-cure-q3 t-cure-q4 t-peng-q1 t-peng-q2 t-peng-q3 t-peng-q4 t-ceng-q1 t-ceng-q2 t-ceng-q3 t-ceng-q4 + t-sach-q1 t-sach-q2 t-sach-q3 t-psol-q1 t-psol-q2 t-psol-q3 t-expa-q2 t-expa-q3 t-phya-q2 t-phya-q3] survey_item_ids.map do |survey_item_id| create(:survey_item, survey_item_id:) @@ -81,49 +77,49 @@ RSpec.describe SurveyItemValues, type: :model do (survey_item_ids << common_headers).flatten end - context '.recorded_date' do - it 'returns the recorded date' do - row = { 'RecordedDate' => '2017-01-01' } + context ".recorded_date" do + it "returns the recorded date" do + row = {"RecordedDate" => "2017-01-01"} values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) - expect(values.recorded_date).to eq Date.parse('2017-01-01') + expect(values.recorded_date).to eq Date.parse("2017-01-01") - headers = ['Recorded Date'] - row = { 'Recorded Date' => '2017-01-02' } + headers = ["Recorded Date"] + row = {"Recorded Date" => "2017-01-02"} values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) - expect(values.recorded_date).to eq Date.parse('2017-01-02') + expect(values.recorded_date).to eq Date.parse("2017-01-02") end end - context '.school' do - it 'returns the school that maps to the dese id provided' do + context ".school" do + it "returns the school that maps to the dese id provided" do attleboro - row = { 'Dese ID' => '1234' } + row = {"Dese ID" => "1234"} values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) expect(values.school).to eq attleboro - row = { 'DeseID' => '1234' } + row = {"DeseID" => "1234"} values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) expect(values.school).to eq attleboro end end - context '.grade' do - it 'returns the grade that maps to the grade provided' do - row = { 'Grade' => '1' } + context ".grade" do + it "returns the grade that maps to the grade provided" do + row = {"Grade" => "1"} values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) expect(values.grade).to eq 1 end end - context '.gender' do - it 'returns the grade that maps to the grade provided' do - row = { 'Gender' => '1' } + context ".gender" do + it "returns the grade that maps to the grade provided" do + row = {"Gender" => "1"} values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:) expect(values.gender.qualtrics_code).to eq 1 end end - context '.respondent_type' do - it 'reads header to find the survey type' do + context ".respondent_type" do + it "reads header to find the survey type" do headers = %w[s-sbel-q5 s-phys-q2 RecordedDate] values = SurveyItemValues.new(row: {}, headers:, genders:, survey_items:, schools:) expect(values.respondent_type).to eq :student @@ -134,32 +130,32 @@ RSpec.describe SurveyItemValues, type: :model do end end - context '.survey_type' do - context 'when survey type is standard form' do - it 'returns the survey type' do + context ".survey_type" do + context "when survey type is standard form" do + it "returns the survey type" do headers = standard_survey_items values = SurveyItemValues.new(row: {}, headers:, genders:, survey_items:, schools:) expect(values.survey_type).to eq :standard end end - context 'when survey type is teacher form' do - it 'returns the survey type' do + context "when survey type is teacher form" do + it "returns the survey type" do headers = teacher_survey_items values = SurveyItemValues.new(row: {}, headers:, genders:, survey_items:, schools:) expect(values.survey_type).to eq :teacher end end - context 'when survey type is short form' do - it 'returns the survey type' do + context "when survey type is short form" do + it "returns the survey type" do headers = short_form_survey_items values = SurveyItemValues.new(row: {}, headers:, genders:, survey_items:, schools:) expect(values.survey_type).to eq :short_form end end - context 'when survey type is early education' do - it 'returns the survey type' do + context "when survey type is early education" do + it "returns the survey type" do headers = early_education_survey_items values = SurveyItemValues.new(row: {}, headers:, genders:, survey_items:, schools:) expect(values.survey_type).to eq :early_education @@ -167,159 +163,236 @@ RSpec.describe SurveyItemValues, type: :model do end end - context '.valid_duration' do - context 'when duration is valid' do - it 'returns true' do + context ".income" do + context "when no disaggregation data is provided" do + it "returns an empty string " do + disaggregation_data = {} + values = SurveyItemValues.new(row: {}, headers:, genders:, survey_items:, schools:, disaggregation_data:) + expect(values.income).to eq "Unknown" + end + end + + context "when disaggregation data is provided" do + before :each do + attleboro + ay_2022_23 + end + + it "translates Free Lunch to Economically Disadvantaged - Y" do + headers = ["District", "Academic Year", "LASID", "LowIncome"] + row = {"District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Free Lunch"} + disaggregation_data = {%w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:)} + + headers = ["LASID", "Dese Id", "RecordedDate"] + row = {"LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1"} + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, + disaggregation_data:) + expect(values.income).to eq "Economically Disadvantaged - Y" + end + + it "translates Reduced Lunch to Economically Disadvantaged - Y" do + headers = ["District", "Academic Year", "LASID", "LowIncome"] + row = {"District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Reduced Lunch"} + disaggregation_data = {%w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:)} + + headers = ["LASID", "Dese Id", "RecordedDate"] + row = {"LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1"} + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, + disaggregation_data:) + expect(values.income).to eq "Economically Disadvantaged - Y" + end + + it "translates LowIncome to Economically Disadvantaged - Y" do + headers = ["District", "Academic Year", "LASID", "LowIncome"] + row = {"District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "LowIncome"} + disaggregation_data = {%w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:)} + + headers = ["LASID", "Dese Id", "RecordedDate"] + row = {"LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1"} + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, + disaggregation_data:) + expect(values.income).to eq "Economically Disadvantaged - Y" + end + + it "translates Not Eligible to Economically Disadvantaged - N" do + headers = ["District", "Academic Year", "LASID", "LowIncome"] + row = {"District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => "Not Eligible"} + disaggregation_data = {%w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:)} + + headers = ["LASID", "Dese Id", "RecordedDate"] + row = {"LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1"} + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, + disaggregation_data:) + expect(values.income).to eq "Economically Disadvantaged - N" + end + + it "translates blanks to Unknown" do + headers = ["District", "Academic Year", "LASID", "LowIncome"] + row = {"District" => "Attleboro", "AcademicYear" => "2022-23", "LASID" => "1", "LowIncome" => ""} + disaggregation_data = {%w[1 Attleboro 2022-23] => DisaggregationRow.new(row:, headers:)} + + headers = ["LASID", "Dese Id", "RecordedDate"] + row = {"LASID" => "1", "DESE ID" => "1234", "RecordedDate" => "2023-1-1"} + values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:, + disaggregation_data:) + expect(values.income).to eq "Unknown" + end + end + end + + context ".valid_duration" do + context "when duration is valid" do + it "returns true" do headers = standard_survey_items - values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '240', 'Gender' => 'Male' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Duration (in seconds)" => "240", "Gender" => "Male"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_duration?).to eq true headers = teacher_survey_items - values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '300' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Duration (in seconds)" => "300"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_duration?).to eq true headers = short_form_survey_items - values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '100' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Duration (in seconds)" => "100"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_duration?).to eq true # When duration is blank or N/A or NA, we don't have enough information to kick out the row as invalid so we keep it in headers = short_form_survey_items - values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Duration (in seconds)" => ""}, headers:, genders:, survey_items:, + schools:) expect(values.valid_duration?).to eq true headers = short_form_survey_items - values = SurveyItemValues.new(row: { 'Duration (in seconds)' => 'N/A' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Duration (in seconds)" => "N/A"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_duration?).to eq true headers = short_form_survey_items - values = SurveyItemValues.new(row: { 'Duration (in seconds)' => 'NA' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Duration (in seconds)" => "NA"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_duration?).to eq true end end - context 'when duration is invalid' do - it 'returns false' do + context "when duration is invalid" do + it "returns false" do headers = standard_survey_items - values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '239' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Duration (in seconds)" => "239"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_duration?).to eq false headers = teacher_survey_items - values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '299' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Duration (in seconds)" => "299"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_duration?).to eq false headers = short_form_survey_items - values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '99' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Duration (in seconds)" => "99"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_duration?).to eq false end end end - context '.valid_progress' do - context 'when progress is valid' do - it 'returns true' do + context ".valid_progress" do + context "when progress is valid" do + it "returns true" do headers = %w[s-sbel-q5 s-phys-q2 RecordedDate] - values = SurveyItemValues.new(row: { 'Progress' => '25' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Progress" => "25"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_progress?).to eq true # When progress is blank or N/A or NA, we don't have enough information to kick out the row as invalid so we keep it in headers = %w[s-sbel-q5 s-phys-q2 RecordedDate] - values = SurveyItemValues.new(row: { 'Progress' => '' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Progress" => ""}, headers:, genders:, survey_items:, + schools:) expect(values.valid_progress?).to eq true headers = %w[s-sbel-q5 s-phys-q2 RecordedDate] - values = SurveyItemValues.new(row: { 'Progress' => 'N/A' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Progress" => "N/A"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_progress?).to eq true headers = %w[s-sbel-q5 s-phys-q2 RecordedDate] - values = SurveyItemValues.new(row: { 'Progress' => 'NA' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Progress" => "NA"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_progress?).to eq true end end - context 'when progress is invalid' do - it 'returns false' do + context "when progress is invalid" do + it "returns false" do headers = %w[s-sbel-q5 s-phys-q2 RecordedDate] - values = SurveyItemValues.new(row: { 'Progress' => '24' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"Progress" => "24"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_progress?).to eq false end end end - context '.valid_grade?' do - context 'when grade is valid' do + context ".valid_grade?" do + context "when grade is valid" do before :each do attleboro attleboro_respondents end - it 'returns true for students' do + it "returns true for students" do headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { 'grade' => '9', 'RecordedDate' => recorded_date, 'Dese ID' => '1234' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"grade" => "9", "RecordedDate" => recorded_date, "Dese ID" => "1234"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_grade?).to eq true end - it 'returns true for teachers' do + it "returns true for teachers" do headers = %w[t-sbel-q5 t-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { 'RecordedDate' => recorded_date, 'Dese ID' => '1234' }, headers:, genders:, survey_items:, - schools:) + values = SurveyItemValues.new(row: {"RecordedDate" => recorded_date, "Dese ID" => "1234"}, headers:, genders:, survey_items:, + schools:) expect(values.valid_grade?).to eq true end end - context 'when grade is invalid' do + context "when grade is invalid" do before :each do attleboro attleboro_respondents end - it 'returns false' do + it "returns false" do headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate] - values = SurveyItemValues.new(row: { 'grade' => '2', 'RecordedDate' => recorded_date, 'Dese ID' => '1234' }, headers:, genders:, survey_items:, - schools: School.school_hash) + values = SurveyItemValues.new(row: {"grade" => "2", "RecordedDate" => recorded_date, "Dese ID" => "1234"}, headers:, genders:, survey_items:, + schools: School.school_hash) expect(values.valid_grade?).to eq false end end end - context '.valid_sd?' do - context 'when the standard deviation is valid' do - it 'returns true for student questions' do + context ".valid_sd?" do + context "when the standard deviation is valid" do + it "returns true for student questions" do headers = %w[s-sbel-q5 s-phys-q1 s-phys-q2 RecordedDate] - values = SurveyItemValues.new(row: { 'RecordedDate' => recorded_date, 'Dese ID' => '1234', 's-sbel-q5' => '1', 's-phys-q1' => '', 's-phys-q2' => '5' }, headers:, genders:, survey_items:, - schools: School.school_hash) + values = SurveyItemValues.new(row: {"RecordedDate" => recorded_date, "Dese ID" => "1234", "s-sbel-q5" => "1", "s-phys-q1" => "", "s-phys-q2" => "5"}, headers:, genders:, survey_items:, + schools: School.school_hash) expect(values.valid_sd?).to eq true end - it 'returns true for teacher questions' do + it "returns true for teacher questions" do headers = %w[t-sbel-q5 t-phys-q2] - values = SurveyItemValues.new(row: { 'RecordedDate' => recorded_date, 'Dese ID' => '1234', 't-sbel-q5' => '1', 't-phys-q2' => '5' }, headers:, genders:, survey_items:, - schools: School.school_hash) + values = SurveyItemValues.new(row: {"RecordedDate" => recorded_date, "Dese ID" => "1234", "t-sbel-q5" => "1", "t-phys-q2" => "5"}, headers:, genders:, survey_items:, + schools: School.school_hash) expect(values.valid_sd?).to eq true end end - context 'when the standard deviation is invalid' do - it 'returns false for student questions' do + context "when the standard deviation is invalid" do + it "returns false for student questions" do headers = %w[s-sbel-q5 s-phys-q1 s-phys-q2 RecordedDate] - values = SurveyItemValues.new(row: { 'RecordedDate' => recorded_date, 'Dese ID' => '1234', 's-sbel-q5' => '1', 's-phys-q2' => '', 's-phys-q2' => '1' }, headers:, genders:, survey_items:, - schools: School.school_hash) + values = SurveyItemValues.new(row: {"RecordedDate" => recorded_date, "Dese ID" => "1234", "s-sbel-q5" => "1", "s-phys-q2" => "", "s-phys-q2" => "1"}, headers:, genders:, survey_items:, + schools: School.school_hash) expect(values.valid_sd?).to eq false end - it 'returns false for teacher questions' do + it "returns false for teacher questions" do headers = %w[t-sbel-q5 t-phys-q1 t-phys-q2 RecordedDate] - values = SurveyItemValues.new(row: { 'RecordedDate' => recorded_date, 'Dese ID' => '1234', 't-sbel-q5' => '1', 't-phys-q2' => '', 't-phys-q2' => '1' }, headers:, genders:, survey_items:, - schools: School.school_hash) + values = SurveyItemValues.new(row: {"RecordedDate" => recorded_date, "Dese ID" => "1234", "t-sbel-q5" => "1", "t-phys-q2" => "", "t-phys-q2" => "1"}, headers:, genders:, survey_items:, + schools: School.school_hash) expect(values.valid_sd?).to eq false end end