From 9aeb5f92af58b61ef95ac4d57b6a0635735daf72 Mon Sep 17 00:00:00 2001 From: rebuilt Date: Fri, 2 Jun 2023 15:13:53 -0700 Subject: [PATCH] Missing progress or duration information does not result in a row removed in the cleaning process --- app/services/survey_item_values.rb | 35 +++++++++++++--------- spec/services/survey_item_values_spec.rb | 37 ++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index cb3530d4..66cd2733 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -51,7 +51,7 @@ class SurveyItemValues def dese_id @dese_id ||= begin dese_id = nil - dese_headers = ['DESE ID', 'Dese ID', 'DeseId', 'DeseID', 'School', 'school'] + dese_headers = ["DESE ID", "Dese ID", "DeseId", "DeseID", "School", "school"] school_headers = headers.select { |header| /School-\s\w/.match(header) } dese_headers << school_headers dese_headers.flatten.each do |header| @@ -110,7 +110,7 @@ class SurveyItemValues end def duration - @duration ||= value_from(pattern: /Duration|Duration \(in seconds\)|Duration\.\.\(in\.seconds\)/i).to_i + @duration ||= value_from(pattern: /Duration|Duration \(in seconds\)|Duration\.\.\(in\.seconds\)/i) end def valid? @@ -119,30 +119,37 @@ class SurveyItemValues def respondent_type return :teacher if headers - .filter(&:present?) - .filter { |header| header.start_with? 't-' }.count > 0 + .filter(&:present?) + .filter { |header| header.start_with? "t-" }.count > 0 :student end def survey_type survey_item_ids = headers - .filter(&:present?) - .filter { |header| header.start_with?('t-', 's-') } + .filter(&:present?) + .filter { |header| header.start_with?("t-", "s-") } SurveyItem.survey_type(survey_item_ids:) end def valid_duration? - return duration >= 300 if survey_type == :teacher - return duration >= 240 if survey_type == :standard - return duration >= 100 if survey_type == :short_form + return true if duration.nil? || duration == "" || duration.downcase == "n/a" || duration.downcase == "na" + + span_in_seconds = duration.to_i + return span_in_seconds >= 300 if survey_type == :teacher + return span_in_seconds >= 240 if survey_type == :standard + return span_in_seconds >= 100 if survey_type == :short_form true end def valid_progress? - row['Progress'].to_i >= 25 + progress = row["Progress"] + return true if progress.nil? || progress == "" || progress.downcase == "n/a" || progress.downcase == "na" + + progress = progress.to_i + progress.to_i >= 25 end def valid_grade? @@ -162,7 +169,7 @@ class SurveyItemValues end def valid_sd? - survey_item_headers = headers.filter(&:present?).filter { |header| header.start_with?('s-', 't-') } + survey_item_headers = headers.filter(&:present?).filter { |header| header.start_with?("s-", "t-") } likert_scores = [] survey_item_headers.each do |header| likert_scores << likert_score(survey_item_id: header).to_i @@ -180,9 +187,9 @@ class SurveyItemValues private def copy_likert_scores_from_variant_survey_items - headers.filter(&:present?).filter { |header| header.end_with? '-1' }.each do |header| + headers.filter(&:present?).filter { |header| header.end_with? "-1" }.each do |header| likert_score = row[header] - main_item = header.gsub('-1', '') + main_item = header.gsub("-1", "") row[main_item] = likert_score if likert_score.present? end end @@ -192,7 +199,7 @@ module RowMonkeyPatches def remove_unwanted_columns to_h.filter do |key, _value| key.present? - end.reject { |key, _value| key.start_with? 'Q' }.reject { |key, _value| key.end_with? '-1' }.values + end.reject { |key, _value| key.start_with? "Q" }.reject { |key, _value| key.end_with? "-1" }.values end end diff --git a/spec/services/survey_item_values_spec.rb b/spec/services/survey_item_values_spec.rb index 78827e7f..9a6ab7ef 100644 --- a/spec/services/survey_item_values_spec.rb +++ b/spec/services/survey_item_values_spec.rb @@ -179,6 +179,27 @@ RSpec.describe SurveyItemValues, type: :model do values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '300' }, headers:, genders:, survey_items:, schools:) expect(values.valid_duration?).to eq true + + headers = short_form_survey_items + values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '100' }, headers:, genders:, survey_items:, + schools:) + expect(values.valid_duration?).to eq true + + # When duration is blank or N/A or NA, we don't have enough information to kick out the row as invalid so we keep it in + headers = short_form_survey_items + values = SurveyItemValues.new(row: { 'Duration (in seconds)' => '' }, headers:, genders:, survey_items:, + schools:) + expect(values.valid_duration?).to eq true + + headers = short_form_survey_items + values = SurveyItemValues.new(row: { 'Duration (in seconds)' => 'N/A' }, headers:, genders:, survey_items:, + schools:) + expect(values.valid_duration?).to eq true + + headers = short_form_survey_items + values = SurveyItemValues.new(row: { 'Duration (in seconds)' => 'NA' }, headers:, genders:, survey_items:, + schools:) + expect(values.valid_duration?).to eq true end end @@ -208,6 +229,22 @@ RSpec.describe SurveyItemValues, type: :model do values = SurveyItemValues.new(row: { 'Progress' => '25' }, headers:, genders:, survey_items:, schools:) expect(values.valid_progress?).to eq true + + # When progress is blank or N/A or NA, we don't have enough information to kick out the row as invalid so we keep it in + headers = %w[s-sbel-q5 s-phys-q2 RecordedDate] + values = SurveyItemValues.new(row: { 'Progress' => '' }, headers:, genders:, survey_items:, + schools:) + expect(values.valid_progress?).to eq true + + headers = %w[s-sbel-q5 s-phys-q2 RecordedDate] + values = SurveyItemValues.new(row: { 'Progress' => 'N/A' }, headers:, genders:, survey_items:, + schools:) + expect(values.valid_progress?).to eq true + + headers = %w[s-sbel-q5 s-phys-q2 RecordedDate] + values = SurveyItemValues.new(row: { 'Progress' => 'NA' }, headers:, genders:, survey_items:, + schools:) + expect(values.valid_progress?).to eq true end end