diff --git a/app/models/ell.rb b/app/models/ell.rb index 9312e786..b94d1eb2 100644 --- a/app/models/ell.rb +++ b/app/models/ell.rb @@ -5,13 +5,22 @@ class Ell < ApplicationRecord friendly_id :designation, use: [:slugged] def self.to_designation(ell) + return "Not ELL" if ell.blank? || ell.nil? + + ell = ell.delete(",") case ell - in /lep student 1st year|LEP student not 1st year|EL Student First Year|LEP\s*student|true/i + in /lep\s*student\s*1st\s*year|LEP\s*student\s*not\s*1st\s*year|EL\s*Student\s*First\s*Year|LEP\s*student|^EL|true/i "ELL" - in /^NA$|^#NA$/i - "Unknown" - else + in /^NA$|^#NA$|^NA|0|Does\s*not\s*apply|Unknown/i "Not ELL" + else + puts "************************************" + puts "******** ERROR **********" + puts "" + puts "Error parsing English Language Learner column. '#{ell}' is not a known value. Halting execution" + puts "" + puts "************************************" + exit end end end diff --git a/app/models/income.rb b/app/models/income.rb index 7adcc600..f0c37179 100644 --- a/app/models/income.rb +++ b/app/models/income.rb @@ -7,13 +7,23 @@ class Income < ApplicationRecord friendly_id :designation, use: [:slugged] def self.to_designation(income) + return "Unknown" if income.blank? or income.nil? + case income - in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income|Reduced\s*price\s*lunch|true/i + in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income|Reduced\s*price\s*lunch|true|1/i "Economically Disadvantaged - Y" - in /Not\s*Eligible|false/i + in /Not\s*Eligible|false|0/i "Economically Disadvantaged - N" - else + in /Unknown/i "Unknown" + else + puts "************************************" + puts "******** ERROR **********" + puts "" + puts "Error parsing Income column. '#{income}' is not a known value. Halting execution" + puts "" + puts "************************************" + exit end end diff --git a/app/models/sped.rb b/app/models/sped.rb index 8769ada3..c1ca20a9 100644 --- a/app/models/sped.rb +++ b/app/models/sped.rb @@ -5,13 +5,23 @@ class Sped < ApplicationRecord friendly_id :designation, use: [:slugged] def self.to_designation(sped) + return "Not Special Education" if sped.blank? || sped.nil? + case sped - in /active/i + in /active|^A$/i "Special Education" - in /^NA$|^#NA$/i + in /^I$|exited/i + "Not Special Education" + in /^NA$|^#NA$|Unknown/i "Unknown" else - "Not Special Education" + puts "************************************" + puts "******** ERROR **********" + puts "" + puts "Error parsing Special Education column. '#{sped}' is not a known value. Halting execution" + puts "" + puts "************************************" + exit end end end diff --git a/spec/fixtures/raw/sample_file_with_duplicate_headers.csv b/spec/fixtures/raw/sample_file_with_duplicate_headers.csv index 305af074..2eaf0323 100644 --- a/spec/fixtures/raw/sample_file_with_duplicate_headers.csv +++ b/spec/fixtures/raw/sample_file_with_duplicate_headers.csv @@ -1,2 +1,2 @@ -StartDate,StartDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,District,School,LASID,Gender,Race,What grade are you in?,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,s-tint-q3,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,s-cure-q1,s-cure-q2,s-cure-q3,s-cure-q4,s-sten-q1,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,s-grmi-q2,s-grmi-q3,s-grmi-q4,s-appa-q1,s-appa-q2,s-appa-q3,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-sust-q1,s-sust-q2,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,s-expa-q1,s-poaf-q1,s-poaf-q2,s-poaf-q3,s-poaf-q4,s-tint-q1-1,s-tint-q2-1,s-tint-q3-1,s-tint-q4-1,s-tint-q5-1,s-acpr-q1-1,s-acpr-q2-1,s-acpr-q3-1,s-acpr-q4-1,s-peff-q1-1,s-peff-q2-1,s-peff-q3-1,s-peff-q4-1,s-peff-q5-1,s-peff-q6-1 -2023-03-17 7:57:47,2023-03-17 8:09:15,0,71.174.81.214,100,1000,1,2023-03-17T8:9:15,1000,2,1740505,1,2,4,9,3,5,5,,,,,,,,,,,,,,,,,,,,,,,,,4,4,3,5,,,,,,,,,4,4,2,3,2,5,5,5,5,4,2,2,4,3,2,3,3,5,4,4,3,5,2,3,3,4,4,4,1,2,5,5,,,,,4,4,4,3,4,5 +Raw Income,StartDate,StartDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,District,School,LASID,Gender,Race,What grade are you in?,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,s-tint-q3,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,s-cure-q1,s-cure-q2,s-cure-q3,s-cure-q4,s-sten-q1,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,s-grmi-q2,s-grmi-q3,s-grmi-q4,s-appa-q1,s-appa-q2,s-appa-q3,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-sust-q1,s-sust-q2,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,s-expa-q1,s-poaf-q1,s-poaf-q2,s-poaf-q3,s-poaf-q4,s-tint-q1-1,s-tint-q2-1,s-tint-q3-1,s-tint-q4-1,s-tint-q5-1,s-acpr-q1-1,s-acpr-q2-1,s-acpr-q3-1,s-acpr-q4-1,s-peff-q1-1,s-peff-q2-1,s-peff-q3-1,s-peff-q4-1,s-peff-q5-1,s-peff-q6-1 +Unknown,2023-03-17 7:57:47,2023-03-17 8:09:15,0,71.174.81.214,100,1000,1,2023-03-17T8:9:15,1000,2,1740505,1,2,4,9,3,5,5,,,,,,,,,,,,,,,,,,,,,,,,,4,4,3,5,,,,,,,,,4,4,2,3,2,5,5,5,5,4,2,2,4,3,2,3,3,5,4,4,3,5,2,3,3,4,4,4,1,2,5,5,,,,,4,4,4,3,4,5 diff --git a/spec/services/survey_item_values_spec.rb b/spec/services/survey_item_values_spec.rb index 1ddbd785..501859ed 100644 --- a/spec/services/survey_item_values_spec.rb +++ b/spec/services/survey_item_values_spec.rb @@ -505,13 +505,34 @@ RSpec.describe SurveyItemValues, type: :model do expect(values.income).to eq "Economically Disadvantaged - N" end - it "translates blanks to Unknown" do + it "translates ones to Economically Disadvantaged - Y" do headers = ["LowIncome"] - row = { "LowIncome" => "" } + row = { "LowIncome" => "1" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:) + expect(values.income).to eq "Economically Disadvantaged - Y" + end + it "translates zeros to Economically Disadvantaged - N" do + headers = ["LowIncome"] + row = { "LowIncome" => "0" } values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:) + expect(values.income).to eq "Economically Disadvantaged - N" + end + + it "translates blanks to Unknown" do + headers = ["LowIncome"] + row = { "LowIncome" => "" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:) expect(values.income).to eq "Unknown" end + + # NOTE: This will halt test runs too + # it "halts execution if there is a value it cannot parse" do + # headers = ["LowIncome"] + # row = { "LowIncome" => "ArbitraryUnknownValue" } + # output = capture_stdout { SurveyItemValues.new(row:, headers:, survey_items:, schools:) } + # expect(output).to match "ArbitraryUnknownValue is not a known value. Halting execution" + # end end context ".ell" do @@ -530,8 +551,20 @@ RSpec.describe SurveyItemValues, type: :model do values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:) expect(values.ell).to eq "ELL" - row = { "Raw ELL" => "LEP Student Not 1st Year" } - values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:) + row = { "Raw ELL" => "LEP Student, Not 1st Year" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:) + expect(values.ell).to eq "ELL" + + row = { "Raw ELL" => "EL student, not 1st year" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:) + expect(values.ell).to eq "ELL" + + row = { "Raw ELL" => "EL student, 1st year" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:) + expect(values.ell).to eq "ELL" + + row = { "Raw ELL" => "EL - Early Child. or PK" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:) expect(values.ell).to eq "ELL" end @@ -546,16 +579,40 @@ RSpec.describe SurveyItemValues, type: :model do expect(values.ell).to eq "Not ELL" end - it 'tranlsates blanks into "Not Ell"' do + it 'tranlsates zeros into "Not ELL"' do + headers = ["Raw ELL"] + row = { "Raw ELL" => "0" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:) + expect(values.ell).to eq "Not ELL" + end + + it 'tranlsates NAs and blanks into "Not ELL"' do headers = ["Raw ELL"] row = { "Raw ELL" => "" } values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:) expect(values.ell).to eq "Not ELL" - row = { "Raw ELL" => "Anything else" } - values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:) + row = { "Raw ELL" => "NA" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:) + expect(values.ell).to eq "Not ELL" + + row = { "Raw ELL" => "#NA" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:) + expect(values.ell).to eq "Not ELL" + + row = { "Raw ELL" => "#N/A" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:) expect(values.ell).to eq "Not ELL" end + + # NOTE: This will halt test runs too + # it "halts execution if there is a value it cannot parse" do + # headers = ["Raw ELL"] + # row = { "Raw ELL" => "ArbitraryUnknownValue" } + + # output = capture_stdout { SurveyItemValues.new(row:, headers:, survey_items:, schools:) } + # expect(output).to match "ArbitraryUnknownValue is not a known value. Halting execution" + # end end context ".sped" do @@ -571,6 +628,20 @@ RSpec.describe SurveyItemValues, type: :model do expect(values.sped).to eq "Special Education" end + it 'translates "A" into "Special Education"' do + headers = ["Raw SpEd"] + row = { "Raw SpEd" => "A" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:) + expect(values.sped).to eq "Special Education" + end + + it 'translates "I" into "Not Special Education"' do + headers = ["Raw SpEd"] + row = { "Raw SpEd" => "I" } + values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:) + expect(values.sped).to eq "Not Special Education" + end + it 'translates "exited" into "Not Special Education"' do headers = ["Raw SpEd"] row = { "Raw SpEd" => "exited" } @@ -585,7 +656,7 @@ RSpec.describe SurveyItemValues, type: :model do expect(values.sped).to eq "Not Special Education" end - it 'tranlsates NA into "Not Special Education"' do + it 'translates NA into "Not Special Education"' do headers = ["Raw SpEd"] row = { "Raw SpEd" => "NA" } values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:) @@ -595,6 +666,14 @@ RSpec.describe SurveyItemValues, type: :model do values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:) expect(values.sped).to eq "Not Special Education" end + + # NOTE: This will halt test runs too + # it "halts execution if there is a value it cannot parse" do + # headers = ["Raw SpEd"] + # row = { "Raw SpEd" => "ArbitraryUnknownValue" } + # output = capture_stdout { SurveyItemValues.new(row:, headers:, survey_items:, schools:) } + # expect(output).to match "ArbitraryUnknownValue is not a known value. Halting execution" + # end end context ".valid_duration" do