chore: add parsing rules for disaggregation columns; ell, income, sped. Add tests for new rules

app/models/ell.rb

@@ -5,13 +5,22 @@ class Ell < ApplicationRecord
 
   friendly_id :designation, use: [:slugged]
   def self.to_designation(ell)
+    return "Not ELL" if ell.blank? || ell.nil?
+
+    ell = ell.delete(",")
     case ell
-    in /lep student 1st year|LEP student not 1st year|EL Student First Year|LEP\s*student|true/i
+    in /lep\s*student\s*1st\s*year|LEP\s*student\s*not\s*1st\s*year|EL\s*Student\s*First\s*Year|LEP\s*student|^EL|true/i
       "ELL"
-    in /^NA$|^#NA$/i
+    in /^NA$|^#NA$|^NA|0|Does\s*not\s*apply|Unknown/i
-      "Unknown"
-    else
       "Not ELL"
+    else
+      puts "************************************"
+      puts "********      ERROR       **********"
+      puts ""
+      puts "Error parsing English Language Learner column.  '#{ell}' is not a known value. Halting execution"
+      puts ""
+      puts "************************************"
+      exit
     end
   end
 end

app/models/income.rb

@@ -7,13 +7,23 @@ class Income < ApplicationRecord
   friendly_id :designation, use: [:slugged]
 
   def self.to_designation(income)
+    return "Unknown" if income.blank? or income.nil?
+
     case income
-    in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income|Reduced\s*price\s*lunch|true/i
+    in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income|Reduced\s*price\s*lunch|true|1/i
       "Economically Disadvantaged - Y"
-    in /Not\s*Eligible|false/i
+    in /Not\s*Eligible|false|0/i
       "Economically Disadvantaged - N"
-    else
+    in /Unknown/i
       "Unknown"
+    else
+      puts "************************************"
+      puts "********      ERROR       **********"
+      puts ""
+      puts "Error parsing Income column. '#{income}' is not a known value. Halting execution"
+      puts ""
+      puts "************************************"
+      exit
     end
   end
 

app/models/sped.rb

@@ -5,13 +5,23 @@ class Sped < ApplicationRecord
 
   friendly_id :designation, use: [:slugged]
   def self.to_designation(sped)
+    return "Not Special Education" if sped.blank? || sped.nil?
+
     case sped
-    in /active/i
+    in /active|^A$|1|Special\s*Education/i
       "Special Education"
-    in /^NA$|^#NA$/i
+    in /^I$|exited|0|Not\s*Special\s*Education|Does\s*not\s*apply/i
+      "Not Special Education"
+    in /^NA$|^#NA$|Unknown/i
       "Unknown"
     else
-      "Not Special Education"
+      puts "************************************"
+      puts "********      ERROR       **********"
+      puts ""
+      puts "Error parsing Special Education column. '#{sped}' is not a known value. Halting execution"
+      puts ""
+      puts "************************************"
+      exit
     end
   end
 end

app/services/survey_item_values.rb

@@ -169,7 +169,7 @@ class SurveyItemValues
   end
 
   def raw_income
-    @raw_income ||= value_from(pattern: /Low\s*Income|Raw\s*Income|SES-\s*SIS/i)
+    @raw_income ||= value_from(pattern: /Low\s*Income|Raw\s*Income|SES-\s*SIS|EconDisadvantaged|Income\s*SIS/i)
   end
 
   def income
@@ -177,7 +177,7 @@ class SurveyItemValues
   end
 
   def raw_ell
-    @raw_ell ||= value_from(pattern: /EL Student First Year|Raw\s*ELL|ELL-\s*SIS/i)
+    @raw_ell ||= value_from(pattern: /EL Student First Year|Raw\s*ELL|ELL-*\s*SIS/i)
   end
 
   def ell
@@ -185,7 +185,7 @@ class SurveyItemValues
   end
 
   def raw_sped
-    @raw_sped ||= value_from(pattern: /Special\s*Ed\s*Status|Raw\s*SpEd|SpEd-\s*SIS/i)
+    @raw_sped ||= value_from(pattern: /Special\s*Ed\s*Status|Raw\s*SpEd|SpEd-\s*SIS|SPED/i)
   end
 
   def sped

spec/services/survey_item_values_spec.rb

@@ -505,13 +505,34 @@ RSpec.describe SurveyItemValues, type: :model do
       expect(values.income).to eq "Economically Disadvantaged - N"
     end
 
-    it "translates blanks to Unknown" do
+    it "translates ones to Economically Disadvantaged - Y" do
       headers = ["LowIncome"]
-      row = { "LowIncome" => "" }
+      row = { "LowIncome" => "1" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
+      expect(values.income).to eq "Economically Disadvantaged - Y"
+    end
 
+    it "translates zeros to Economically Disadvantaged - N" do
+      headers = ["LowIncome"]
+      row = { "LowIncome" => "0" }
       values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
+      expect(values.income).to eq "Economically Disadvantaged - N"
+    end
+
+    it "translates blanks to Unknown" do
+      headers = ["LowIncome"]
+      row = { "LowIncome" => "" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
       expect(values.income).to eq "Unknown"
     end
+
+    # NOTE: This will halt test runs too
+    # it "halts execution if there is a value it cannot parse" do
+    #   headers = ["LowIncome"]
+    #   row = { "LowIncome" => "ArbitraryUnknownValue" }
+    #   output = capture_stdout { SurveyItemValues.new(row:, headers:, survey_items:, schools:) }
+    #   expect(output).to match "ArbitraryUnknownValue is not a known value. Halting execution"
+    # end
   end
 
   context ".ell" do
@@ -530,8 +551,20 @@ RSpec.describe SurveyItemValues, type: :model do
       values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
       expect(values.ell).to eq "ELL"
 
-      row = { "Raw ELL" => "LEP Student Not 1st Year" }
+      row = { "Raw ELL" => "LEP Student, Not 1st Year" }
-      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
+      expect(values.ell).to eq "ELL"
+
+      row = { "Raw ELL" => "EL student, not 1st year" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
+      expect(values.ell).to eq "ELL"
+
+      row = { "Raw ELL" => "EL student, 1st year" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
+      expect(values.ell).to eq "ELL"
+
+      row = { "Raw ELL" => "EL - Early Child. or PK" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
       expect(values.ell).to eq "ELL"
     end
 
@@ -546,16 +579,40 @@ RSpec.describe SurveyItemValues, type: :model do
       expect(values.ell).to eq "Not ELL"
     end
 
-    it 'tranlsates blanks into "Not Ell"' do
+    it 'tranlsates zeros into "Not ELL"' do
+      headers = ["Raw ELL"]
+      row = { "Raw ELL" => "0" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
+      expect(values.ell).to eq "Not ELL"
+    end
+
+    it 'tranlsates NAs and blanks into "Not ELL"' do
       headers = ["Raw ELL"]
       row = { "Raw ELL" => "" }
       values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
       expect(values.ell).to eq "Not ELL"
 
-      row = { "Raw ELL" => "Anything else" }
+      row = { "Raw ELL" => "NA" }
-      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
+      expect(values.ell).to eq "Not ELL"
+
+      row = { "Raw ELL" => "#NA" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
+      expect(values.ell).to eq "Not ELL"
+
+      row = { "Raw ELL" => "#N/A" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
       expect(values.ell).to eq "Not ELL"
     end
+
+    # NOTE: This will halt test runs too
+    # it "halts execution if there is a value it cannot parse" do
+    #   headers = ["Raw ELL"]
+    #   row = { "Raw ELL" => "ArbitraryUnknownValue" }
+
+    #   output = capture_stdout { SurveyItemValues.new(row:, headers:, survey_items:, schools:) }
+    #   expect(output).to match "ArbitraryUnknownValue is not a known value. Halting execution"
+    # end
   end
 
   context ".sped" do
@@ -571,6 +628,34 @@ RSpec.describe SurveyItemValues, type: :model do
       expect(values.sped).to eq "Special Education"
     end
 
+    it 'translates "1" into "Special Education"' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "1" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
+      expect(values.sped).to eq "Special Education"
+    end
+
+    it 'translates "A" into "Special Education"' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "A" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
+      expect(values.sped).to eq "Special Education"
+    end
+
+    it 'translates "I" into "Not Special Education"' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "I" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
+      expect(values.sped).to eq "Not Special Education"
+    end
+
+    it 'translates "0" into "Not Special Education"' do
+      headers = ["Raw SpEd"]
+      row = { "Raw SpEd" => "0" }
+      values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
+      expect(values.sped).to eq "Not Special Education"
+    end
+
     it 'translates "exited" into "Not Special Education"' do
       headers = ["Raw SpEd"]
       row = { "Raw SpEd" => "exited" }
@@ -585,7 +670,7 @@ RSpec.describe SurveyItemValues, type: :model do
       expect(values.sped).to eq "Not Special Education"
     end
 
-    it 'tranlsates NA into "Not Special Education"' do
+    it 'translates NA into "Not Special Education"' do
       headers = ["Raw SpEd"]
       row = { "Raw SpEd" => "NA" }
       values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
@@ -595,6 +680,14 @@ RSpec.describe SurveyItemValues, type: :model do
       values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
       expect(values.sped).to eq "Not Special Education"
     end
+
+    # NOTE: This will halt test runs too
+    # it "halts execution if there is a value it cannot parse" do
+    #   headers = ["Raw SpEd"]
+    #   row = { "Raw SpEd" => "ArbitraryUnknownValue" }
+    #   output = capture_stdout { SurveyItemValues.new(row:, headers:, survey_items:, schools:) }
+    #   expect(output).to match "ArbitraryUnknownValue is not a known value. Halting execution"
+    # end
   end
 
   context ".valid_duration" do