edcommonwealth
/
sqm-dashboards
mirror of https://github.com/edcommonwealth/sqm-dashboards.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add ell income and sped parsing rules for SIS data. Add tests for the

Browse Source 
new inputs.
rpp-main
Nelson Jovel 2 years ago
parent 2561fa28fc
commit 65c25fc3c7
5 changed files with 128 additions and 20 deletions
Show Stats Download Patch File Download Diff File

17
app/models/ell.rb
Unescape View File

@ -5,13 +5,22 @@ class Ell < ApplicationRecord
friendly_id :designation, use: [:slugged]
def self.to_designation(ell)
return "Not ELL" if ell.blank? || ell.nil?
ell = ell.delete(",")
case ell
in /lep student 1st year|LEP student not 1st year|EL Student First Year|LEP\s*student|true/i
in /lep\s*student\s*1st\s*year|LEP\s*student\s*not\s*1st\s*year|EL\s*Student\s*First\s*Year|LEP\s*student|^EL|true/i
"ELL"
in /^NA$|^#NA$/i
"Unknown"
else
in /^NA$|^#NA$|^NA|0|Does\s*not\s*apply|Unknown/i
"Not ELL"
else
puts "************************************"
puts "******** ERROR **********"
puts ""
puts "Error parsing English Language Learner column. '#{ell}' is not a known value. Halting execution"
puts ""
puts "************************************"
exit
end
end
end

16
app/models/income.rb
Unescape View File

@ -7,13 +7,23 @@ class Income < ApplicationRecord
friendly_id :designation, use: [:slugged]
def self.to_designation(income)
return "Unknown" if income.blank? or income.nil?
case income
in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income|Reduced\s*price\s*lunch|true/i
in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income|Reduced\s*price\s*lunch|true|1/i
"Economically Disadvantaged - Y"
in /Not\s*Eligible|false/i
in /Not\s*Eligible|false|0/i
"Economically Disadvantaged - N"
else
in /Unknown/i
"Unknown"
else
puts "************************************"
puts "******** ERROR **********"
puts ""
puts "Error parsing Income column. '#{income}' is not a known value. Halting execution"
puts ""
puts "************************************"
exit
end
end

16
app/models/sped.rb
Unescape View File

@ -5,13 +5,23 @@ class Sped < ApplicationRecord
friendly_id :designation, use: [:slugged]
def self.to_designation(sped)
return "Not Special Education" if sped.blank? || sped.nil?
case sped
in /active/i
in /active|^A$/i
"Special Education"
in /^NA$|^#NA$/i
in /^I$|exited/i
"Not Special Education"
in /^NA$|^#NA$|Unknown/i
"Unknown"
else
"Not Special Education"
puts "************************************"
puts "******** ERROR **********"
puts ""
puts "Error parsing Special Education column. '#{sped}' is not a known value. Halting execution"
puts ""
puts "************************************"
exit
end
end
end

4
spec/fixtures/raw/sample_file_with_duplicate_headers.csv vendored
Unescape View File

@ -1,2 +1,2 @@
StartDate,StartDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,District,School,LASID,Gender,Race,What grade are you in?,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,s-tint-q3,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,s-cure-q1,s-cure-q2,s-cure-q3,s-cure-q4,s-sten-q1,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,s-grmi-q2,s-grmi-q3,s-grmi-q4,s-appa-q1,s-appa-q2,s-appa-q3,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-sust-q1,s-sust-q2,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,s-expa-q1,s-poaf-q1,s-poaf-q2,s-poaf-q3,s-poaf-q4,s-tint-q1-1,s-tint-q2-1,s-tint-q3-1,s-tint-q4-1,s-tint-q5-1,s-acpr-q1-1,s-acpr-q2-1,s-acpr-q3-1,s-acpr-q4-1,s-peff-q1-1,s-peff-q2-1,s-peff-q3-1,s-peff-q4-1,s-peff-q5-1,s-peff-q6-1
2023-03-17 7:57:47,2023-03-17 8:09:15,0,71.174.81.214,100,1000,1,2023-03-17T8:9:15,1000,2,1740505,1,2,4,9,3,5,5,,,,,,,,,,,,,,,,,,,,,,,,,4,4,3,5,,,,,,,,,4,4,2,3,2,5,5,5,5,4,2,2,4,3,2,3,3,5,4,4,3,5,2,3,3,4,4,4,1,2,5,5,,,,,4,4,4,3,4,5
Raw Income,StartDate,StartDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,District,School,LASID,Gender,Race,What grade are you in?,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,s-tint-q3,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,s-cure-q1,s-cure-q2,s-cure-q3,s-cure-q4,s-sten-q1,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,s-grmi-q2,s-grmi-q3,s-grmi-q4,s-appa-q1,s-appa-q2,s-appa-q3,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,s-vale-q2,s-vale-q3,s-vale-q4,s-acst-q1,s-acst-q2,s-acst-q3,s-sust-q1,s-sust-q2,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,s-expa-q1,s-poaf-q1,s-poaf-q2,s-poaf-q3,s-poaf-q4,s-tint-q1-1,s-tint-q2-1,s-tint-q3-1,s-tint-q4-1,s-tint-q5-1,s-acpr-q1-1,s-acpr-q2-1,s-acpr-q3-1,s-acpr-q4-1,s-peff-q1-1,s-peff-q2-1,s-peff-q3-1,s-peff-q4-1,s-peff-q5-1,s-peff-q6-1
Unknown,2023-03-17 7:57:47,2023-03-17 8:09:15,0,71.174.81.214,100,1000,1,2023-03-17T8:9:15,1000,2,1740505,1,2,4,9,3,5,5,,,,,,,,,,,,,,,,,,,,,,,,,4,4,3,5,,,,,,,,,4,4,2,3,2,5,5,5,5,4,2,2,4,3,2,3,3,5,4,4,3,5,2,3,3,4,4,4,1,2,5,5,,,,,4,4,4,3,4,5

1 Raw Income StartDate StartDate Status IPAddress Progress Duration (in seconds) Finished RecordedDate ResponseId District School LASID Gender Race What grade are you in? s-emsa-q1 s-emsa-q2 s-emsa-q3 s-tint-q1 s-tint-q2 s-tint-q3 s-tint-q4 s-tint-q5 s-acpr-q1 s-acpr-q2 s-acpr-q3 s-acpr-q4 s-cure-q1 s-cure-q2 s-cure-q3 s-cure-q4 s-sten-q1 s-sten-q2 s-sten-q3 s-sper-q1 s-sper-q2 s-sper-q3 s-sper-q4 s-civp-q1 s-civp-q2 s-civp-q3 s-civp-q4 s-grmi-q1 s-grmi-q2 s-grmi-q3 s-grmi-q4 s-appa-q1 s-appa-q2 s-appa-q3 s-peff-q1 s-peff-q2 s-peff-q3 s-peff-q4 s-peff-q5 s-sbel-q1 s-sbel-q2 s-sbel-q3 s-sbel-q4 s-sbel-q5 s-phys-q1 s-phys-q2 s-phys-q3 s-phys-q4 s-vale-q1 s-vale-q2 s-vale-q3 s-vale-q4 s-acst-q1 s-acst-q2 s-acst-q3 s-sust-q1 s-sust-q2 s-grit-q1 s-grit-q2 s-grit-q3 s-grit-q4 s-expa-q1 s-poaf-q1 s-poaf-q2 s-poaf-q3 s-poaf-q4 s-tint-q1-1 s-tint-q2-1 s-tint-q3-1 s-tint-q4-1 s-tint-q5-1 s-acpr-q1-1 s-acpr-q2-1 s-acpr-q3-1 s-acpr-q4-1 s-peff-q1-1 s-peff-q2-1 s-peff-q3-1 s-peff-q4-1 s-peff-q5-1 s-peff-q6-1
2 Unknown 2023-03-17 7:57:47 2023-03-17 8:09:15 0 71.174.81.214 100 1000 1 2023-03-17T8:9:15 1000 2 1740505 1 2 4 9 3 5 5 4 4 3 5 4 4 2 3 2 5 5 5 5 4 2 2 4 3 2 3 3 5 4 4 3 5 2 3 3 4 4 4 1 2 5 5 4 4 4 3 4 5

95
spec/services/survey_item_values_spec.rb
Unescape View File

@ -505,13 +505,34 @@ RSpec.describe SurveyItemValues, type: :model do
expect(values.income).to eq "Economically Disadvantaged - N"
end
it "translates blanks to Unknown" do
it "translates ones to Economically Disadvantaged - Y" do
headers = ["LowIncome"]
row = { "LowIncome" => "" }
row = { "LowIncome" => "1" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
expect(values.income).to eq "Economically Disadvantaged - Y"
end
it "translates zeros to Economically Disadvantaged - N" do
headers = ["LowIncome"]
row = { "LowIncome" => "0" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
expect(values.income).to eq "Economically Disadvantaged - N"
end
it "translates blanks to Unknown" do
headers = ["LowIncome"]
row = { "LowIncome" => "" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.income).to eq "Unknown"
end
# NOTE: This will halt test runs too
# it "halts execution if there is a value it cannot parse" do
# headers = ["LowIncome"]
# row = { "LowIncome" => "ArbitraryUnknownValue" }
# output = capture_stdout { SurveyItemValues.new(row:, headers:, survey_items:, schools:) }
# expect(output).to match "ArbitraryUnknownValue is not a known value. Halting execution"
# end
end
context ".ell" do
@ -530,8 +551,20 @@ RSpec.describe SurveyItemValues, type: :model do
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
expect(values.ell).to eq "ELL"
row = { "Raw ELL" => "LEP Student Not 1st Year" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
row = { "Raw ELL" => "LEP Student, Not 1st Year" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.ell).to eq "ELL"
row = { "Raw ELL" => "EL student, not 1st year" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.ell).to eq "ELL"
row = { "Raw ELL" => "EL student, 1st year" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.ell).to eq "ELL"
row = { "Raw ELL" => "EL - Early Child. or PK" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.ell).to eq "ELL"
end
@ -546,16 +579,40 @@ RSpec.describe SurveyItemValues, type: :model do
expect(values.ell).to eq "Not ELL"
end
it 'tranlsates blanks into "Not Ell"' do
it 'tranlsates zeros into "Not ELL"' do
headers = ["Raw ELL"]
row = { "Raw ELL" => "0" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.ell).to eq "Not ELL"
end
it 'tranlsates NAs and blanks into "Not ELL"' do
headers = ["Raw ELL"]
row = { "Raw ELL" => "" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
expect(values.ell).to eq "Not ELL"
row = { "Raw ELL" => "Anything else" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
row = { "Raw ELL" => "NA" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.ell).to eq "Not ELL"
row = { "Raw ELL" => "#NA" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.ell).to eq "Not ELL"
row = { "Raw ELL" => "#N/A" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.ell).to eq "Not ELL"
end
# NOTE: This will halt test runs too
# it "halts execution if there is a value it cannot parse" do
# headers = ["Raw ELL"]
# row = { "Raw ELL" => "ArbitraryUnknownValue" }
# output = capture_stdout { SurveyItemValues.new(row:, headers:, survey_items:, schools:) }
# expect(output).to match "ArbitraryUnknownValue is not a known value. Halting execution"
# end
end
context ".sped" do
@ -571,6 +628,20 @@ RSpec.describe SurveyItemValues, type: :model do
expect(values.sped).to eq "Special Education"
end
it 'translates "A" into "Special Education"' do
headers = ["Raw SpEd"]
row = { "Raw SpEd" => "A" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
expect(values.sped).to eq "Special Education"
end
it 'translates "I" into "Not Special Education"' do
headers = ["Raw SpEd"]
row = { "Raw SpEd" => "I" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
expect(values.sped).to eq "Not Special Education"
end
it 'translates "exited" into "Not Special Education"' do
headers = ["Raw SpEd"]
row = { "Raw SpEd" => "exited" }
@ -585,7 +656,7 @@ RSpec.describe SurveyItemValues, type: :model do
expect(values.sped).to eq "Not Special Education"
end
it 'tranlsates NA into "Not Special Education"' do
it 'translates NA into "Not Special Education"' do
headers = ["Raw SpEd"]
row = { "Raw SpEd" => "NA" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
@ -595,6 +666,14 @@ RSpec.describe SurveyItemValues, type: :model do
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:, academic_years:)
expect(values.sped).to eq "Not Special Education"
end
# NOTE: This will halt test runs too
# it "halts execution if there is a value it cannot parse" do
# headers = ["Raw SpEd"]
# row = { "Raw SpEd" => "ArbitraryUnknownValue" }
# output = capture_stdout { SurveyItemValues.new(row:, headers:, survey_items:, schools:) }
# expect(output).to match "ArbitraryUnknownValue is not a known value. Halting execution"
# end
end
context ".valid_duration" do

Write Preview
Loading…
Cancel
Save