edcommonwealth
/
sqm-dashboards
mirror of https://github.com/edcommonwealth/sqm-dashboards.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update parsing rules from glossary

Browse Source
mciea-main
Nelson Jovel 2 years ago
parent d4db4e0fa2
commit acf25cb525
6 changed files with 9 additions and 12 deletions
Show Stats Download Patch File Download Diff File

2
app/models/income.rb
Unescape View File

@ -7,7 +7,7 @@ class Income < ApplicationRecord
friendly_id :designation, use: [:slugged] friendly_id :designation, use: [:slugged]
def self.to_designation(income) def self.to_designation(income)
return "Unknown" if income.blank? or income.nil? return "Economically Disadvantaged - N" if income.blank? or income.nil?
case income case income
in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income|Reduced\s*price\s*lunch|true|1/i in /Free\s*Lunch|Reduced\s*Lunch|Low\s*Income|Reduced\s*price\s*lunch|true|1/i

2
app/models/sped.rb
Unescape View File

@ -10,7 +10,7 @@ class Sped < ApplicationRecord
case sped case sped
in /active|^A$|1|^Special\s*Education$/i in /active|^A$|1|^Special\s*Education$/i
"Special Education" "Special Education"
in /^I$|exited|0|^Not\s*Special\s*Education$|Does\s*not\s*apply/i in /^I$|exited|0|^Not\s*Special\s*Education$|Does\s*not\s*apply|Referred|Ineligible/i
"Not Special Education" "Not Special Education"
in %r{^#*N/*A$|Unknown}i in %r{^#*N/*A$|Unknown}i
"Unknown" "Unknown"

7
spec/fixtures/demographic_glossary.csv vendored
Unescape View File

@ -3,14 +3,13 @@ EL student not 1st year,ELL,EL Student First Year,A,Special Education,Special Ed
"EL student, not 1st year",ELL,Raw ELL,active,Special Education,Raw SPED,Reduced Lunch,Economically Disadvantaged – Y,Raw Income "EL student, not 1st year",ELL,Raw ELL,active,Special Education,Raw SPED,Reduced Lunch,Economically Disadvantaged – Y,Raw Income
EL student 1st year,ELL,ELL- SIS,1,Special Education,SPED- SIS,LowIncome,Economically Disadvantaged – Y,SES- SIS EL student 1st year,ELL,ELL- SIS,1,Special Education,SPED- SIS,LowIncome,Economically Disadvantaged – Y,SES- SIS
"EL student, 1st year",ELL,DirectCert,Special Education,Special Education,SPED,Low Income,Economically Disadvantaged – Y,EconDisadvantaged "EL student, 1st year",ELL,DirectCert,Special Education,Special Education,SPED,Low Income,Economically Disadvantaged – Y,EconDisadvantaged
EL - Early Child. or PK,ELL,,Referred,Special Education,,Reduced price lunch,Economically Disadvantaged – Y,Income SIS EL - Early Child. or PK,ELL,ELL,Referred,Not Special Education,,Reduced price lunch,Economically Disadvantaged – Y,Income SIS
1,ELL,,Ineligible,Not Special Education,,TRUE,Economically Disadvantaged – Y, 1,ELL,,Ineligible,Not Special Education,,TRUE,Economically Disadvantaged – Y,SES
lep student 1st year,ELL,,I,Not Special Education,,1,Economically Disadvantaged – Y, lep student 1st year,ELL,,I,Not Special Education,,1,Economically Disadvantaged – Y,
lep student not 1st year,ELL,,exited,Not Special Education,,Not Eligible,Economically Disadvantaged – N, lep student not 1st year,ELL,,exited,Not Special Education,,Not Eligible,Economically Disadvantaged – N,
LEP Not1stYr,ELL,,0,Not Special Education,,FALSE,Economically Disadvantaged – N, LEP Not1stYr,ELL,,0,Not Special Education,,FALSE,Economically Disadvantaged – N,
LEP1stYr US Sch,ELL,,Not Special Education,Not Special Education,,0,Economically Disadvantaged – N, LEP1stYr US Sch,ELL,,Not Special Education,Not Special Education,,0,Economically Disadvantaged – N,
"Does not apply Does not apply,Not ELL,,Does not apply,Not Special Education,,[blanks],Economically Disadvantaged – N,
",Not ELL,,Does not apply,Not Special Education,,[blanks],Unknown,
0,Not ELL,,[blanks],Not Special Education,,#NA,Unknown, 0,Not ELL,,[blanks],Not Special Education,,#NA,Unknown,
2,Not ELL,,#NA,Unknown,,NA,Unknown, 2,Not ELL,,#NA,Unknown,,NA,Unknown,
3,Not ELL,,NA,Unknown,,N/A,Unknown, 3,Not ELL,,NA,Unknown,,N/A,Unknown,

1 ELL Value ELL Type ELL Headers Sped Value Sped Type Sped Headers Income Value Income Type Income Headers
3 EL student, not 1st year ELL Raw ELL active Special Education Raw SPED Reduced Lunch Economically Disadvantaged – Y Raw Income
4 EL student 1st year ELL ELL- SIS 1 Special Education SPED- SIS LowIncome Economically Disadvantaged – Y SES- SIS
5 EL student, 1st year ELL DirectCert Special Education Special Education SPED Low Income Economically Disadvantaged – Y EconDisadvantaged
6 EL - Early Child. or PK ELL ELL Referred Special Education Not Special Education Reduced price lunch Economically Disadvantaged – Y Income SIS
7 1 ELL Ineligible Not Special Education TRUE Economically Disadvantaged – Y SES
8 lep student 1st year ELL I Not Special Education 1 Economically Disadvantaged – Y
9 lep student not 1st year ELL exited Not Special Education Not Eligible Economically Disadvantaged – N
10 LEP Not1stYr ELL 0 Not Special Education FALSE Economically Disadvantaged – N
11 LEP1stYr US Sch ELL Not Special Education Not Special Education 0 Economically Disadvantaged – N
12 Does not apply Does not apply Not ELL Does not apply Not Special Education [blanks] Unknown Economically Disadvantaged – N
0 Not ELL [blanks] Not Special Education #NA Unknown
13 2 0 Not ELL #NA [blanks] Unknown Not Special Education NA #NA Unknown
14 3 2 Not ELL NA #NA Unknown N/A NA Unknown
15 [blanks] 3 Not ELL N/A NA Unknown #N/A N/A Unknown

2
spec/fixtures/test_2020-21_student_survey_responses.csv vendored
Unescape View File

@ -2,7 +2,7 @@ Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Fini
2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,1500025,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888,11th,1,Free Lunch,Economically Disadvantaged – Y,Does not apply,Not ELL,exited 2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,123456,,,,,,,anonymous,EN,1,8,1500025,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,,,1,888,11th,1,Free Lunch,Economically Disadvantaged – Y,Does not apply,Not ELL,exited
2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,,888,10,,Not Eligible,Economically Disadvantaged – N,,Not ELL,Unknown 2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,234567,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,,,,888,10,,Not Eligible,Economically Disadvantaged – N,,Not ELL,Unknown
2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1500505,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,"2,3,4",888,8,2,Reduced Lunch,Economically Disadvantaged – Y,,Not ELL,Unknown 2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,345678,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,1500505,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,,,"2,3,4",888,8,2,Reduced Lunch,Economically Disadvantaged – Y,,Not ELL,Unknown
2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1500505,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888,8,3,,Unknown,LEP student not 1st year,ELL,active 2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,456789,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,1500505,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher,7,888,8,3,,Economically Disadvantaged – N,LEP student not 1st year,ELL,active
2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888,7,4,Free Lunch,Economically Disadvantaged – Y,EL Student First Year,ELL,active 2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_5,567890,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8,6,7",888,7,4,Free Lunch,Economically Disadvantaged – Y,EL Student First Year,ELL,active
2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888,3,NA,Not Eligible,Economically Disadvantaged – N,Unknown,Unknown,NA 2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_6,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,"1,2,3,4,5,8",888,3,NA,Not Eligible,Economically Disadvantaged – N,Unknown,Unknown,NA
2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,5,,4,7,Reduced Lunch,Economically Disadvantaged – Y,#N/A,Unknown, 2021-03-31 9:51:39,2021-03-31 10:01:36,0,73.47.153.77,100,596,1,2021-03-31T10:01:36,student_survey_response_7,,,,,,42.65820313,-71.30580139,anonymous,EN,3,2,1500505,6,15,109,3710,7,1,,2,2,2,,,,,,,,,,3,3,4,3,3,3,3,4,3,4,3,4,4,5,4,3,4,3,5,2,2,3,,,,,,,,,,,,1,,2,5,1,3,3,2,4,3,5,4,,,,,,,,,,,,5,4,3,4,4,4,4,4,4,,,,,,,2,,2,,EN,,,Social Studies teacher,,5,,4,7,Reduced Lunch,Economically Disadvantaged – Y,#N/A,Unknown,

1 Start Date End Date Response Type IP Address Progress Duration (in seconds) Finished RecordedDate ResponseId LASID Recipient Last Name Recipient First Name Recipient Email External Data Reference Location Latitude Location Longitude Distribution Channel User Language district school DESE ID #N/A #N/A #N/A #N/A #N/A #N/A #N/A s-emsa-q1 s-emsa-q2 s-emsa-q3 s-tint-q1 s-tint-q2 #N/A s-tint-q4 s-tint-q5 s-acpr-q1 s-acpr-q2 s-acpr-q3 s-acpr-q4 #N/A #N/A s-cure-q3 s-cure-q4 #N/A s-sten-q2 s-sten-q3 s-sper-q1 s-sper-q2 s-sper-q3 s-sper-q4 s-civp-q1 s-civp-q2 s-civp-q3 s-civp-q4 s-grmi-q1 #N/A #N/A s-grmi-q4 s-appa-q1 s-appa-q2 #N/A s-peff-q1 s-peff-q2 s-peff-q3 s-peff-q4 s-peff-q5 s-peff-q6 s-sbel-q1 s-sbel-q2 s-sbel-q3 s-sbel-q4 s-sbel-q5 s-phys-q1 s-phys-q1-1 s-phys-q2 s-phys-q3 s-phys-q4 s-vale-q1 s-vale-q2 s-vale-q3 s-vale-q4 s-acst-q1 s-acst-q2 s-acst-q3 s-acst-q4 s-acst-q5 s-grit-q1 s-grit-q2 s-grit-q3 s-grit-q4 #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A #N/A race What is your race/ethnicity?(Please select all that apply) - Selected Choice grade gender Raw Income Income Raw ELL ELL Raw SpEd
2 2020-09-29 18:28:41 2020-09-29 18:48:28 0 73.249.89.226 6 1186 0 2020-09-30T18:48:50 student_survey_response_1 123456 anonymous EN 1 8 1500025 dddd 4 3 3 0 some non-integer response 6 5 1 EN 1 888 11th 1 Free Lunch Economically Disadvantaged – Y Does not apply Not ELL exited
3 2021-02-23 15:12:58 2021-02-23 15:13:17 0 50.207.254.114 0 19 0 2021-02-24T15:13:19 student_survey_response_2 234567 anonymous EN NA EN 888 10 Not Eligible Economically Disadvantaged – N Not ELL Unknown
4 2021-03-31 9:50:19 2021-03-31 9:59:01 0 108.7.17.250 100 522 1 2021-03-31T09:59:02 student_survey_response_3 345678 42.53340149 -70.96530151 anonymous EN 3 2 1500505 12 4 108 3300 7 1 2 4 2 1 4 3 3 3 3 3 3 NA 3 2 3 3 2 1 3 3 4 1 3 3 4 4 2 4 3 3 4 3 3 3 4 3 3 3 3 3 3 4 4 2 3 3 1 3 EN Math teacher 2,3,4 888 8 2 Reduced Lunch Economically Disadvantaged – Y Not ELL Unknown
5 2021-03-31 9:50:09 2021-03-31 10:00:16 0 67.186.188.168 100 607 1 2021-03-31T10:00:17 student_survey_response_4 456789 42.63510132 -71.30139923 anonymous EN 3 2 1500505 12 18 108 2064 7 1 2 2 1 3 5 3 3 4 4 3 4 5 1 1 5 1 3 2 4 4 1 2 1 3 2 3 3 3 4 2 5 3 4 5 5 3 3 4 3 4 4 4 4 3 5 2 2 EN English teacher 7 888 8 3 Unknown Economically Disadvantaged – N LEP student not 1st year ELL active
6 2021-03-31 9:51:39 2021-03-31 10:01:36 0 73.47.153.77 100 596 1 2021-03-31T10:01:36 student_survey_response_5 567890 42.65820313 -71.30580139 anonymous EN 3 2 1500505 6 15 109 3710 7 1 2 2 2 3 3 4 3 3 3 3 4 3 4 3 4 4 5 4 3 4 3 5 2 2 3 1 2 5 1 3 3 2 4 3 5 4 5 4 3 4 4 4 4 4 4 2 2 EN Social Studies teacher 1,2,3,4,5,8,6,7 888 7 4 Free Lunch Economically Disadvantaged – Y EL Student First Year ELL active
7 2021-03-31 9:51:39 2021-03-31 10:01:36 0 73.47.153.77 100 596 1 2021-03-31T10:01:36 student_survey_response_6 42.65820313 -71.30580139 anonymous EN 3 2 1500505 6 15 109 3710 7 1 2 2 2 3 3 4 3 3 3 3 4 3 4 3 4 4 5 4 3 4 3 5 2 2 3 1 2 5 1 3 3 2 4 3 5 4 5 4 3 4 4 4 4 4 4 2 2 EN Social Studies teacher 1,2,3,4,5,8 888 3 NA Not Eligible Economically Disadvantaged – N Unknown Unknown NA
8 2021-03-31 9:51:39 2021-03-31 10:01:36 0 73.47.153.77 100 596 1 2021-03-31T10:01:36 student_survey_response_7 42.65820313 -71.30580139 anonymous EN 3 2 1500505 6 15 109 3710 7 1 2 2 2 3 3 4 3 3 3 3 4 3 4 3 4 4 5 4 3 4 3 5 2 2 3 1 2 5 1 3 3 2 4 3 5 4 5 4 3 4 4 4 4 4 4 2 2 EN Social Studies teacher 5 4 7 Reduced Lunch Economically Disadvantaged – Y #N/A Unknown

4
spec/services/survey_item_values_spec.rb
Unescape View File

@ -523,11 +523,11 @@ RSpec.describe SurveyItemValues, type: :model do
expect(values.income).to eq "Economically Disadvantaged - N" expect(values.income).to eq "Economically Disadvantaged - N"
end end
it "translates blanks to Unknown" do it "translates blanks to 'Economically Disadvantaged - N'" do
headers = ["LowIncome"] headers = ["LowIncome"]
row = { "LowIncome" => "" } row = { "LowIncome" => "" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:) values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.income).to eq "Unknown" expect(values.income).to eq "Economically Disadvantaged - N"
end end
# NOTE: This will halt test runs too # NOTE: This will halt test runs too

4
spec/services/survey_responses_data_loader_spec.rb
Unescape View File

@ -328,7 +328,7 @@ end
def assigns_income_to_responses def assigns_income_to_responses
results = { "student_survey_response_1" => low_income, results = { "student_survey_response_1" => low_income,
"student_survey_response_3" => low_income, "student_survey_response_3" => low_income,
"student_survey_response_4" => unknown_income, "student_survey_response_4" => high_income,
"student_survey_response_5" => low_income, "student_survey_response_5" => low_income,
"student_survey_response_6" => high_income, "student_survey_response_6" => high_income,
"student_survey_response_7" => low_income } "student_survey_response_7" => low_income }
@ -349,8 +349,6 @@ def assigns_ell_to_responses
results.each do |key, value| results.each do |key, value|
ell = SurveyItemResponse.find_by_response_id(key).ell ell = SurveyItemResponse.find_by_response_id(key).ell
puts key
puts value
expect(ell).to eq value expect(ell).to eq value
end end
end end

Write Preview
Loading…
Cancel
Save