edcommonwealth
/
sqm-dashboards
mirror of https://github.com/edcommonwealth/sqm-dashboards.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add scrapers for admin data and their corresponding csv files of data.

Browse Source 
Make sure specs are off so they don't run in github CI
pull/1/head
rebuilt 3 years ago
parent 94056a80c6
commit 1be6e79fd8
21 changed files with 67804 additions and 39 deletions
Show Stats Download Patch File Download Diff File

46
app/services/dese/five_c_one.rb
Unescape View File

@ -0,0 +1,46 @@
require 'watir'
require 'csv'
module Dese
class FiveCOne
include Dese::Scraper
attr_reader :filepaths
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '5C_1_art_course.csv')])
@filepaths = filepaths
end
def run_all
filepath = filepaths[0]
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'K', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
'11', '12', 'All Grades', 'Total Students']
write_headers(filepath:, headers:)
run_a_picp_i1(filepath:)
browser.close
end
def run_a_picp_i1(filepath:)
run do |academic_year|
admin_data_item_id = 'a-picp-i1'
url = 'https://profiles.doe.mass.edu/statereport/artcourse.aspx'
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
'ctl00_ContentPlaceHolder1_ddYear' => range,
'ctl00_ContentPlaceHolder1_ddView' => 'Percent' }
submit_id = 'btnViewReport'
calculation = lambda { |headers, items|
percent_graduated_index = headers['All Grades'] - 1
return 'NA' if items[percent_graduated_index] == '' || items[percent_graduated_index].strip == '.0'
percent_passing = items[percent_graduated_index].to_f
benchmark = 77.5
percent_passing * 4 / benchmark if percent_graduated_index.present?
}
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
end
end
end
end

56
app/services/dese/five_d_two.rb
Unescape View File

@ -0,0 +1,56 @@
require 'watir'
require 'csv'
module Dese
class FiveDTwo
include Dese::Scraper
include Dese::Enrollments
attr_reader :filepaths
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '5D_2_enrollments.csv'),
Rails.root.join('data', 'admin_data', 'dese', '5D_2_age_staffing.csv')])
@filepaths = filepaths
end
def run_all
filepath = filepaths[0]
scrape_enrollments(filepath:)
filepath = filepaths[1]
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'<26 yrs (# )', '26-32 yrs (#)', '33-40 yrs (#)', '41-48 yrs (#)', '49-56 yrs (#)', '57-64 yrs (#)', 'Over 64 yrs (#)', 'FTE Count']
write_headers(filepath:, headers:)
run_a_phya_i1(filepath:)
browser.close
end
def run_a_phya_i1(filepath:)
run do |academic_year|
admin_data_item_id = 'a-phya-i1'
url = 'https://profiles.doe.mass.edu/statereport/agestaffing.aspx'
range = academic_year.range
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
'ctl00_ContentPlaceHolder1_ddYear' => range,
'ctl00_ContentPlaceHolder1_ddJobClassification' => 'School Nurse -- Non-Special Education' }
submit_id = 'btnViewReport'
calculation = lambda { |headers, items|
nurse_index = headers['FTE Count']
return 'NA' if items[nurse_index] == '' || items[nurse_index].strip == '.0'
nurse_count = items[nurse_index].to_f
benchmark = 750
nurse_count * 4 / benchmark if nurse_index.present?
dese_id = items[headers['School Code']].to_i
num_of_students = student_count(filepath: filepaths[0], dese_id:, year: academic_year.range) || 0
items << num_of_students
items << (num_of_students / nurse_count)
((benchmark - (num_of_students / nurse_count)) + benchmark) * 4 / benchmark
}
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
end
end
end
end

44
app/services/dese/four_a_one.rb
Unescape View File

@ -0,0 +1,44 @@
require 'watir'
require 'csv'
module Dese
class FourAOne
include Dese::Scraper
attr_reader :filepaths
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '4A_1_grade_nine_course_pass.csv')])
@filepaths = filepaths
end
def run_all
filepath = filepaths[0]
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'# Grade Nine Students', '# Passing All Courses', '% Passing All Courses']
write_headers(filepath:, headers:)
run_a_ovpe_i1(filepath:)
browser.close
end
def run_a_ovpe_i1(filepath:)
run do |academic_year|
admin_data_item_id = 'a-ovpe-i1'
url = 'https://profiles.doe.mass.edu/statereport/gradeninecoursepass.aspx'
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
'ctl00_ContentPlaceHolder1_ddYear' => range }
submit_id = 'btnViewReport'
calculation = lambda { |headers, items|
passing_index = headers['% Passing All Courses']
return 'NA' if items[passing_index] == '' || items[passing_index].strip == '.0'
percent_passing = items[passing_index].to_f
benchmark = 95
percent_passing * 4 / benchmark if passing_index.present?
}
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
end
end
end
end

105
app/services/dese/four_b_two.rb
Unescape View File

@ -0,0 +1,105 @@
require 'watir'
require 'csv'
module Dese
class FourBTwo
include Dese::Scraper
attr_reader :filepaths
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '4B_2_four_year_grad.csv'),
Rails.root.join('data', 'admin_data', 'dese', '4B_2_retention.csv'),
Rails.root.join('data', 'admin_data', 'dese', '4B_2_five_year_grad.csv')])
@filepaths = filepaths
end
def run_all
filepath = filepaths[0]
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'# in Cohort', '% Graduated', '% Still in School', '% Non-Grad Completers', '% H.S. Equiv.',
'% Dropped Out', '% Permanently Excluded']
write_headers(filepath:, headers:)
run_a_degr_i1(filepath:)
filepath = filepaths[1]
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'# Enrolled', '# Retained', '% Retained', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
'11', '12']
write_headers(filepath:, headers:)
run_a_degr_i2(filepath:)
filepath = filepaths[2]
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'# in Cohort', '% Graduated', '% Still in School', '% Non-Grad Completers', '% H.S. Equiv.',
'% Dropped Out', '% Permanently Excluded']
write_headers(filepath:, headers:)
run_a_degr_i3(filepath:)
browser.close
end
def run_a_degr_i1(filepath:)
run do |academic_year|
admin_data_item_id = 'a-degr-i1'
url = 'https://profiles.doe.mass.edu/statereport/gradrates.aspx'
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
'ctl00_ContentPlaceHolder1_ddYear' => range,
'ctl00_ContentPlaceHolder1_ddRateType' => '4yr Grad' }
submit_id = 'btnViewReport'
calculation = lambda { |headers, items|
percent_graduated_index = headers['% Graduated']
return 'NA' if items[percent_graduated_index] == '' || items[percent_graduated_index].strip == '.0'
percent_passing = items[percent_graduated_index].to_f
benchmark = 80
percent_passing * 4 / benchmark if percent_graduated_index.present?
}
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
end
end
def run_a_degr_i2(filepath:)
run do |academic_year|
admin_data_item_id = 'a-degr-i2'
url = 'https://profiles.doe.mass.edu/statereport/retention.aspx'
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
'ctl00_ContentPlaceHolder1_ddYear' => range,
'ctl00_ContentPlaceHolder1_ddView' => 'Percent' }
submit_id = 'btnViewReport'
calculation = lambda { |headers, items|
retained_index = headers['% Retained']
return 'NA' if items[retained_index] == '' || items[retained_index].strip == '.0'
percent_retained = items[retained_index].to_f
benchmark = 2
((benchmark - percent_retained) + benchmark) * 4 / benchmark if retained_index.present?
}
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
end
end
def run_a_degr_i3(filepath:)
run do |academic_year|
admin_data_item_id = 'a-degr-i3'
url = 'https://profiles.doe.mass.edu/statereport/gradrates.aspx'
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
'ctl00_ContentPlaceHolder1_ddYear' => range,
'ctl00_ContentPlaceHolder1_ddRateType' => '5yr Grad' }
submit_id = 'btnViewReport'
calculation = lambda { |headers, items|
percent_graduated_index = headers['% Graduated']
return 'NA' if items[percent_graduated_index] == '' || items[percent_graduated_index].strip == '.0'
percent_passing = items[percent_graduated_index].to_f
benchmark = 85
percent_passing * 4 / benchmark if percent_graduated_index.present?
}
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
end
end
end
end

1
app/services/dese/scraper.rb
Unescape View File

@ -71,6 +71,7 @@ module Dese
likert_score = likert_score.round(2)
end
# byebug if dese_id == 30_305
output = []
output << raw_likert_score
output << likert_score

61
app/services/dese/three_b_two.rb
Unescape View File

@ -7,26 +7,26 @@ module Dese
include Dese::Enrollments
attr_reader :filepaths
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', 'enrollments.csv'),
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '3B_2_enrollments.csv'),
Rails.root.join('data', 'admin_data', 'dese', '3B_2_teacher_by_race_and_gender.csv'),
Rails.root.join('data', 'admin_data', 'dese', '3B_2_student_by_race_and_gender.csv')])
@filepaths = filepaths
end
def run_all
filepath = filepaths[0]
scrape_enrollments(filepath:)
# filepath = filepaths[0]
# scrape_enrollments(filepath:)
filepath = filepaths[1]
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Teachers of color (#)', 'School Name', 'DESE ID',
'African American (#)', 'Asian (#)', 'Hispanic (#)', 'White (#)', 'Native American (#)',
'Native Hawaiian Pacific Islander (#)', 'Multi-Race Non-Hispanic (#)', 'Females (#)',
'Males (#)', 'FTE Count']
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Teachers of color (%)', 'School Name', 'DESE ID',
'African American (%)', 'Asian (%)', 'Hispanic (%)', 'White (%)', 'Native American (%)',
'Native Hawaiian Pacific Islander (%)', 'Multi-Race Non-Hispanic (%)', 'Females (%)',
'Males (%)', 'FTE Count']
write_headers(filepath:, headers:)
run_teacher_demographics(filepath:)
filepath = filepaths[2]
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Non-White Teachers', 'Non-White Students', 'School Name', 'DESE ID',
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Non-White Teachers %', 'Non-White Students %', 'School Name', 'DESE ID',
'African American', 'Asian', 'Hispanic', 'White', 'Native American',
'Native Hawaiian or Pacific Islander', 'Multi-Race or Non-Hispanic', 'Males',
'Females', 'Non-Binary', 'Students of color (%)']
@ -42,31 +42,32 @@ module Dese
url = 'https://profiles.doe.mass.edu/statereport/teacherbyracegender.aspx'
range = academic_year.range
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
'ctl00_ContentPlaceHolder1_ddYear' => range }
'ctl00_ContentPlaceHolder1_ddYear' => range,
'ctl00_ContentPlaceHolder1_ddDisplay' => 'Percentages' }
submit_id = 'ctl00_ContentPlaceHolder1_btnViewReport'
calculation = lambda { |headers, items|
african_american_index = headers['African American (#)']
african_american_index = headers['African American (%)']
african_american_number = items[african_american_index].to_f
asian_index = headers['Asian (#)']
asian_index = headers['Asian (%)']
asian_number = items[asian_index].to_f
hispanic_index = headers['Hispanic (#)']
hispanic_index = headers['Hispanic (%)']
hispanic_number = items[hispanic_index].to_f
native_american_index = headers['Native American (#)']
native_american_index = headers['Native American (%)']
native_american_number = items[native_american_index].to_f
native_hawaiian_index = headers['Native Hawaiian, Pacific Islander (#)']
native_hawaiian_index = headers['Native Hawaiian, Pacific Islander (%)']
native_hawaiian_number = items[native_hawaiian_index].to_f
multi_race_index = headers['Multi-Race,Non-Hispanic (#)']
multi_race_index = headers['Multi-Race,Non-Hispanic (%)']
multi_race_number = items[multi_race_index].to_f
total_non_white_teachers = african_american_number + asian_number + hispanic_number + native_american_number + native_hawaiian_number + multi_race_number
items.unshift(total_non_white_teachers)
non_white_teachers = african_american_number + asian_number + hispanic_number + native_american_number + native_hawaiian_number + multi_race_number
items.unshift(non_white_teachers)
total_non_white_teachers
non_white_teachers
}
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
end
@ -78,7 +79,7 @@ module Dese
CSV.parse(File.read(filepath), headers: true).map do |row|
academic_year = row['Academic Year']
school_id = row['DESE ID'].to_i
total = row['Teachers of color (#)'].gsub(',', '').to_f
total = row['Teachers of color (%)'].gsub(',', '').to_f
@teachers[[school_id, academic_year]] = total
end
end
@ -96,17 +97,19 @@ module Dese
calculation = lambda { |headers, items|
white_index = headers['White']
white_number = items[white_index].to_f
non_white_student_percentage = 100 - white_number
dese_id = items[headers['School Code']].to_i
num_of_students = student_count(filepath: filepaths[0], dese_id:, year: academic_year.range) || 0
num_of_non_white_students = num_of_students * non_white_student_percentage / 100
items.unshift(num_of_non_white_students)
num_of_non_white_teachers = teacher_count(filepath: filepaths[1], dese_id:, year: academic_year.range)
items.unshift(num_of_non_white_teachers)
parity_index = num_of_non_white_teachers.to_f / num_of_non_white_students.to_f
parity_index * 4 / 0.26
non_white_student_percentage = (100 - white_number).to_f
items.unshift(non_white_student_percentage)
non_white_teachers = teacher_count(filepath: filepaths[1], dese_id:, year: academic_year.range).to_f
items.unshift(non_white_teachers)
# if non_white_teachers >= 10
parity_index = non_white_teachers / non_white_student_percentage
likert_score = parity_index * 4 / 0.25
# else
# likert_score = 1
# end
likert_score
}
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
end

11067
data/admin_data/dese/3B_2_student_by_race_and_gender.csv
View File

File diff suppressed because it is too large Load Diff

11067
data/admin_data/dese/3B_2_teacher_by_race_and_gender.csv
View File

File diff suppressed because it is too large Load Diff

1582
data/admin_data/dese/4A_1_grade_nine_course_pass.csv
View File

File diff suppressed because it is too large Load Diff

1544
data/admin_data/dese/4B_2_five_year_grad.csv
View File

File diff suppressed because it is too large Load Diff

1936
data/admin_data/dese/4B_2_four_year_grad.csv
View File

File diff suppressed because it is too large Load Diff

8799
data/admin_data/dese/4B_2_retention.csv
View File

File diff suppressed because it is too large Load Diff

1979
data/admin_data/dese/4D_1_plans_of_grads.csv
View File

File diff suppressed because it is too large Load Diff

7130
data/admin_data/dese/5C_1_art_course.csv
View File

File diff suppressed because it is too large Load Diff

11059
data/admin_data/dese/5D_2_age_staffing.csv
View File

File diff suppressed because it is too large Load Diff

11067
data/admin_data/dese/5D_2_enrollments.csv
View File

File diff suppressed because it is too large Load Diff

55
spec/services/dese/five_c_one_spec.rb
Unescape View File

@ -0,0 +1,55 @@
require 'rails_helper'
RSpec.describe Dese::FiveCOne do
let(:academic_years) do
[
create(:academic_year, range: '2020-21'),
create(:academic_year, range: '2019-20')
# create(:academic_year, range: '2018-19'),
# create(:academic_year, range: '2017-18'),
# create(:academic_year, range: '2016-17')
]
end
let(:i1_filepath) { Rails.root.join('tmp', 'spec', 'dese', '5C_1_art_course.csv') }
let(:filepaths) do
[i1_filepath]
end
before do
FileUtils.mkdir_p 'tmp/spec/dese'
end
before :each do
academic_years
end
xcontext '#run_all' do
it 'creates a csv file with the scraped data' do
Dese::FiveCOne.new(filepaths:).run_all
expect(i1_filepath).to exist
end
it 'has the correct headers for i1' do
headers = File.open(i1_filepath) do |file|
headers = file.first
end.split(',')
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'K', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
'11', '12', 'All Grades', "Total Students\n"]
end
it 'has the right likert score results for a-picp-i1' do
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
next unless row['Admin Data Item'] == 'a-picp-i1' && row['Academic Year'] == '2020-21'
likert_score = row['Likert Score']
likert_score == 'NA' ? likert_score : likert_score.to_f
end.flatten.compact
expect(results.take(20)).to eq [4.95, 2.39, 4.81, 4.89, 4.63, 4.95, 2.25, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.75,
4.82, 1.0, 3.88, 3.14, 4.84, 5.0]
end
end
end

66
spec/services/dese/five_d_two_spec.rb
Unescape View File

@ -0,0 +1,66 @@
require 'rails_helper'
RSpec.describe Dese::FiveDTwo do
let(:academic_years) do
[
create(:academic_year, range: '2021-22'),
create(:academic_year, range: '2020-21')
# create(:academic_year, range: '2019-20')
# create(:academic_year, range: '2018-19'),
# create(:academic_year, range: '2017-18'),
# create(:academic_year, range: '2016-17')
]
end
let(:enrollments_filepath) { Rails.root.join('tmp', 'spec', 'dese', '5D_2_enrollments.csv') }
let(:i1_filepath) { Rails.root.join('tmp', 'spec', 'dese', '5D_2_age_staffing.csv') }
let(:filepaths) do
[enrollments_filepath, i1_filepath]
end
before do
FileUtils.mkdir_p 'tmp/spec/dese'
end
before :each do
academic_years
end
xcontext '#run_all' do
it 'creates a csv file with the scraped data' do
Dese::FiveDTwo.new(filepaths:).run_all
expect(enrollments_filepath).to exist
expect(i1_filepath).to exist
end
it 'has the correct headers for enrollements' do
headers = File.open(enrollments_filepath) do |file|
headers = file.first
end.split(',')
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'PK', 'K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', 'SP', "Total\n"]
end
it 'has the correct headers for i1' do
headers = File.open(i1_filepath) do |file|
headers = file.first
end.split(',')
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'<26 yrs (# )', '26-32 yrs (#)', '33-40 yrs (#)', '41-48 yrs (#)', '49-56 yrs (#)', '57-64 yrs (#)', 'Over 64 yrs (#)', "FTE Count\n"]
end
it 'has the right likert score results for a-phya-i1' do
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
next unless row['Admin Data Item'] == 'a-phya-i1' && row['Academic Year'] == '2020-21'
likert_score = row['Likert Score']
likert_score == 'NA' ? likert_score : likert_score.to_f
end.flatten.compact
expect(results.take(20)).to eq [5.0, 1.0, 4.7, 4.59, 5.0, 5.0, 1.0, 3.33, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0,
5.0, 5.0, 4.78, 5.0]
end
end
end

56
spec/services/dese/four_a_one_spec.rb
Unescape View File

@ -0,0 +1,56 @@
require 'rails_helper'
require 'fileutils'
require 'csv'
RSpec.describe Dese::FourAOne do
let(:academic_years) do
[
create(:academic_year, range: '2021-22'),
create(:academic_year, range: '2020-21')
# create(:academic_year, range: '2019-20'),
# create(:academic_year, range: '2018-19'),
# create(:academic_year, range: '2017-18'),
# create(:academic_year, range: '2016-17')
]
end
let(:i1_filepath) { Rails.root.join('tmp', 'spec', 'dese', '4A_1_grade_nine_course_pass.csv') }
let(:filepaths) do
[i1_filepath]
end
before do
FileUtils.mkdir_p 'tmp/spec/dese'
end
before :each do
academic_years
end
xcontext '#run_all' do
it 'creates a csv file with the scraped data' do
Dese::FourAOne.new(filepaths:).run_all
expect(i1_filepath).to exist
end
it 'has the correct headers' do
headers = File.open(i1_filepath) do |file|
headers = file.first
end.split(',')
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'# Grade Nine Students', '# Passing All Courses', "% Passing All Courses\n"]
end
it 'has the right likert score results for a-ovpe-i1' do
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
next unless row['Admin Data Item'] == 'a-ovpe-i1' && row['Academic Year'] == '2020-21'
likert_score = row['Likert Score']
likert_score == 'NA' ? likert_score : likert_score.to_f
end.flatten.compact
expect(results.take(20)).to eq [3.73, 3.37, 3.03, 4.03, 3.78, 3.17, 2.93, 'NA', 3.5, 4.0, 2.98, 3.84, 3.76, 3.93,
4.05, 3.13, 3.92, 3.62, 3.49, 2.5]
end
end
end

105
spec/services/dese/four_b_two_spec.rb
Unescape View File

@ -0,0 +1,105 @@
require 'rails_helper'
require 'fileutils'
require 'csv'
RSpec.describe Dese::FourBTwo do
let(:academic_years) do
[
create(:academic_year, range: '2021-22'),
create(:academic_year, range: '2020-21'),
create(:academic_year, range: '2019-20')
# create(:academic_year, range: '2018-19'),
# create(:academic_year, range: '2017-18'),
# create(:academic_year, range: '2016-17')
]
end
let(:i1_filepath) { Rails.root.join('tmp', 'spec', 'dese', '4B_2_four_year_grad.csv') }
let(:i2_filepath) { Rails.root.join('tmp', 'spec', 'dese', '4B_2_retention.csv') }
let(:i3_filepath) { Rails.root.join('tmp', 'spec', 'dese', '4B_2_five_year_grad.csv') }
let(:filepaths) do
[i1_filepath, i2_filepath, i3_filepath]
end
before do
FileUtils.mkdir_p 'tmp/spec/dese'
end
before :each do
academic_years
end
xcontext '#run_all' do
it 'creates a csv file with the scraped data' do
Dese::FourBTwo.new(filepaths:).run_all
expect(i1_filepath).to exist
expect(i2_filepath).to exist
expect(i3_filepath).to exist
end
it 'has the correct headers for i1' do
headers = File.open(i1_filepath) do |file|
headers = file.first
end.split(',')
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'# in Cohort', '% Graduated', '% Still in School', '% Non-Grad Completers', '% H.S. Equiv.',
'% Dropped Out', "% Permanently Excluded\n"]
end
it 'has the correct headers for i2' do
headers = File.open(i2_filepath) do |file|
headers = file.first
end.split(',')
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'# Enrolled', '# Retained', '% Retained', '01', '02', '03', '04', '05', '06', '07', '08', '09',
'10', '11', "12\n"]
end
it 'has the correct headers for i3' do
headers = File.open(i3_filepath) do |file|
headers = file.first
end.split(',')
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'# in Cohort', '% Graduated', '% Still in School', '% Non-Grad Completers', '% H.S. Equiv.',
'% Dropped Out', "% Permanently Excluded\n"]
end
it 'has the right likert score results for a-degr-i1' do
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
next unless row['Admin Data Item'] == 'a-degr-i1' && row['Academic Year'] == '2020-21'
likert_score = row['Likert Score']
likert_score == 'NA' ? likert_score : likert_score.to_f
end.flatten.compact
expect(results.take(20)).to eq [4.94, 4.69, 4.66, 4.94, 4.93, 4.63, 4.68, 4.29, 4.6, 4.9, 3.43, 4.84, 4.8, 4.86,
4.93, 3.62, 4.83, 3.4, 4.7, 4.62]
end
it 'has the right likert score results for a-degr-i2' do
results = CSV.parse(File.read(i2_filepath), headers: true).map do |row|
next unless row['Admin Data Item'] == 'a-degr-i2' && row['Academic Year'] == '2020-21'
likert_score = row['Likert Score']
likert_score == 'NA' ? likert_score : likert_score.to_f
end.flatten.compact
expect(results.take(20)).to eq [5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0,
5.0, 5.0, 5.0, 5.0]
end
it 'has the right likert score results for a-degr-i3' do
results = CSV.parse(File.read(i3_filepath), headers: true).map do |row|
next unless row['Admin Data Item'] == 'a-degr-i3' && row['Academic Year'] == '2019-20'
likert_score = row['Likert Score']
likert_score == 'NA' ? likert_score : likert_score.to_f
end.flatten.compact
expect(results.take(20)).to eq [4.55, 4.47, 4.5, 4.65, 4.71, 4.38, 4.51, 3.22, 4.44, 4.55, 4.57, 4.59, 4.58,
4.67, 4.04, 4.33, 4.07, 4.48, 4.5, 4.52]
end
end
end

18
spec/services/dese/three_b_two_spec.rb
Unescape View File

@ -14,7 +14,7 @@ RSpec.describe Dese::ThreeBTwo do
]
end
let(:enrollment_filepath) { Rails.root.join('tmp', 'spec', 'dese', 'enrollments.csv') }
let(:enrollment_filepath) { Rails.root.join('tmp', 'spec', 'dese', '3B_2_enrollments.csv') }
let(:teacher_race_filepath) { Rails.root.join('tmp', 'spec', 'dese', '3B_2_teacher_by_race_and_gender.csv') }
let(:student_race_filepath) { Rails.root.join('tmp', 'spec', 'dese', '3B_2_student_by_race_and_gender.csv') }
@ -41,33 +41,31 @@ RSpec.describe Dese::ThreeBTwo do
headers = file.first
end.split(',')
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Teachers of color (#)', 'School Name', 'DESE ID',
'African American (#)', 'Asian (#)', 'Hispanic (#)', 'White (#)', 'Native American (#)',
'Native Hawaiian Pacific Islander (#)', 'Multi-Race Non-Hispanic (#)', 'Females (#)',
'Males (#)', "FTE Count\n"]
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Teachers of color (%)', 'School Name', 'DESE ID',
'African American (%)', 'Asian (%)', 'Hispanic (%)', 'White (%)', 'Native American (%)',
'Native Hawaiian Pacific Islander (%)', 'Multi-Race Non-Hispanic (%)', 'Females (%)',
'Males (%)', "FTE Count\n"]
end
it 'has the correct headers for student demographic information' do
pending 'need feedback from peter'
headers = File.open(student_race_filepath) do |file|
headers = file.first
end.split(',')
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Non-White Teachers', 'Non-White Students', 'School Name', 'DESE ID',
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Non-White Teachers %', 'Non-White Students %', 'School Name', 'DESE ID',
'African American', 'Asian', 'Hispanic', 'White', 'Native American',
'Native Hawaiian or Pacific Islander', 'Multi-Race or Non-Hispanic', 'Males',
'Females', 'Non-Binary', "Students of color (%)\n"]
end
it 'has the right likert score results for a-cure-i1' do
pending 'not yet implemented'
results = CSV.parse(File.read(student_race_filepath), headers: true).map do |row|
next unless row['Admin Data Item'] == 'a-cure-i1' && row['Academic Year'] == '2020-21'
row['Likert Score'].to_f
end.flatten.compact
expect(results.take(20)).to eq [4.44, 4.44, 3.33, 3.83, 4.44, 3.6, 4.44, 4.44, 1, 4.44, 4.44, 4.44, 4.44, 3.89,
4.44, 4.44, 4.44, 4.44, 4.01, 3.92]
expect(results.take(20)).to eq [1.78, 1.0, 5.0, 5.0, 1.0, 1.0, 5.0, 1.25, 1.68, 2.22, 2.48, 2.84, 1.27, 2.15,
3.0, 1.83, 2.23, 1.0, 3.28, 2.74]
end
end
end

Write Preview
Loading…
Cancel
Save