mirror of
https://github.com/edcommonwealth/sqm-dashboards.git
synced 2026-03-07 21:48:16 -08:00
Add scrapers for admin data and their corresponding csv files of data.
Make sure specs are off so they don't run in github CI
This commit is contained in:
rebuilt
parent
94056a80c6
commit
1be6e79fd8
21 changed files with 67803 additions and 38 deletions
46
app/services/dese/five_c_one.rb
Normal file
46
app/services/dese/five_c_one.rb
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
require 'watir'
|
||||
require 'csv'
|
||||
|
||||
module Dese
|
||||
class FiveCOne
|
||||
include Dese::Scraper
|
||||
attr_reader :filepaths
|
||||
|
||||
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '5C_1_art_course.csv')])
|
||||
@filepaths = filepaths
|
||||
end
|
||||
|
||||
def run_all
|
||||
filepath = filepaths[0]
|
||||
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'K', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
|
||||
'11', '12', 'All Grades', 'Total Students']
|
||||
write_headers(filepath:, headers:)
|
||||
|
||||
run_a_picp_i1(filepath:)
|
||||
|
||||
browser.close
|
||||
end
|
||||
|
||||
def run_a_picp_i1(filepath:)
|
||||
run do |academic_year|
|
||||
admin_data_item_id = 'a-picp-i1'
|
||||
url = 'https://profiles.doe.mass.edu/statereport/artcourse.aspx'
|
||||
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
|
||||
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
|
||||
'ctl00_ContentPlaceHolder1_ddYear' => range,
|
||||
'ctl00_ContentPlaceHolder1_ddView' => 'Percent' }
|
||||
submit_id = 'btnViewReport'
|
||||
calculation = lambda { |headers, items|
|
||||
percent_graduated_index = headers['All Grades'] - 1
|
||||
return 'NA' if items[percent_graduated_index] == '' || items[percent_graduated_index].strip == '.0'
|
||||
|
||||
percent_passing = items[percent_graduated_index].to_f
|
||||
benchmark = 77.5
|
||||
percent_passing * 4 / benchmark if percent_graduated_index.present?
|
||||
}
|
||||
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
56
app/services/dese/five_d_two.rb
Normal file
56
app/services/dese/five_d_two.rb
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
require 'watir'
|
||||
require 'csv'
|
||||
|
||||
module Dese
|
||||
class FiveDTwo
|
||||
include Dese::Scraper
|
||||
include Dese::Enrollments
|
||||
attr_reader :filepaths
|
||||
|
||||
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '5D_2_enrollments.csv'),
|
||||
Rails.root.join('data', 'admin_data', 'dese', '5D_2_age_staffing.csv')])
|
||||
@filepaths = filepaths
|
||||
end
|
||||
|
||||
def run_all
|
||||
filepath = filepaths[0]
|
||||
scrape_enrollments(filepath:)
|
||||
|
||||
filepath = filepaths[1]
|
||||
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'<26 yrs (# )', '26-32 yrs (#)', '33-40 yrs (#)', '41-48 yrs (#)', '49-56 yrs (#)', '57-64 yrs (#)', 'Over 64 yrs (#)', 'FTE Count']
|
||||
write_headers(filepath:, headers:)
|
||||
|
||||
run_a_phya_i1(filepath:)
|
||||
|
||||
browser.close
|
||||
end
|
||||
|
||||
def run_a_phya_i1(filepath:)
|
||||
run do |academic_year|
|
||||
admin_data_item_id = 'a-phya-i1'
|
||||
url = 'https://profiles.doe.mass.edu/statereport/agestaffing.aspx'
|
||||
range = academic_year.range
|
||||
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
|
||||
'ctl00_ContentPlaceHolder1_ddYear' => range,
|
||||
'ctl00_ContentPlaceHolder1_ddJobClassification' => 'School Nurse -- Non-Special Education' }
|
||||
submit_id = 'btnViewReport'
|
||||
calculation = lambda { |headers, items|
|
||||
nurse_index = headers['FTE Count']
|
||||
return 'NA' if items[nurse_index] == '' || items[nurse_index].strip == '.0'
|
||||
|
||||
nurse_count = items[nurse_index].to_f
|
||||
benchmark = 750
|
||||
nurse_count * 4 / benchmark if nurse_index.present?
|
||||
|
||||
dese_id = items[headers['School Code']].to_i
|
||||
num_of_students = student_count(filepath: filepaths[0], dese_id:, year: academic_year.range) || 0
|
||||
items << num_of_students
|
||||
items << (num_of_students / nurse_count)
|
||||
((benchmark - (num_of_students / nurse_count)) + benchmark) * 4 / benchmark
|
||||
}
|
||||
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
44
app/services/dese/four_a_one.rb
Normal file
44
app/services/dese/four_a_one.rb
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
require 'watir'
|
||||
require 'csv'
|
||||
|
||||
module Dese
|
||||
class FourAOne
|
||||
include Dese::Scraper
|
||||
attr_reader :filepaths
|
||||
|
||||
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '4A_1_grade_nine_course_pass.csv')])
|
||||
@filepaths = filepaths
|
||||
end
|
||||
|
||||
def run_all
|
||||
filepath = filepaths[0]
|
||||
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'# Grade Nine Students', '# Passing All Courses', '% Passing All Courses']
|
||||
write_headers(filepath:, headers:)
|
||||
|
||||
run_a_ovpe_i1(filepath:)
|
||||
|
||||
browser.close
|
||||
end
|
||||
|
||||
def run_a_ovpe_i1(filepath:)
|
||||
run do |academic_year|
|
||||
admin_data_item_id = 'a-ovpe-i1'
|
||||
url = 'https://profiles.doe.mass.edu/statereport/gradeninecoursepass.aspx'
|
||||
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
|
||||
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
|
||||
'ctl00_ContentPlaceHolder1_ddYear' => range }
|
||||
submit_id = 'btnViewReport'
|
||||
calculation = lambda { |headers, items|
|
||||
passing_index = headers['% Passing All Courses']
|
||||
return 'NA' if items[passing_index] == '' || items[passing_index].strip == '.0'
|
||||
|
||||
percent_passing = items[passing_index].to_f
|
||||
benchmark = 95
|
||||
percent_passing * 4 / benchmark if passing_index.present?
|
||||
}
|
||||
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
105
app/services/dese/four_b_two.rb
Normal file
105
app/services/dese/four_b_two.rb
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
require 'watir'
|
||||
require 'csv'
|
||||
|
||||
module Dese
|
||||
class FourBTwo
|
||||
include Dese::Scraper
|
||||
attr_reader :filepaths
|
||||
|
||||
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '4B_2_four_year_grad.csv'),
|
||||
Rails.root.join('data', 'admin_data', 'dese', '4B_2_retention.csv'),
|
||||
Rails.root.join('data', 'admin_data', 'dese', '4B_2_five_year_grad.csv')])
|
||||
@filepaths = filepaths
|
||||
end
|
||||
|
||||
def run_all
|
||||
filepath = filepaths[0]
|
||||
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'# in Cohort', '% Graduated', '% Still in School', '% Non-Grad Completers', '% H.S. Equiv.',
|
||||
'% Dropped Out', '% Permanently Excluded']
|
||||
write_headers(filepath:, headers:)
|
||||
|
||||
run_a_degr_i1(filepath:)
|
||||
|
||||
filepath = filepaths[1]
|
||||
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'# Enrolled', '# Retained', '% Retained', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
|
||||
'11', '12']
|
||||
write_headers(filepath:, headers:)
|
||||
|
||||
run_a_degr_i2(filepath:)
|
||||
|
||||
filepath = filepaths[2]
|
||||
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'# in Cohort', '% Graduated', '% Still in School', '% Non-Grad Completers', '% H.S. Equiv.',
|
||||
'% Dropped Out', '% Permanently Excluded']
|
||||
write_headers(filepath:, headers:)
|
||||
|
||||
run_a_degr_i3(filepath:)
|
||||
browser.close
|
||||
end
|
||||
|
||||
def run_a_degr_i1(filepath:)
|
||||
run do |academic_year|
|
||||
admin_data_item_id = 'a-degr-i1'
|
||||
url = 'https://profiles.doe.mass.edu/statereport/gradrates.aspx'
|
||||
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
|
||||
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
|
||||
'ctl00_ContentPlaceHolder1_ddYear' => range,
|
||||
'ctl00_ContentPlaceHolder1_ddRateType' => '4yr Grad' }
|
||||
submit_id = 'btnViewReport'
|
||||
calculation = lambda { |headers, items|
|
||||
percent_graduated_index = headers['% Graduated']
|
||||
return 'NA' if items[percent_graduated_index] == '' || items[percent_graduated_index].strip == '.0'
|
||||
|
||||
percent_passing = items[percent_graduated_index].to_f
|
||||
benchmark = 80
|
||||
percent_passing * 4 / benchmark if percent_graduated_index.present?
|
||||
}
|
||||
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
|
||||
end
|
||||
end
|
||||
|
||||
def run_a_degr_i2(filepath:)
|
||||
run do |academic_year|
|
||||
admin_data_item_id = 'a-degr-i2'
|
||||
url = 'https://profiles.doe.mass.edu/statereport/retention.aspx'
|
||||
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
|
||||
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
|
||||
'ctl00_ContentPlaceHolder1_ddYear' => range,
|
||||
'ctl00_ContentPlaceHolder1_ddView' => 'Percent' }
|
||||
submit_id = 'btnViewReport'
|
||||
calculation = lambda { |headers, items|
|
||||
retained_index = headers['% Retained']
|
||||
return 'NA' if items[retained_index] == '' || items[retained_index].strip == '.0'
|
||||
|
||||
percent_retained = items[retained_index].to_f
|
||||
benchmark = 2
|
||||
((benchmark - percent_retained) + benchmark) * 4 / benchmark if retained_index.present?
|
||||
}
|
||||
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
|
||||
end
|
||||
end
|
||||
|
||||
def run_a_degr_i3(filepath:)
|
||||
run do |academic_year|
|
||||
admin_data_item_id = 'a-degr-i3'
|
||||
url = 'https://profiles.doe.mass.edu/statereport/gradrates.aspx'
|
||||
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
|
||||
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
|
||||
'ctl00_ContentPlaceHolder1_ddYear' => range,
|
||||
'ctl00_ContentPlaceHolder1_ddRateType' => '5yr Grad' }
|
||||
submit_id = 'btnViewReport'
|
||||
calculation = lambda { |headers, items|
|
||||
percent_graduated_index = headers['% Graduated']
|
||||
return 'NA' if items[percent_graduated_index] == '' || items[percent_graduated_index].strip == '.0'
|
||||
|
||||
percent_passing = items[percent_graduated_index].to_f
|
||||
benchmark = 85
|
||||
percent_passing * 4 / benchmark if percent_graduated_index.present?
|
||||
}
|
||||
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -71,6 +71,7 @@ module Dese
|
|||
likert_score = likert_score.round(2)
|
||||
end
|
||||
|
||||
# byebug if dese_id == 30_305
|
||||
output = []
|
||||
output << raw_likert_score
|
||||
output << likert_score
|
||||
|
|
|
|||
|
|
@ -7,26 +7,26 @@ module Dese
|
|||
include Dese::Enrollments
|
||||
attr_reader :filepaths
|
||||
|
||||
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', 'enrollments.csv'),
|
||||
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '3B_2_enrollments.csv'),
|
||||
Rails.root.join('data', 'admin_data', 'dese', '3B_2_teacher_by_race_and_gender.csv'),
|
||||
Rails.root.join('data', 'admin_data', 'dese', '3B_2_student_by_race_and_gender.csv')])
|
||||
@filepaths = filepaths
|
||||
end
|
||||
|
||||
def run_all
|
||||
filepath = filepaths[0]
|
||||
scrape_enrollments(filepath:)
|
||||
# filepath = filepaths[0]
|
||||
# scrape_enrollments(filepath:)
|
||||
|
||||
filepath = filepaths[1]
|
||||
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Teachers of color (#)', 'School Name', 'DESE ID',
|
||||
'African American (#)', 'Asian (#)', 'Hispanic (#)', 'White (#)', 'Native American (#)',
|
||||
'Native Hawaiian Pacific Islander (#)', 'Multi-Race Non-Hispanic (#)', 'Females (#)',
|
||||
'Males (#)', 'FTE Count']
|
||||
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Teachers of color (%)', 'School Name', 'DESE ID',
|
||||
'African American (%)', 'Asian (%)', 'Hispanic (%)', 'White (%)', 'Native American (%)',
|
||||
'Native Hawaiian Pacific Islander (%)', 'Multi-Race Non-Hispanic (%)', 'Females (%)',
|
||||
'Males (%)', 'FTE Count']
|
||||
write_headers(filepath:, headers:)
|
||||
run_teacher_demographics(filepath:)
|
||||
|
||||
filepath = filepaths[2]
|
||||
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Non-White Teachers', 'Non-White Students', 'School Name', 'DESE ID',
|
||||
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Non-White Teachers %', 'Non-White Students %', 'School Name', 'DESE ID',
|
||||
'African American', 'Asian', 'Hispanic', 'White', 'Native American',
|
||||
'Native Hawaiian or Pacific Islander', 'Multi-Race or Non-Hispanic', 'Males',
|
||||
'Females', 'Non-Binary', 'Students of color (%)']
|
||||
|
|
@ -42,31 +42,32 @@ module Dese
|
|||
url = 'https://profiles.doe.mass.edu/statereport/teacherbyracegender.aspx'
|
||||
range = academic_year.range
|
||||
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
|
||||
'ctl00_ContentPlaceHolder1_ddYear' => range }
|
||||
'ctl00_ContentPlaceHolder1_ddYear' => range,
|
||||
'ctl00_ContentPlaceHolder1_ddDisplay' => 'Percentages' }
|
||||
submit_id = 'ctl00_ContentPlaceHolder1_btnViewReport'
|
||||
calculation = lambda { |headers, items|
|
||||
african_american_index = headers['African American (#)']
|
||||
african_american_index = headers['African American (%)']
|
||||
african_american_number = items[african_american_index].to_f
|
||||
|
||||
asian_index = headers['Asian (#)']
|
||||
asian_index = headers['Asian (%)']
|
||||
asian_number = items[asian_index].to_f
|
||||
|
||||
hispanic_index = headers['Hispanic (#)']
|
||||
hispanic_index = headers['Hispanic (%)']
|
||||
hispanic_number = items[hispanic_index].to_f
|
||||
|
||||
native_american_index = headers['Native American (#)']
|
||||
native_american_index = headers['Native American (%)']
|
||||
native_american_number = items[native_american_index].to_f
|
||||
|
||||
native_hawaiian_index = headers['Native Hawaiian, Pacific Islander (#)']
|
||||
native_hawaiian_index = headers['Native Hawaiian, Pacific Islander (%)']
|
||||
native_hawaiian_number = items[native_hawaiian_index].to_f
|
||||
|
||||
multi_race_index = headers['Multi-Race,Non-Hispanic (#)']
|
||||
multi_race_index = headers['Multi-Race,Non-Hispanic (%)']
|
||||
multi_race_number = items[multi_race_index].to_f
|
||||
|
||||
total_non_white_teachers = african_american_number + asian_number + hispanic_number + native_american_number + native_hawaiian_number + multi_race_number
|
||||
items.unshift(total_non_white_teachers)
|
||||
non_white_teachers = african_american_number + asian_number + hispanic_number + native_american_number + native_hawaiian_number + multi_race_number
|
||||
items.unshift(non_white_teachers)
|
||||
|
||||
total_non_white_teachers
|
||||
non_white_teachers
|
||||
}
|
||||
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
|
||||
end
|
||||
|
|
@ -78,7 +79,7 @@ module Dese
|
|||
CSV.parse(File.read(filepath), headers: true).map do |row|
|
||||
academic_year = row['Academic Year']
|
||||
school_id = row['DESE ID'].to_i
|
||||
total = row['Teachers of color (#)'].gsub(',', '').to_f
|
||||
total = row['Teachers of color (%)'].gsub(',', '').to_f
|
||||
@teachers[[school_id, academic_year]] = total
|
||||
end
|
||||
end
|
||||
|
|
@ -96,17 +97,19 @@ module Dese
|
|||
calculation = lambda { |headers, items|
|
||||
white_index = headers['White']
|
||||
white_number = items[white_index].to_f
|
||||
non_white_student_percentage = 100 - white_number
|
||||
|
||||
dese_id = items[headers['School Code']].to_i
|
||||
num_of_students = student_count(filepath: filepaths[0], dese_id:, year: academic_year.range) || 0
|
||||
num_of_non_white_students = num_of_students * non_white_student_percentage / 100
|
||||
items.unshift(num_of_non_white_students)
|
||||
non_white_student_percentage = (100 - white_number).to_f
|
||||
items.unshift(non_white_student_percentage)
|
||||
|
||||
num_of_non_white_teachers = teacher_count(filepath: filepaths[1], dese_id:, year: academic_year.range)
|
||||
items.unshift(num_of_non_white_teachers)
|
||||
parity_index = num_of_non_white_teachers.to_f / num_of_non_white_students.to_f
|
||||
parity_index * 4 / 0.26
|
||||
non_white_teachers = teacher_count(filepath: filepaths[1], dese_id:, year: academic_year.range).to_f
|
||||
items.unshift(non_white_teachers)
|
||||
# if non_white_teachers >= 10
|
||||
parity_index = non_white_teachers / non_white_student_percentage
|
||||
likert_score = parity_index * 4 / 0.25
|
||||
# else
|
||||
# likert_score = 1
|
||||
# end
|
||||
likert_score
|
||||
}
|
||||
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
|
||||
end
|
||||
|
|
|
|||
11067
data/admin_data/dese/3B_2_student_by_race_and_gender.csv
Normal file
11067
data/admin_data/dese/3B_2_student_by_race_and_gender.csv
Normal file
File diff suppressed because it is too large
Load diff
11067
data/admin_data/dese/3B_2_teacher_by_race_and_gender.csv
Normal file
11067
data/admin_data/dese/3B_2_teacher_by_race_and_gender.csv
Normal file
File diff suppressed because it is too large
Load diff
1582
data/admin_data/dese/4A_1_grade_nine_course_pass.csv
Normal file
1582
data/admin_data/dese/4A_1_grade_nine_course_pass.csv
Normal file
File diff suppressed because it is too large
Load diff
1544
data/admin_data/dese/4B_2_five_year_grad.csv
Normal file
1544
data/admin_data/dese/4B_2_five_year_grad.csv
Normal file
File diff suppressed because it is too large
Load diff
1936
data/admin_data/dese/4B_2_four_year_grad.csv
Normal file
1936
data/admin_data/dese/4B_2_four_year_grad.csv
Normal file
File diff suppressed because it is too large
Load diff
8799
data/admin_data/dese/4B_2_retention.csv
Normal file
8799
data/admin_data/dese/4B_2_retention.csv
Normal file
File diff suppressed because it is too large
Load diff
1979
data/admin_data/dese/4D_1_plans_of_grads.csv
Normal file
1979
data/admin_data/dese/4D_1_plans_of_grads.csv
Normal file
File diff suppressed because it is too large
Load diff
7130
data/admin_data/dese/5C_1_art_course.csv
Normal file
7130
data/admin_data/dese/5C_1_art_course.csv
Normal file
File diff suppressed because it is too large
Load diff
11059
data/admin_data/dese/5D_2_age_staffing.csv
Normal file
11059
data/admin_data/dese/5D_2_age_staffing.csv
Normal file
File diff suppressed because it is too large
Load diff
11067
data/admin_data/dese/5D_2_enrollments.csv
Normal file
11067
data/admin_data/dese/5D_2_enrollments.csv
Normal file
File diff suppressed because it is too large
Load diff
55
spec/services/dese/five_c_one_spec.rb
Normal file
55
spec/services/dese/five_c_one_spec.rb
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Dese::FiveCOne do
|
||||
let(:academic_years) do
|
||||
[
|
||||
create(:academic_year, range: '2020-21'),
|
||||
create(:academic_year, range: '2019-20')
|
||||
# create(:academic_year, range: '2018-19'),
|
||||
# create(:academic_year, range: '2017-18'),
|
||||
# create(:academic_year, range: '2016-17')
|
||||
]
|
||||
end
|
||||
|
||||
let(:i1_filepath) { Rails.root.join('tmp', 'spec', 'dese', '5C_1_art_course.csv') }
|
||||
|
||||
let(:filepaths) do
|
||||
[i1_filepath]
|
||||
end
|
||||
before do
|
||||
FileUtils.mkdir_p 'tmp/spec/dese'
|
||||
end
|
||||
|
||||
before :each do
|
||||
academic_years
|
||||
end
|
||||
|
||||
xcontext '#run_all' do
|
||||
it 'creates a csv file with the scraped data' do
|
||||
Dese::FiveCOne.new(filepaths:).run_all
|
||||
expect(i1_filepath).to exist
|
||||
end
|
||||
|
||||
it 'has the correct headers for i1' do
|
||||
headers = File.open(i1_filepath) do |file|
|
||||
headers = file.first
|
||||
end.split(',')
|
||||
|
||||
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'K', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
|
||||
'11', '12', 'All Grades', "Total Students\n"]
|
||||
end
|
||||
|
||||
it 'has the right likert score results for a-picp-i1' do
|
||||
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
|
||||
next unless row['Admin Data Item'] == 'a-picp-i1' && row['Academic Year'] == '2020-21'
|
||||
|
||||
likert_score = row['Likert Score']
|
||||
likert_score == 'NA' ? likert_score : likert_score.to_f
|
||||
end.flatten.compact
|
||||
|
||||
expect(results.take(20)).to eq [4.95, 2.39, 4.81, 4.89, 4.63, 4.95, 2.25, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.75,
|
||||
4.82, 1.0, 3.88, 3.14, 4.84, 5.0]
|
||||
end
|
||||
end
|
||||
end
|
||||
66
spec/services/dese/five_d_two_spec.rb
Normal file
66
spec/services/dese/five_d_two_spec.rb
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Dese::FiveDTwo do
|
||||
let(:academic_years) do
|
||||
[
|
||||
create(:academic_year, range: '2021-22'),
|
||||
create(:academic_year, range: '2020-21')
|
||||
# create(:academic_year, range: '2019-20')
|
||||
# create(:academic_year, range: '2018-19'),
|
||||
# create(:academic_year, range: '2017-18'),
|
||||
# create(:academic_year, range: '2016-17')
|
||||
]
|
||||
end
|
||||
|
||||
let(:enrollments_filepath) { Rails.root.join('tmp', 'spec', 'dese', '5D_2_enrollments.csv') }
|
||||
let(:i1_filepath) { Rails.root.join('tmp', 'spec', 'dese', '5D_2_age_staffing.csv') }
|
||||
|
||||
let(:filepaths) do
|
||||
[enrollments_filepath, i1_filepath]
|
||||
end
|
||||
before do
|
||||
FileUtils.mkdir_p 'tmp/spec/dese'
|
||||
end
|
||||
|
||||
before :each do
|
||||
academic_years
|
||||
end
|
||||
|
||||
xcontext '#run_all' do
|
||||
it 'creates a csv file with the scraped data' do
|
||||
Dese::FiveDTwo.new(filepaths:).run_all
|
||||
expect(enrollments_filepath).to exist
|
||||
expect(i1_filepath).to exist
|
||||
end
|
||||
|
||||
it 'has the correct headers for enrollements' do
|
||||
headers = File.open(enrollments_filepath) do |file|
|
||||
headers = file.first
|
||||
end.split(',')
|
||||
|
||||
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'PK', 'K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', 'SP', "Total\n"]
|
||||
end
|
||||
|
||||
it 'has the correct headers for i1' do
|
||||
headers = File.open(i1_filepath) do |file|
|
||||
headers = file.first
|
||||
end.split(',')
|
||||
|
||||
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'<26 yrs (# )', '26-32 yrs (#)', '33-40 yrs (#)', '41-48 yrs (#)', '49-56 yrs (#)', '57-64 yrs (#)', 'Over 64 yrs (#)', "FTE Count\n"]
|
||||
end
|
||||
|
||||
it 'has the right likert score results for a-phya-i1' do
|
||||
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
|
||||
next unless row['Admin Data Item'] == 'a-phya-i1' && row['Academic Year'] == '2020-21'
|
||||
|
||||
likert_score = row['Likert Score']
|
||||
likert_score == 'NA' ? likert_score : likert_score.to_f
|
||||
end.flatten.compact
|
||||
|
||||
expect(results.take(20)).to eq [5.0, 1.0, 4.7, 4.59, 5.0, 5.0, 1.0, 3.33, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0,
|
||||
5.0, 5.0, 4.78, 5.0]
|
||||
end
|
||||
end
|
||||
end
|
||||
56
spec/services/dese/four_a_one_spec.rb
Normal file
56
spec/services/dese/four_a_one_spec.rb
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
require 'rails_helper'
|
||||
require 'fileutils'
|
||||
require 'csv'
|
||||
|
||||
RSpec.describe Dese::FourAOne do
|
||||
let(:academic_years) do
|
||||
[
|
||||
create(:academic_year, range: '2021-22'),
|
||||
create(:academic_year, range: '2020-21')
|
||||
# create(:academic_year, range: '2019-20'),
|
||||
# create(:academic_year, range: '2018-19'),
|
||||
# create(:academic_year, range: '2017-18'),
|
||||
# create(:academic_year, range: '2016-17')
|
||||
]
|
||||
end
|
||||
|
||||
let(:i1_filepath) { Rails.root.join('tmp', 'spec', 'dese', '4A_1_grade_nine_course_pass.csv') }
|
||||
|
||||
let(:filepaths) do
|
||||
[i1_filepath]
|
||||
end
|
||||
before do
|
||||
FileUtils.mkdir_p 'tmp/spec/dese'
|
||||
end
|
||||
|
||||
before :each do
|
||||
academic_years
|
||||
end
|
||||
|
||||
xcontext '#run_all' do
|
||||
it 'creates a csv file with the scraped data' do
|
||||
Dese::FourAOne.new(filepaths:).run_all
|
||||
expect(i1_filepath).to exist
|
||||
end
|
||||
|
||||
it 'has the correct headers' do
|
||||
headers = File.open(i1_filepath) do |file|
|
||||
headers = file.first
|
||||
end.split(',')
|
||||
|
||||
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'# Grade Nine Students', '# Passing All Courses', "% Passing All Courses\n"]
|
||||
end
|
||||
it 'has the right likert score results for a-ovpe-i1' do
|
||||
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
|
||||
next unless row['Admin Data Item'] == 'a-ovpe-i1' && row['Academic Year'] == '2020-21'
|
||||
|
||||
likert_score = row['Likert Score']
|
||||
likert_score == 'NA' ? likert_score : likert_score.to_f
|
||||
end.flatten.compact
|
||||
|
||||
expect(results.take(20)).to eq [3.73, 3.37, 3.03, 4.03, 3.78, 3.17, 2.93, 'NA', 3.5, 4.0, 2.98, 3.84, 3.76, 3.93,
|
||||
4.05, 3.13, 3.92, 3.62, 3.49, 2.5]
|
||||
end
|
||||
end
|
||||
end
|
||||
105
spec/services/dese/four_b_two_spec.rb
Normal file
105
spec/services/dese/four_b_two_spec.rb
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
require 'rails_helper'
|
||||
require 'fileutils'
|
||||
require 'csv'
|
||||
|
||||
RSpec.describe Dese::FourBTwo do
|
||||
let(:academic_years) do
|
||||
[
|
||||
create(:academic_year, range: '2021-22'),
|
||||
create(:academic_year, range: '2020-21'),
|
||||
create(:academic_year, range: '2019-20')
|
||||
# create(:academic_year, range: '2018-19'),
|
||||
# create(:academic_year, range: '2017-18'),
|
||||
# create(:academic_year, range: '2016-17')
|
||||
]
|
||||
end
|
||||
|
||||
let(:i1_filepath) { Rails.root.join('tmp', 'spec', 'dese', '4B_2_four_year_grad.csv') }
|
||||
let(:i2_filepath) { Rails.root.join('tmp', 'spec', 'dese', '4B_2_retention.csv') }
|
||||
let(:i3_filepath) { Rails.root.join('tmp', 'spec', 'dese', '4B_2_five_year_grad.csv') }
|
||||
|
||||
let(:filepaths) do
|
||||
[i1_filepath, i2_filepath, i3_filepath]
|
||||
end
|
||||
before do
|
||||
FileUtils.mkdir_p 'tmp/spec/dese'
|
||||
end
|
||||
|
||||
before :each do
|
||||
academic_years
|
||||
end
|
||||
|
||||
xcontext '#run_all' do
|
||||
it 'creates a csv file with the scraped data' do
|
||||
Dese::FourBTwo.new(filepaths:).run_all
|
||||
expect(i1_filepath).to exist
|
||||
expect(i2_filepath).to exist
|
||||
expect(i3_filepath).to exist
|
||||
end
|
||||
|
||||
it 'has the correct headers for i1' do
|
||||
headers = File.open(i1_filepath) do |file|
|
||||
headers = file.first
|
||||
end.split(',')
|
||||
|
||||
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'# in Cohort', '% Graduated', '% Still in School', '% Non-Grad Completers', '% H.S. Equiv.',
|
||||
'% Dropped Out', "% Permanently Excluded\n"]
|
||||
end
|
||||
|
||||
it 'has the correct headers for i2' do
|
||||
headers = File.open(i2_filepath) do |file|
|
||||
headers = file.first
|
||||
end.split(',')
|
||||
|
||||
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'# Enrolled', '# Retained', '% Retained', '01', '02', '03', '04', '05', '06', '07', '08', '09',
|
||||
'10', '11', "12\n"]
|
||||
end
|
||||
|
||||
it 'has the correct headers for i3' do
|
||||
headers = File.open(i3_filepath) do |file|
|
||||
headers = file.first
|
||||
end.split(',')
|
||||
|
||||
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
|
||||
'# in Cohort', '% Graduated', '% Still in School', '% Non-Grad Completers', '% H.S. Equiv.',
|
||||
'% Dropped Out', "% Permanently Excluded\n"]
|
||||
end
|
||||
it 'has the right likert score results for a-degr-i1' do
|
||||
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
|
||||
next unless row['Admin Data Item'] == 'a-degr-i1' && row['Academic Year'] == '2020-21'
|
||||
|
||||
likert_score = row['Likert Score']
|
||||
likert_score == 'NA' ? likert_score : likert_score.to_f
|
||||
end.flatten.compact
|
||||
|
||||
expect(results.take(20)).to eq [4.94, 4.69, 4.66, 4.94, 4.93, 4.63, 4.68, 4.29, 4.6, 4.9, 3.43, 4.84, 4.8, 4.86,
|
||||
4.93, 3.62, 4.83, 3.4, 4.7, 4.62]
|
||||
end
|
||||
|
||||
it 'has the right likert score results for a-degr-i2' do
|
||||
results = CSV.parse(File.read(i2_filepath), headers: true).map do |row|
|
||||
next unless row['Admin Data Item'] == 'a-degr-i2' && row['Academic Year'] == '2020-21'
|
||||
|
||||
likert_score = row['Likert Score']
|
||||
likert_score == 'NA' ? likert_score : likert_score.to_f
|
||||
end.flatten.compact
|
||||
|
||||
expect(results.take(20)).to eq [5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0,
|
||||
5.0, 5.0, 5.0, 5.0]
|
||||
end
|
||||
|
||||
it 'has the right likert score results for a-degr-i3' do
|
||||
results = CSV.parse(File.read(i3_filepath), headers: true).map do |row|
|
||||
next unless row['Admin Data Item'] == 'a-degr-i3' && row['Academic Year'] == '2019-20'
|
||||
|
||||
likert_score = row['Likert Score']
|
||||
likert_score == 'NA' ? likert_score : likert_score.to_f
|
||||
end.flatten.compact
|
||||
|
||||
expect(results.take(20)).to eq [4.55, 4.47, 4.5, 4.65, 4.71, 4.38, 4.51, 3.22, 4.44, 4.55, 4.57, 4.59, 4.58,
|
||||
4.67, 4.04, 4.33, 4.07, 4.48, 4.5, 4.52]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -14,7 +14,7 @@ RSpec.describe Dese::ThreeBTwo do
|
|||
]
|
||||
end
|
||||
|
||||
let(:enrollment_filepath) { Rails.root.join('tmp', 'spec', 'dese', 'enrollments.csv') }
|
||||
let(:enrollment_filepath) { Rails.root.join('tmp', 'spec', 'dese', '3B_2_enrollments.csv') }
|
||||
let(:teacher_race_filepath) { Rails.root.join('tmp', 'spec', 'dese', '3B_2_teacher_by_race_and_gender.csv') }
|
||||
let(:student_race_filepath) { Rails.root.join('tmp', 'spec', 'dese', '3B_2_student_by_race_and_gender.csv') }
|
||||
|
||||
|
|
@ -41,33 +41,31 @@ RSpec.describe Dese::ThreeBTwo do
|
|||
headers = file.first
|
||||
end.split(',')
|
||||
|
||||
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Teachers of color (#)', 'School Name', 'DESE ID',
|
||||
'African American (#)', 'Asian (#)', 'Hispanic (#)', 'White (#)', 'Native American (#)',
|
||||
'Native Hawaiian Pacific Islander (#)', 'Multi-Race Non-Hispanic (#)', 'Females (#)',
|
||||
'Males (#)', "FTE Count\n"]
|
||||
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Teachers of color (%)', 'School Name', 'DESE ID',
|
||||
'African American (%)', 'Asian (%)', 'Hispanic (%)', 'White (%)', 'Native American (%)',
|
||||
'Native Hawaiian Pacific Islander (%)', 'Multi-Race Non-Hispanic (%)', 'Females (%)',
|
||||
'Males (%)', "FTE Count\n"]
|
||||
end
|
||||
it 'has the correct headers for student demographic information' do
|
||||
pending 'need feedback from peter'
|
||||
headers = File.open(student_race_filepath) do |file|
|
||||
headers = file.first
|
||||
end.split(',')
|
||||
|
||||
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Non-White Teachers', 'Non-White Students', 'School Name', 'DESE ID',
|
||||
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'Non-White Teachers %', 'Non-White Students %', 'School Name', 'DESE ID',
|
||||
'African American', 'Asian', 'Hispanic', 'White', 'Native American',
|
||||
'Native Hawaiian or Pacific Islander', 'Multi-Race or Non-Hispanic', 'Males',
|
||||
'Females', 'Non-Binary', "Students of color (%)\n"]
|
||||
end
|
||||
|
||||
it 'has the right likert score results for a-cure-i1' do
|
||||
pending 'not yet implemented'
|
||||
results = CSV.parse(File.read(student_race_filepath), headers: true).map do |row|
|
||||
next unless row['Admin Data Item'] == 'a-cure-i1' && row['Academic Year'] == '2020-21'
|
||||
|
||||
row['Likert Score'].to_f
|
||||
end.flatten.compact
|
||||
|
||||
expect(results.take(20)).to eq [4.44, 4.44, 3.33, 3.83, 4.44, 3.6, 4.44, 4.44, 1, 4.44, 4.44, 4.44, 4.44, 3.89,
|
||||
4.44, 4.44, 4.44, 4.44, 4.01, 3.92]
|
||||
expect(results.take(20)).to eq [1.78, 1.0, 5.0, 5.0, 1.0, 1.0, 5.0, 1.25, 1.68, 2.22, 2.48, 2.84, 1.27, 2.15,
|
||||
3.0, 1.83, 2.23, 1.0, 3.28, 2.74]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue