gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-09 07:28:41 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add scraper for 3A-1 and 2C-1. Finishes #183105787 and #183105716

Browse source
This commit is contained in:
rebuilt 2022-09-07 11:34:42 -07:00
parent b5bc9c9de4
commit 83ef9310a4
10 changed files with 18754 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

10
spec/services/dese/scraper.rb
View file

@ -1,10 +0,0 @@
require 'watir'
require 'csv'
module Dese
class OneAScraper
def initialize(filepath: Rails.root.join('data', 'admin_data', 'dese', 'scraped.csv')); end
def run; end
end
end

57
spec/services/dese/three_a_one_spec.rb Normal file
View file

@ -0,0 +1,57 @@
require 'rails_helper'
require 'fileutils'
require 'csv'
RSpec.describe Dese::ThreeAOne do
let(:academic_years) do
[
create(:academic_year, range: '2021-22'),
create(:academic_year, range: '2020-21'),
create(:academic_year, range: '2019-20'),
create(:academic_year, range: '2018-19'),
create(:academic_year, range: '2017-18'),
create(:academic_year, range: '2016-17')
]
end
let(:i1_filepath) { Rails.root.join('tmp', 'spec', 'dese', 'three_a_one_average_class_size.csv') }
let(:filepaths) do
[i1_filepath]
end
before do
FileUtils.mkdir_p 'tmp/spec/dese'
end
before :each do
academic_years
end
context 'Creating a new Scraper' do
it 'creates a csv file with the scraped data' do
Dese::ThreeAOne.new(filepaths:)
expect(i1_filepath).to exist
end
it 'has the correct headers' do
headers = File.open(i1_filepath) do |file|
headers = file.first
end.split(',')
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'Total # of Classes', 'Average Class Size', 'Number of Students', 'Female %', 'Male %',
'English Language Learner %', 'Students with Disabilities %', "Economically Disadvantaged %\n"]
end
it 'has the right likert score results for a-reso-i1' do
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
next unless row['Admin Data Item'] == 'a-reso-i1' && row['Academic Year'] == '2020-21'
row['Likert Score'].to_f
end.flatten.compact
expect(results.take(20)).to eq [4.22, 5.0, 4.58, 3.46, 3.98, 3.68, 4.06, 4.84, 4.42, 4.66, 5.0, 4.6, 4.26, 4.46,
4.2, 4.66, 5.0, 4.6, 4.28, 5.0]
end
end
end

11
spec/services/dese/two_a_one_scraper_spec.rb
View file

@ -47,20 +47,21 @@ RSpec.describe Dese::TwoAOneScraper do
it 'has the right likert score results for a-phys-i1' do
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
row['Likert Score'].to_f
end
row['Likert Score'].to_f unless row['Likert Score'] == 'NA' || row['Likert Score'].nil?
end.flatten.compact
expect(results.take(20)).to eq [5.0, 5.0, 4.33, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0,
expect(results.take(20)).to eq [5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0,
5.0, 5.0, 5.0, 5.0]
end
it 'has the right likert score results for a-exp-i3' do
results = CSV.parse(File.read(i3_filepath), headers: true).map do |row|
next unless row['Admin Data Item'] == 'a-phys-i3' && row['Academic Year'] == '2020-21'
row['Likert Score'].to_f
row['Likert Score'].to_f unless row['Likert Score'] == 'NA' || row['Likert Score'].nil?
end.flatten.compact
expect(results.take(20)).to eq [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5]
expect(results.take(20)).to eq [5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.6, 5.0,
5.0, 5.0, 5.0, 5.0]
end
end
end

67
spec/services/dese/two_c_one_scraper_spec.rb Normal file
View file

@ -0,0 +1,67 @@
require 'rails_helper'
require 'fileutils'
require 'csv'
RSpec.describe Dese::TwoCOneScraper do
let(:academic_years) do
[
create(:academic_year, range: '2021-22'),
create(:academic_year, range: '2020-21'),
create(:academic_year, range: '2019-20'),
create(:academic_year, range: '2018-19'),
create(:academic_year, range: '2017-18'),
create(:academic_year, range: '2016-17')
]
end
let(:i1_filepath) { Rails.root.join('tmp', 'spec', 'dese', 'two_c_one_attendance.csv') }
let(:filepaths) do
[i1_filepath]
end
before do
FileUtils.mkdir_p 'tmp/spec/dese'
end
before :each do
academic_years
end
context 'Creating a new Scraper' do
it 'creates a csv file with the scraped data' do
Dese::TwoCOneScraper.new(filepaths:)
expect(i1_filepath).to exist
end
it 'has the correct headers' do
headers = File.open(i1_filepath) do |file|
headers = file.first
end.split(',')
expect(headers).to eq ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'Attendance Rate', 'Average # of Absences', 'Absent 10 or more days', 'Chronically Absent (10% or more)',
'Chronically Absent (20% or more)', "Unexcused > 9 days\n"]
end
it 'has the right likert score results for a-vale-i1' do
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
next unless row['Admin Data Item'] == 'a-vale-i1' && row['Academic Year'] == '2021-22'
row['Likert Score'].to_f
end.flatten.compact
expect(results.take(20)).to eq [1.88, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 3.16, 3.84, 1.0, 4.6, 3.64, 4.84, 2.68, 2.84,
3.08, 1.0, 2.56, 3.96, 1.0]
end
it 'has the right likert score results for a-vale-i2' do
results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
next unless row['Admin Data Item'] == 'a-vale-i2' && row['Academic Year'] == '2021-22'
row['Likert Score'].to_f
end.flatten.compact
expect(results.take(20)).to eq [4.2, 4.07, 4.11, 4.14, 4.0, 3.93, 3.92, 4.22, 4.21, 4.09, 4.24, 4.2, 4.26, 4.19,
4.2, 4.22, 4.16, 4.18, 4.23, 4.02]
end
end
end