gabehf/Dashboard
1
0
Fork 0
mirror of https://github.com/edcommonwealth/Dashboard.git synced 2026-03-19 02:26:24 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: scrape enrollment and staffing data

Browse source
This commit is contained in:
Nelson Jovel 2024-02-06 14:51:00 -08:00
parent 725348bf95
commit 1810ee0074
7 changed files with 15026 additions and 13173 deletions
Download patch file Download diff file Expand all files Collapse all files

64
app/services/dashboard/dese/enrollments.rb
View file

@ -1,38 +1,40 @@
require 'watir'
require 'csv'
require "watir"
require "csv"
module Dese
module Enrollments
include Dese::Scraper
attr_reader :filepaths
module Dashboard
module Dese
module Enrollments
include Dashboard::Dese::Scraper
attr_reader :filepaths
def scrape_enrollments(filepath:)
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'PK', 'K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', 'SP', 'Total']
write_headers(filepath:, headers:)
run do |academic_year|
admin_data_item_id = ''
url = 'https://profiles.doe.mass.edu/statereport/enrollmentbygrade.aspx'
range = academic_year.range
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
'ctl00_ContentPlaceHolder1_ddYear' => range }
submit_id = 'btnViewReport'
calculation = ->(_headers, _items) { 'NA' }
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
end
end
def student_count(filepath:, dese_id:, year:)
@students ||= {}
if @students.count == 0
CSV.parse(File.read(filepath), headers: true).map do |row|
academic_year = row['Academic Year']
school_id = row['DESE ID'].to_i
total = row['Total'].gsub(',', '').to_i
@students[[school_id, academic_year]] = total
def scrape_enrollments(filepath:)
headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "School Name", "DESE ID",
"PK", "K", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "SP", "Total"]
write_headers(filepath:, headers:)
run do |academic_year|
admin_data_item_id = ""
url = "https://profiles.doe.mass.edu/statereport/enrollmentbygrade.aspx"
range = academic_year.range
selectors = { "ctl00_ContentPlaceHolder1_ddReportType" => "School",
"ctl00_ContentPlaceHolder1_ddYear" => range }
submit_id = "btnViewReport"
calculation = ->(_headers, _items) { "NA" }
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
end
end
@students[[dese_id, year]]
def student_count(filepath:, dese_id:, year:)
@students ||= {}
if @students.count == 0
CSV.parse(File.read(filepath), headers: true).map do |row|
academic_year = row["Academic Year"]
school_id = row["DESE ID"].to_i
total = row["Total"].gsub(",", "").to_i
@students[[school_id, academic_year]] = total
end
end
@students[[dese_id, year]]
end
end
end
end