gabehf/Dashboard
1
0
Fork 0
mirror of https://github.com/edcommonwealth/Dashboard.git synced 2026-03-17 09:46:25 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: get admin data scraper running

Browse source
This commit is contained in:
Nelson Jovel 2024-02-07 11:47:25 -08:00
parent 1810ee0074
commit a142afe022
102 changed files with 41933 additions and 8756 deletions
Download patch file Download diff file Expand all files Collapse all files

69
app/services/dashboard/dese/four_a_one.rb
View file

@ -1,43 +1,46 @@
require 'watir'
require 'csv'
require "watir"
require "csv"
module Dese
class FourAOne
include Dese::Scraper
attr_reader :filepaths
module Dashboard
module Dese
class FourAOne
include Dese::Scraper
attr_reader :filepaths
def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '4A_1_grade_nine_course_pass.csv')])
@filepaths = filepaths
end
def initialize(filepaths: [Dashboard::Engine.root.join("data", "admin_data", "dese",
"4A_1_grade_nine_course_pass.csv")])
@filepaths = filepaths
end
def run_all
filepath = filepaths[0]
headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
'# Grade Nine Students', '# Passing All Courses', '% Passing All Courses']
write_headers(filepath:, headers:)
def run_all
filepath = filepaths[0]
headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "School Name", "DESE ID",
"# Grade Nine Students", "# Passing All Courses", "% Passing All Courses"]
write_headers(filepath:, headers:)
run_a_ovpe_i1(filepath:)
run_a_ovpe_i1(filepath:)
browser.close
end
browser.close
end
def run_a_ovpe_i1(filepath:)
run do |academic_year|
admin_data_item_id = 'a-ovpe-i1'
url = 'https://profiles.doe.mass.edu/statereport/gradeninecoursepass.aspx'
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
'ctl00_ContentPlaceHolder1_ddYear' => range }
submit_id = 'btnViewReport'
calculation = lambda { |headers, items|
passing_index = headers['% Passing All Courses']
return 'NA' if items[passing_index] == '' || items[passing_index].strip == '.0'
def run_a_ovpe_i1(filepath:)
run do |academic_year|
admin_data_item_id = "a-ovpe-i1"
url = "https://profiles.doe.mass.edu/statereport/gradeninecoursepass.aspx"
range = "#{academic_year.range.split('-')[1].to_i + 2000}"
selectors = { "ctl00_ContentPlaceHolder1_ddReportType" => "School",
"ctl00_ContentPlaceHolder1_ddYear" => range }
submit_id = "btnViewReport"
calculation = lambda { |headers, items|
passing_index = headers["% Passing All Courses"]
return "NA" if items[passing_index] == "" || items[passing_index].strip == ".0"
percent_passing = items[passing_index].to_f
benchmark = 95
percent_passing * 4 / benchmark if passing_index.present?
}
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
percent_passing = items[passing_index].to_f
benchmark = 95
percent_passing * 4 / benchmark if passing_index.present?
}
Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
end
end
end
end