require "watir" require "csv" module Dashboard module Dese class ThreeBOne include Dese::Scraper include Dese::Enrollments attr_reader :filepaths def initialize(filepaths: [Dashboard::Engine.root.join("data", "admin_data", "dese", "3B_1_masscore.csv"), Dashboard::Engine.root.join("data", "admin_data", "dese", "3B_1_advcoursecomprate.csv"), Dashboard::Engine.root.join("data", "admin_data", "dese", "3B_1_ap.csv"), Dashboard::Engine.root.join("data", "admin_data", "dese", "3B_1_student_courses_ratio.csv")]) @filepaths = filepaths end def run_all filepath = filepaths[0] headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "School Name", "DESE ID", "# Graduated", "# Completed MassCore", "% Completed MassCore"] write_headers(filepath:, headers:) run_a_curv_i1(filepath:) filepath = filepaths[1] headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "School Name", "DESE ID", "# Grade 11 and 12 Students", "# Students Completing Advanced", "% Students Completing Advanced", "% ELA", "% Math", "% Science and Technology", "% Computer and Information Science", "% History and Social Sciences", "% Arts", "% All Other Subjects", "% All Other Subjects"] write_headers(filepath:, headers:) run_a_curv_i2(filepath:) filepath = filepaths[2] headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "School Name", "DESE ID", "Tests Taken", "Score=1", "Score=2", "Score=3", "Score=4", "Score=5", "% Score 1-2", "% Score 3-5"] write_headers(filepath:, headers:) run_a_curv_i3(filepath:) filepath = filepaths[3] headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "School Name", "DESE ID", "Total # of Classes", "Average Class Size", "Number of Students", "Female %", "Male %", "English Language Learner %", "Students with Disabilities %", "Low Income %", "Number of Students"] write_headers(filepath:, headers:) run_a_curv_i5(filepath:) browser.close end def run_a_curv_i1(filepath:) run do |academic_year| url = "https://profiles.doe.mass.edu/statereport/masscore.aspx" range = academic_year.range selectors = { "ctl00_ContentPlaceHolder1_ddReportType" => "School", "ctl00_ContentPlaceHolder1_ddYear" => range } submit_id = "btnViewReport" calculation = lambda { |headers, items| completed_index = headers["% Completed MassCore"] percent_completed = items[completed_index].to_f benchmark = 90 percent_completed * 4 / benchmark if completed_index.present? && !items[completed_index] != "" } admin_data_item_id = "a-curv-i1" Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation) end end def run_a_curv_i2(filepath:) run do |academic_year| url = "https://profiles.doe.mass.edu/statereport/advcoursecomprate.aspx" range = "#{academic_year.range.split('-')[1].to_i + 2000}" selectors = { "ctl00_ContentPlaceHolder1_ddReportType" => "School", "ctl00_ContentPlaceHolder1_ddYear" => range } submit_id = "btnViewReport" calculation = lambda { |headers, items| completed_index = headers["% Students Completing Advanced"] percent_completed = items[completed_index].to_f benchmark = 30 percent_completed * 4 / benchmark if completed_index.present? && !items[completed_index] != "" } admin_data_item_id = "a-curv-i2" Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation) end end def run_a_curv_i3(filepath:) run do |academic_year| url = "https://profiles.doe.mass.edu/statereport/ap.aspx" range = academic_year.range selectors = { "ctl00_ContentPlaceHolder1_ddReportType" => "School", "ctl00_ContentPlaceHolder1_ddYear" => range } submit_id = "ctl00_ContentPlaceHolder1_btnViewReport" calculation = lambda { |headers, items| completed_index = headers["% Score 3-5"] percent_score = items[completed_index].to_f benchmark = 20 percent_score * 4 / benchmark if completed_index.present? && !items[completed_index] != "" } admin_data_item_id = "a-curv-i3" Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation) end end def run_a_curv_i5(filepath:) run do |academic_year| url = "https://profiles.doe.mass.edu/statereport/classsizebygenderpopulation.aspx" range = academic_year.range selectors = { "ctl00_ContentPlaceHolder1_ddReportType" => "School", "ctl00_ContentPlaceHolder1_ddYear" => range } submit_id = "btnViewReport" calculation = lambda { |headers, items| row = headers.keys.zip(items).to_h dese_id = row["School Code"].to_i is_hs = (row["School Name"] in /High School/i) school = School.find_by(dese_id:) is_hs = school.is_hs if school.present? next "NA" unless is_hs num_of_classes = row["Total # of Classes"].delete(",").to_f num_of_students = student_count(filepath: Dashboard::Engine.root.join("data", "admin_data", "dese", "enrollments.csv"), dese_id:, year: academic_year.range) || 0 items << num_of_students actual = num_of_students / num_of_classes benchmark = 5 ((benchmark - actual) + benchmark) * 4 / benchmark if num_of_classes.present? && num_of_students.present? } admin_data_item_id = "a-curv-i5" Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation) end end end end end