sqm-dashboards/app/services/dese/three_b_one.rb

require 'watir'
require 'csv'

module Dese
  class ThreeBOne
    include Dese::Scraper
    include Dese::Enrollments
    attr_reader :filepaths

    def initialize(filepaths: [Rails.root.join("data", "admin_data", "dese", "3B_1_masscore.csv"),
                               Rails.root.join("data", "admin_data", "dese", "3B_1_advcoursecomprate.csv"),
                               Rails.root.join("data", "admin_data", "dese", "3B_1_ap.csv"),
                               Rails.root.join("data", "admin_data", "dese", "3B_1_course_ratio.csv")])
      @filepaths = filepaths
    end

    def run_all
      filepath = filepaths[0]
      headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "School Name", "DESE ID",
                 "# Graduated", "# Completed MassCore", "% Completed MassCore"]
      write_headers(filepath:, headers:)

      run_a_curv_i1(filepath:)

      filepath = filepaths[1]
      headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "School Name", "DESE ID",
                 "# Grade 11 and 12 Students", "# Students Completing Advanced", "% Students Completing Advanced",
                 "% ELA", "% Math", "% Science and Technology", "% Computer and Information Science",
                 "% History and Social Sciences", "% Arts", "% All Other Subjects", "% All Other Subjects"]
      write_headers(filepath:, headers:)
      run_a_curv_i2(filepath:)

      filepath = filepaths[2]
      headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "School Name", "DESE ID",
                 "Tests Taken", "Score=1", "Score=2", "Score=3", "Score=4", "Score=5", "% Score 1-2", "% Score 3-5"]
      write_headers(filepath:, headers:)
      run_a_curv_i3(filepath:)

      filepath = filepaths[3]
      headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "School Name", "DESE ID",
                 "Total # of Classes", "Average Class Size", "Number of Students", "Female %", "Male %", "English Language Learner %", "Students with Disabilities %", "Low Income %"]
      write_headers(filepath:, headers:)
      run_a_curv_i5(filepath:)

      browser.close
    end

    def run_a_curv_i1(filepath:)
      run do |academic_year|
        url = "https://profiles.doe.mass.edu/statereport/masscore.aspx"
        range = academic_year.range
        selectors = { "ctl00_ContentPlaceHolder1_ddReportType" => "School",
                      "ctl00_ContentPlaceHolder1_ddYear" => range }
        submit_id = "btnViewReport"
        calculation = lambda { |headers, items|
          completed_index = headers["% Completed MassCore"]
          percent_completed = items[completed_index].to_f
          benchmark = 90
          percent_completed * 4 / benchmark if completed_index.present? && !items[completed_index] != ""
        }
        admin_data_item_id = "a-curv-i1"
        Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
      end
    end

    def run_a_curv_i2(filepath:)
      run do |academic_year|
        url = "https://profiles.doe.mass.edu/statereport/advcoursecomprate.aspx"
        range = "#{academic_year.range.split('-')[1].to_i + 2000}"
        selectors = { "ctl00_ContentPlaceHolder1_ddReportType" => "School",
                      "ctl00_ContentPlaceHolder1_ddYear" => range }
        submit_id = "btnViewReport"
        calculation = lambda { |headers, items|
          completed_index = headers["% Students Completing Advanced"]
          percent_completed = items[completed_index].to_f
          benchmark = 30
          percent_completed * 4 / benchmark if completed_index.present? && !items[completed_index] != ""
        }
        admin_data_item_id = "a-curv-i2"
        Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
      end
    end

    def run_a_curv_i3(filepath:)
      run do |academic_year|
        url = "https://profiles.doe.mass.edu/statereport/ap.aspx"
        range = academic_year.range
        selectors = { "ctl00_ContentPlaceHolder1_ddReportType" => "School",
                      "ctl00_ContentPlaceHolder1_ddYear" => range }
        submit_id = "ctl00_ContentPlaceHolder1_btnViewReport"
        calculation = lambda { |headers, items|
          completed_index = headers["% Score 3-5"]
          percent_score = items[completed_index].to_f
          benchmark = 20
          percent_score * 4 / benchmark if completed_index.present? && !items[completed_index] != ""
        }
        admin_data_item_id = "a-curv-i3"
        Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
      end
    end

    def run_a_curv_i5(filepath:)
      run do |academic_year|
        url = 'https://profiles.doe.mass.edu/statereport/classsizebygenderpopulation.aspx'
        range = academic_year.range
        selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
                      'ctl00_ContentPlaceHolder1_ddYear' => range }
        submit_id = 'btnViewReport'
        calculation = lambda { |headers, items|
          row = headers.keys.zip(items).to_h
          dese_id = row['School Code'].to_i
          is_hs = (row['School Name'] in /High School/i)
          school = School.find_by(dese_id:)
          is_hs = school.is_hs if school.present?
          next 'NA' unless is_hs

          num_of_classes = row['Total # of Classes'].delete(',').to_f
          num_of_students = student_count(filepath: Rails.root.join('data', 'admin_data', 'dese', 'enrollments.csv'),
                                          dese_id:, year: academic_year.range) || 0
          items << num_of_students
          actual = num_of_students / num_of_classes
          benchmark = 5
          ((benchmark - actual) + benchmark) * 4 / benchmark if num_of_classes.present? && num_of_students.present?
        }
        admin_data_item_id = 'a-curv-i5'
        Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
      end
    end
  end
end