sqm-dashboards/app/services/dese/three_b_one.rb

require 'watir'
require 'csv'

module Dese
  class ThreeBOne
    include Dese::Scraper
    include Dese::Enrollments
    attr_reader :filepaths

    def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '3B_1_masscore.csv'),
                               Rails.root.join('data', 'admin_data', 'dese', '3B_1_advcoursecomprate.csv'),
                               Rails.root.join('data', 'admin_data', 'dese', '3B_1_ap.csv'),
                               Rails.root.join('data', 'admin_data', 'dese', '3B_1_student_courses_ratio.csv')])

      @filepaths = filepaths
    end

    def run_all
      filepath = filepaths[0]
      headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
                 '# Graduated', '# Completed MassCore', '% Completed MassCore']
      write_headers(filepath:, headers:)

      run_a_curv_i1(filepath:)

      filepath = filepaths[1]
      headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
                 '# Grade 11 and 12 Students', '# Students Completing Advanced', '% Students Completing Advanced',
                 '% ELA', '% Math', '% Science and Technology', '% Computer and Information Science',
                 '% History and Social Sciences', '% Arts', '% All Other Subjects', '% All Other Subjects']
      write_headers(filepath:, headers:)
      run_a_curv_i2(filepath:)

      filepath = filepaths[2]
      headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
                 'Tests Taken', 'Score=1', 'Score=2', 'Score=3', 'Score=4', 'Score=5', '% Score 1-2', '% Score 3-5']
      write_headers(filepath:, headers:)
      run_a_curv_i3(filepath:)

      filepath = filepaths[3]
      headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
                 'Total # of Classes', 'Average Class Size', 'Number of Students', 'Female %', 'Male %', 'English Language Learner %', 'Students with Disabilities %', 'Low Income %', 'Number of Students']
      write_headers(filepath:, headers:)
      run_a_curv_i5(filepath:)
      browser.close
    end

    def run_a_curv_i1(filepath:)
      run do |academic_year|
        url = 'https://profiles.doe.mass.edu/statereport/masscore.aspx'
        range = academic_year.range
        selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
                      'ctl00_ContentPlaceHolder1_ddYear' => range }
        submit_id = 'btnViewReport'
        calculation = lambda { |headers, items|
          completed_index = headers['% Completed MassCore']
          percent_completed = items[completed_index].to_f
          benchmark = 90
          percent_completed * 4 / benchmark if completed_index.present? && !items[completed_index] != ''
        }
        admin_data_item_id = 'a-curv-i1'
        Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
      end
    end

    def run_a_curv_i2(filepath:)
      run do |academic_year|
        url = 'https://profiles.doe.mass.edu/statereport/advcoursecomprate.aspx'
        range = "#{academic_year.range.split('-')[1].to_i + 2000}"
        selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
                      'ctl00_ContentPlaceHolder1_ddYear' => range }
        submit_id = 'btnViewReport'
        calculation = lambda { |headers, items|
          completed_index = headers['% Students Completing Advanced']
          percent_completed = items[completed_index].to_f
          benchmark = 30
          percent_completed * 4 / benchmark if completed_index.present? && !items[completed_index] != ''
        }
        admin_data_item_id = 'a-curv-i2'
        Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
      end
    end

    def run_a_curv_i3(filepath:)
      run do |academic_year|
        url = 'https://profiles.doe.mass.edu/statereport/ap.aspx'
        range = academic_year.range
        selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
                      'ctl00_ContentPlaceHolder1_ddYear' => range }
        submit_id = 'ctl00_ContentPlaceHolder1_btnViewReport'
        calculation = lambda { |headers, items|
          completed_index = headers['% Score 3-5']
          percent_score = items[completed_index].to_f
          benchmark = 20
          percent_score * 4 / benchmark if completed_index.present? && !items[completed_index] != ''
        }
        admin_data_item_id = 'a-curv-i3'
        Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
      end
    end

    def run_a_curv_i5(filepath:)
      run do |academic_year|
        url = 'https://profiles.doe.mass.edu/statereport/classsizebygenderpopulation.aspx'
        range = academic_year.range
        selectors = { 'ctl00_ContentPlaceHolder1_ddReportType' => 'School',
                      'ctl00_ContentPlaceHolder1_ddYear' => range }
        submit_id = 'btnViewReport'
        calculation = lambda { |headers, items|
          row = headers.keys.zip(items).to_h
          dese_id = row['School Code'].to_i
          is_hs = (row['School Name'] in /High School/i)
          school = School.find_by(dese_id:)
          is_hs = school.is_hs if school.present?
          next 'NA' unless is_hs

          num_of_classes = row['Total # of Classes'].delete(',').to_f
          num_of_students = student_count(filepath: Rails.root.join('data', 'admin_data', 'dese', 'enrollments.csv'),
                                          dese_id:, year: academic_year.range) || 0
          items << num_of_students
          actual = num_of_students / num_of_classes
          benchmark = 5
          ((benchmark - actual) + benchmark) * 4 / benchmark if num_of_classes.present? && num_of_students.present?
        }
        admin_data_item_id = 'a-curv-i5'
        Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
      end
    end
  end
end