feat: pull down new admin data. Fix problems with some scrapers. For 5D-ii make sure the likert score is 1 if the nurse count is 0. Prevent devision by zero returning infinity. For 3A-ii make sure not to perform division by zero. Likert score will be 1 if the number of guidance couselors, psychologists, or paraprofessionals is zero.

9 months ago · e022af0161
parent ff5d496bec
commit e022af0161
10 changed files with 45432 additions and 41896 deletions
--- a/app/services/dese/enrollments.rb
+++ b/app/services/dese/enrollments.rb
@ -6,7 +6,7 @@ module Dese
    attr_reader :filepaths

    def scrape_enrollments(filepath:)
-      headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
+      headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'School Code',
                 'PK', 'K', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', 'SP', 'Total']
      write_headers(filepath:, headers:)
      run do |academic_year|
--- a/app/services/dese/five_d_two.rb
+++ b/app/services/dese/five_d_two.rb
@ -17,7 +17,7 @@ module Dese

      filepath = filepaths[1]
      headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
-                 '<26 yrs (# )', '26-32 yrs (#)', '33-40 yrs (#)', '41-48 yrs (#)', '49-56 yrs (#)', '57-64 yrs (#)', 'Over 64 yrs (#)', 'FTE Count']
+                 '<26 yrs (# )', '26-32 yrs (#)', '33-40 yrs (#)', '41-48 yrs (#)', '49-56 yrs (#)', '57-64 yrs (#)', 'Over 64 yrs (#)', 'FTE Count', 'Student Count', 'Student to Nurse Ratio']
      write_headers(filepath:, headers:)

      run_a_phya_i1(filepath:)
@ -45,8 +45,12 @@ module Dese
          dese_id = items[headers['School Code']].to_i
          num_of_students = student_count(filepath: filepaths[0], dese_id:, year: academic_year.range) || 0
          items << num_of_students
-          items << (num_of_students / nurse_count)
-          ((benchmark - (num_of_students / nurse_count)) + benchmark) * 4 / benchmark
+
+          return 0 if nurse_count == 0
+
+          student_to_nurse_ratio = num_of_students / nurse_count
+          items << student_to_nurse_ratio
+          ((benchmark - student_to_nurse_ratio) + benchmark) * 4 / benchmark
        }
        Prerequisites.new(filepath, url, selectors, submit_id, admin_data_item_id, calculation)
      end
--- a/app/services/dese/three_a_two.rb
+++ b/app/services/dese/three_a_two.rb
@ -67,7 +67,7 @@ module Dese
          num_of_students = student_count(filepath: filepaths[0], dese_id:, year: academic_year.range) || 0
          items << num_of_students
          benchmark = 250
-          if fte_index.present? && !items[fte_index] != ""
+          if fte_index.present? && num_of_guidance_counselors != 0
            result = ((benchmark - (num_of_students / num_of_guidance_counselors)) + benchmark) * 4 / benchmark
          end
          items << (num_of_students / num_of_guidance_counselors)
@ -93,10 +93,9 @@ module Dese
          num_of_students = student_count(filepath: filepaths[0], dese_id:, year: academic_year.range) || 0
          items << num_of_students
          benchmark = 250
-          if fte_index.present? && !items[fte_index] != ""
+          if fte_index.present? && num_of_psychologists != 0
            result = ((benchmark - (num_of_students / num_of_psychologists)) + benchmark) * 4 / benchmark
          end
-
          items << (num_of_students / num_of_psychologists)
          result
        }
@ -120,7 +119,7 @@ module Dese
          num_of_students = student_count(filepath: filepaths[0], dese_id:, year: academic_year.range) || 0
          items << num_of_students
          benchmark = 43.4
-          if fte_index.present? && !items[fte_index] != ""
+          if fte_index.present? && num_of_paraprofessionals != 0
            result = ((benchmark - (num_of_students / num_of_paraprofessionals)) + benchmark) * 4 / benchmark
          end

@ -148,7 +147,7 @@ module Dese
          num_of_students = student_count(filepath: filepaths[0], dese_id:, year: academic_year.range) || 0
          items << num_of_students
          benchmark = 500
-          if num_of_art_teachers.present?
+          if num_of_art_teachers.present? && num_of_art_teachers != 0
            result = ((benchmark - (num_of_students / num_of_art_teachers)) + benchmark) * 4 / benchmark
          end

--- a/app/services/dese/three_b_two.rb
+++ b/app/services/dese/three_b_two.rb
@ -21,10 +21,10 @@ module Dese
      run_teacher_demographics(filepath:)

      filepath = filepaths[1]
-      headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "Non-White Teachers %", "Non-White Students %", "School Name", "DESE ID",
+      headers = ["Raw likert calculation", "Likert Score", "Admin Data Item", "Academic Year", "School Name", "DESE ID",
                 "African American", "Asian", "Hispanic", "White", "Native American",
                 "Native Hawaiian or Pacific Islander", "Multi-Race or Non-Hispanic", "Males",
-                 "Females", "Non-Binary", "Students of color (%)"]
+                 "Females", "Non-Binary", "Students of color (%)", "Non-White Teachers %", "Non-White Students %"]
      write_headers(filepath:, headers:)
      run_student_demographics(filepath:)

@ -101,12 +101,12 @@ module Dese
          white_number = items[white_index].to_f
          dese_id = items[headers["School Code"]].to_i
          non_white_student_percentage = (100 - white_number).to_f
-          items.unshift(non_white_student_percentage)
+          items << non_white_student_percentage
          count_of_teachers = teacher_count(filepath: filepaths[0], dese_id:, year: academic_year.range)
          return "NA" if count_of_teachers == "NA"

          non_white_teacher_percentage = count_of_teachers.to_f
-          items.unshift(non_white_teacher_percentage)
+          items << non_white_teacher_percentage

          floor = 5
          benchmark = 0.25
--- a/data/admin_data/dese/3A_2_age_staffing.csv
+++ b/data/admin_data/dese/3A_2_age_staffing.csv
--- a/data/admin_data/dese/3A_2_grade_subject_staffing.csv
+++ b/data/admin_data/dese/3A_2_grade_subject_staffing.csv
--- a/data/admin_data/dese/3B_2_student_by_race_and_gender.csv
+++ b/data/admin_data/dese/3B_2_student_by_race_and_gender.csv
--- a/data/admin_data/dese/4B_2_retention.csv
+++ b/data/admin_data/dese/4B_2_retention.csv
--- a/data/admin_data/dese/5D_2_age_staffing.csv
+++ b/data/admin_data/dese/5D_2_age_staffing.csv
--- a/data/admin_data/dese/enrollments.csv
+++ b/data/admin_data/dese/enrollments.csv
@ -1,4 +1,4 @@
-Raw likert calculation,Likert Score,Admin Data Item,Academic Year,School Name,DESE ID,PK,K,1,2,3,4,5,6,7,8,9,10,11,12,SP,Total
+Raw likert calculation,Likert Score,Admin Data Item,Academic Year,School Name,School Code,PK,K,1,2,3,4,5,6,7,8,9,10,11,12,SP,Total
 NA,NA,"",2024-25,Abby Kelley Foster Charter Public (District) - Abby Kelley Foster Charter Public School,04450105,       0,     125,     123,     125,     127,     130,     128,     128,     111,     108,      80,      85,      80,      76,       0,"   1,426"
 NA,NA,"",2024-25,Abington - Abington Early Education Program,00010001,      82,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,      82
 NA,NA,"",2024-25,Abington - Abington High,00010505,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,     127,     138,     131,     128,       6,     530