Merge branch 'scrapers'

app/lib/seeder.rb

@@ -128,7 +128,6 @@ class Seeder
       end
 
       if row['Source'] == 'Admin Data' && row['Active admin & survey items'] == 'TRUE'
-        binding.break if row['Question/item (22-23)'].nil?
         admin_data_item = AdminDataItem.where(admin_data_item_id: data_item_id, scale:).first_or_create
         admin_data_item.watch_low_benchmark = watch_low if watch_low
         admin_data_item.growth_low_benchmark = growth_low if growth_low

app/services/dese/three_a_two.rb

@@ -61,6 +61,10 @@ module Dese
           fte_index = headers['FTE Count']
           num_of_guidance_counselors = items[fte_index].to_f
           dese_id = items[headers['School Code']].to_i
+          school = School.find_by_dese_id(dese_id)
+
+          return 'NA' unless school.present? && school.is_hs?
+
           num_of_students = student_count(filepath: filepaths[0], dese_id:, year: academic_year.range) || 0
           items << num_of_students
           benchmark = 250

app/services/dese/three_b_two.rb

@@ -101,14 +101,17 @@ module Dese
           non_white_student_percentage = (100 - white_number).to_f
           items.unshift(non_white_student_percentage)
 
-          non_white_teachers = teacher_count(filepath: filepaths[1], dese_id:, year: academic_year.range).to_f
-          items.unshift(non_white_teachers)
+          non_white_teacher_percentage = teacher_count(filepath: filepaths[1], dese_id:, year: academic_year.range).to_f
+          items.unshift(non_white_teacher_percentage)
+
+          floor = 5
+          benchmark = 0.25
 
-          return 1 if non_white_student_percentage.zero? && non_white_teachers < 6.7
+          return 1 if non_white_student_percentage.zero? && non_white_teacher_percentage < floor
 
-          if non_white_teachers >= 6.7
-            parity_index = non_white_teachers / non_white_student_percentage
-            likert_score = parity_index * 4 / 0.25
+          if non_white_teacher_percentage >= floor
+            parity_index = non_white_teacher_percentage / non_white_student_percentage
+            likert_score = parity_index * 4 / benchmark
           else
             likert_score = 1
           end

data/sqm_framework.csv

@@ -11,7 +11,7 @@ School Culture,2,"Measures the degree to which the school environment is safe, c
 School Culture,2,"Measures the degree to which the school environment is safe, caring, and academically-oriented.  It considers factors like bullying, student-teacher relationships, and student valuing of learning.","Measures the degree to which the school environment is safe, caring, and academically-oriented.",Safety,2A,Seeks to determine the degree to which school climate is a safe place for students to learn. It includes measures of student physical safety and student emotional safety. ,2.98,2.99,3.49,3.5,3.99,4,4.6,4.61,Student Emotional Safety,Draws on anonymous student and teacher reports about the nature of student relationships with each other.,2A-ii,2.78,2.79,3.29,3.3,3.79,3.8,4.5,4.51,Teachers,,TRUE,TRUE,How often are students bullied at school or online?,How often are students bullied at school or online?,How often are students bullied at school or online?,How often are students bullied at school or online?,How often are students bullied at school?,How often are students bullied at school?,How often are students bullied at school?,t-pvic-q1,X,,,#N/A,,,2.78,2.79,3.29,3.3,3.79,3.8,4.5,4.51
 School Culture,2,"Measures the degree to which the school environment is safe, caring, and academically-oriented.  It considers factors like bullying, student-teacher relationships, and student valuing of learning.","Measures the degree to which the school environment is safe, caring, and academically-oriented.",Safety,2A,Seeks to determine the degree to which school climate is a safe place for students to learn. It includes measures of student physical safety and student emotional safety. ,2.98,2.99,3.49,3.5,3.99,4,4.6,4.61,Student Emotional Safety,Draws on anonymous student and teacher reports about the nature of student relationships with each other.,2A-ii,2.78,2.79,3.29,3.3,3.79,3.8,4.5,4.51,Teachers,,TRUE,TRUE,How often are students bullied because of who they are?,How often are students bullied because of who they are?,How often are students bullied because of who they are?,How often are students bullied because of who they are?,How often are students bullied because of who they are?,How often are students bullied because of who they are?,How often are students bullied because of who they are?,t-pvic-q2,X,,,#N/A,,,2.78,2.79,3.29,3.3,3.79,3.8,4.5,4.51
 School Culture,2,"Measures the degree to which the school environment is safe, caring, and academically-oriented.  It considers factors like bullying, student-teacher relationships, and student valuing of learning.","Measures the degree to which the school environment is safe, caring, and academically-oriented.",Academic Orientation,2C,Seeks to determine the degree to which student academic experiences are challenging and meaningful to them. It includes measures of valuing of learning and academic challenge. ,2.23,2.24,2.74,2.75,3.24,3.25,4.5,4.51,Valuing of Learning,Draws on anonymous student reports about how important school is to them and how much they view themselves as learners. It also includes attendance rates.,2C-i,2.23,2.24,2.74,2.75,3.24,3.25,4.5,4.51,Admin Data,"MA DESE, 2019-20",TRUE,TRUE,Chronic absence rate,Chronic absence rate,Chronic absence rate,Chronic absence rate,Chronic absence rate,Chronic absence rate,Chronic absence rate,a-vale-i1,X,,,#N/A,10,number,2.98,2.99,3.49,3.5,3.99,4,4.8,4.81
-Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Facilities & Personnel,3A,Seeks to determine the sufficiency of a school's staffing and facilities. It includes measures of physical space & materials and content specialists & support staff.,2.455,2.465,2.965,2.975,3.465,3.475,4.5,4.51,Content Specialists & Support Staff,"Draws on anonymous student and teacher reports about the degree to which content specialists and support staff are available to help students with academic and non-academic needs. It also includes student-to-art-teacher, student-to-counselor, and student-to-specialist ratios.",3A-ii,2.68,2.69,3.19,3.2,3.69,3.7,4.5,4.51,Admin Data,American School Counselor Association (2005),TRUE,TRUE,Student to guidance counselor ratio,Student to guidance counselor ratio,Student to guidance counselor ratio,Student to guidance counselor ratio,Student to guidance counselor ratio,Student to guidance counselor ratio,Student to guidance counselor ratio,a-sust-i1,X,,,#N/A,250,ratio,2.98,2.99,3.49,3.5,3.99,4,4.9,4.91
+Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Facilities & Personnel,3A,Seeks to determine the sufficiency of a school's staffing and facilities. It includes measures of physical space & materials and content specialists & support staff.,2.455,2.465,2.965,2.975,3.465,3.475,4.5,4.51,Content Specialists & Support Staff,"Draws on anonymous student and teacher reports about the degree to which content specialists and support staff are available to help students with academic and non-academic needs. It also includes student-to-art-teacher, student-to-counselor, and student-to-specialist ratios.",3A-ii,2.68,2.69,3.19,3.2,3.69,3.7,4.5,4.51,Admin Data,American School Counselor Association (2005),TRUE,TRUE,Student to guidance counselor ratio (HS only),Student to guidance counselor ratio (HS only),Student to guidance counselor ratio (HS only),Student to guidance counselor ratio (HS only),Student to guidance counselor ratio (HS only),Student to guidance counselor ratio (HS only),Student to guidance counselor ratio (HS only),a-sust-i1,X,X,,#N/A,250,ratio,2.98,2.99,3.49,3.5,3.99,4,4.9,4.91
 Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Facilities & Personnel,3A,Seeks to determine the sufficiency of a school's staffing and facilities. It includes measures of physical space & materials and content specialists & support staff.,2.455,2.465,2.965,2.975,3.465,3.475,4.5,4.51,Content Specialists & Support Staff,"Draws on anonymous student and teacher reports about the degree to which content specialists and support staff are available to help students with academic and non-academic needs. It also includes student-to-art-teacher, student-to-counselor, and student-to-specialist ratios.",3A-ii,2.68,2.69,3.19,3.2,3.69,3.7,4.5,4.51,Admin Data,American School Counselor Association (2005),TRUE,TRUE,Student to mental health counselor ratio,Student to mental health counselor ratio,Student to mental health counselor ratio,Student to mental health counselor ratio,Student to mental health counselor ratio,Student to mental health counselor ratio,Student to mental health counselor ratio,a-sust-i2,X,,,#N/A,250,ratio,2.98,2.99,3.49,3.5,3.99,4,4.9,4.91
 Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Facilities & Personnel,3A,Seeks to determine the sufficiency of a school's staffing and facilities. It includes measures of physical space & materials and content specialists & support staff.,2.455,2.465,2.965,2.975,3.465,3.475,4.5,4.51,Content Specialists & Support Staff,"Draws on anonymous student and teacher reports about the degree to which content specialists and support staff are available to help students with academic and non-academic needs. It also includes student-to-art-teacher, student-to-counselor, and student-to-specialist ratios.",3A-ii,2.68,2.69,3.19,3.2,3.69,3.7,4.5,4.51,Admin Data,"NCES, 2007-08",TRUE,TRUE,Student to instructional support staff ratio,Student to instructional support staff ratio,Student to instructional support staff ratio,Student to instructional support staff ratio,Student to instructional support staff ratio,Student to instructional support staff ratio,Student to instructional support staff ratio,a-sust-i3,X,,,#N/A,43.4,ratio,2.98,2.99,3.49,3.5,3.99,4,4.9,4.91
 Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Facilities & Personnel,3A,Seeks to determine the sufficiency of a school's staffing and facilities. It includes measures of physical space & materials and content specialists & support staff.,2.455,2.465,2.965,2.975,3.465,3.475,4.5,4.51,Content Specialists & Support Staff,"Draws on anonymous student and teacher reports about the degree to which content specialists and support staff are available to help students with academic and non-academic needs. It also includes student-to-art-teacher, student-to-counselor, and student-to-specialist ratios.",3A-ii,2.68,2.69,3.19,3.2,3.69,3.7,4.5,4.51,Admin Data,Constructed by MCIEA staff,TRUE,TRUE,Student to art teacher ratio,Student to art teacher ratio,Student to art teacher ratio,Student to art teacher ratio,Student to art teacher ratio,Student to art teacher ratio,Student to art teacher ratio,a-sust-i4,X,,,#N/A,500,ratio,2.98,2.99,3.49,3.5,3.99,4,4.9,4.91

spec/services/dese/three_a_two_spec.rb

@@ -1,8 +1,10 @@
 require 'rails_helper'
 require 'fileutils'
 require 'csv'
+require "#{Rails.root}/app/lib/seeder"
 
 RSpec.describe Dese::ThreeATwo do
+  let(:seeder) { Seeder.new }
   let(:academic_years) do
     [
       create(:academic_year, range: '2021-22'),
@@ -29,8 +31,9 @@ RSpec.describe Dese::ThreeATwo do
     academic_years
   end
 
-  xcontext 'Creating a new Scraper' do
+  xcontext '#run_all' do
     it 'creates a csv file with the scraped data' do
+      seeder.seed_districts_and_schools sample_districts_and_schools_csv
       Dese::ThreeATwo.new(filepaths:).run_all
       expect(enrollment_filepath).to exist
       expect(i1_filepath).to exist
@@ -71,11 +74,11 @@ RSpec.describe Dese::ThreeATwo do
       results = CSV.parse(File.read(i1_filepath), headers: true).map do |row|
         next unless row['Admin Data Item'] == 'a-sust-i1' && row['Academic Year'] == '2021-22'
 
-        row['Likert Score'].to_f
+        row['Likert Score'].to_f unless row['Likert Score'] == 'NA'
       end.flatten.compact
 
-      expect(results.take(20)).to eq [4.0, 1.0, 4.87, 1.0, 1.0, 1.0, 4.6, 5.0, 1.0, 1.89, 1.0, 1.55, 1.0, 1.0, 2.11,
-                                      4.7, 1.0, 1.0, 5.0, 1.0]
+      expect(results.take(20)).to eq [4.96, 1.0, 5.0, 1.0, 5.0, 4.21, 4.45, 4.15, 3.17, 5.0, 4.48, 3.62, 5.0, 1.0, 1.0,
+                                      4.44, 3.8, 1.0, 1.0, 1.0]
     end
 
     it 'has the right likert score results for a-sust-i2' do
@@ -128,4 +131,10 @@ RSpec.describe Dese::ThreeATwo do
       three_a_two.browser.close
     end
   end
+
+  private
+
+  def sample_districts_and_schools_csv
+    Rails.root.join('data', 'master_list_of_schools_and_districts.csv')
+  end
 end