Merge branch 'rpp-response-rate' to bring in changes to test files

app/services/admin_data_loader.rb

@@ -43,6 +43,7 @@ class AdminDataLoader
   end
 
   def self.create_admin_data_value(row:, score:)
+    # byebug unless %w[a-vale-i1 a-sust-i3].include? admin_data_item(row:)
     AdminDataValue.create!(likert_score: score,
                            academic_year: AcademicYear.find_by_range(ay(row:)),
                            school: School.find_by_dese_id(dese_id(row:).to_i),

app/services/dese/one_a_three.rb

@@ -41,7 +41,7 @@ module Dese
     def run_a_pcom_i3
       filepath = filepaths[1]
       headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
-                 'African American (%)', 'Asian (%)', 'Hispanic (%)', 'White (%)', 'Native Hawaiian, Pacific Islander (%)',
+                 'African American (%)', 'Asian (%)', 'Hispanic (%)', 'White (%)', 'Native Amertican (%)', 'Native Hawaiian, Pacific Islander (%)',
                  'Multi-Race,Non-Hispanic (%)', 'Females (%)', 'Males (%)', 'FTE Count']
       write_headers(filepath:, headers:)
 

app/services/dese/scraper.rb

@@ -1,6 +1,6 @@
 module Dese
   module Scraper
-    DELAY = 20
+    DELAY = 20 # The dese site will block you if you hit it too many times in a short period of time
 
     Prerequisites = Struct.new('Prerequisites', :filepath, :url, :selectors, :submit_id, :admin_data_item_id,
                                :calculation)

app/services/enrollment_loader.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+class EnrollmentLoader
+  def self.load_data(filepath:)
+    schools = []
+    enrollments = []
+    CSV.parse(File.read(filepath), headers: true) do |row|
+      row = EnrollmentRowValues.new(row:)
+
+      next unless row.school.present? && row.academic_year.present?
+
+      schools << row.school
+
+      enrollments << create_enrollment_entry(row:)
+    end
+
+    # It's possible that instead of updating all columns on duplicate key, we could just update the student columns and leave total_teachers alone. Right now enrollment data loads before staffing data so it works correctly.
+    Respondent.import enrollments, batch_size: 1000,
+                                   on_duplicate_key_update: %i[pk k one two three four five six seven eight nine ten eleven twelve total_students]
+
+    Respondent.where.not(school: schools).destroy_all
+  end
+
+  private
+
+  def self.create_enrollment_entry(row:)
+    respondent = Respondent.find_or_initialize_by(school: row.school, academic_year: row.academic_year)
+    respondent.pk = row.pk
+    respondent.k = row.k
+    respondent.one = row.one
+    respondent.two = row.two
+    respondent.three = row.three
+    respondent.four = row.four
+    respondent.five = row.five
+    respondent.six = row.six
+    respondent.seven = row.seven
+    respondent.eight = row.eight
+    respondent.nine = row.nine
+    respondent.ten = row.ten
+    respondent.eleven = row.eleven
+    respondent.twelve = row.twelve
+    respondent.total_students = row.total_students
+    respondent
+  end
+
+  private_class_method :create_enrollment_entry
+end
+
+class EnrollmentRowValues
+  attr_reader :row
+
+  def initialize(row:)
+    @row = row
+  end
+
+  def school
+    @school ||= begin
+      dese_id = row['DESE ID'].try(:strip).to_i
+      School.find_by_dese_id(dese_id)
+    end
+  end
+
+  def academic_year
+    @academic_year ||= begin
+      year = row['Academic Year']
+      AcademicYear.find_by_range(year)
+    end
+  end
+
+  def pk
+    row['PK'] || row['pk']
+  end
+
+  def k
+    row['K'] || row['k']
+  end
+
+  def one
+    row['1']
+  end
+
+  def two
+    row['2']
+  end
+
+  def three
+    row['3']
+  end
+
+  def four
+    row['4']
+  end
+
+  def five
+    row['5']
+  end
+
+  def six
+    row['6']
+  end
+
+  def seven
+    row['7']
+  end
+
+  def eight
+    row['8']
+  end
+
+  def nine
+    row['9']
+  end
+
+  def ten
+    row['10']
+  end
+
+  def eleven
+    row['11']
+  end
+
+  def twelve
+    row['12']
+  end
+
+  def total_students
+    row['Total'].gsub(',', '').to_i
+  end
+end

app/services/response_rate_loader.rb

@@ -4,11 +4,7 @@ class ResponseRateLoader
   def self.reset(schools: School.all, academic_years: AcademicYear.all, subcategories: Subcategory.all)
     subcategories.each do |subcategory|
       schools.each do |school|
-        next if test_env? && (school != milford)
-
         academic_years.each do |academic_year|
-          next if test_env? && (academic_year != test_year)
-
           process_response_rate(subcategory:, school:, academic_year:)
         end
       end
@@ -17,18 +13,6 @@ class ResponseRateLoader
 
   private
 
-  def self.milford
-    School.find_by_slug 'milford-high-school'
-  end
-
-  def self.test_year
-    AcademicYear.find_by_range '2020-21'
-  end
-
-  def self.rails_env
-    @rails_env ||= ENV['RAILS_ENV']
-  end
-
   def self.process_response_rate(subcategory:, school:, academic_year:)
     student = StudentResponseRateCalculator.new(subcategory:, school:, academic_year:)
     teacher = TeacherResponseRateCalculator.new(subcategory:, school:, academic_year:)
@@ -41,13 +25,5 @@ class ResponseRateLoader
                           meets_teacher_threshold: teacher.meets_teacher_threshold?)
   end
 
-  def self.test_env?
-    rails_env == 'test'
-  end
-
-  private_class_method :milford
-  private_class_method :test_year
-  private_class_method :rails_env
   private_class_method :process_response_rate
-  private_class_method :test_env?
 end

app/services/staffing_loader.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# TODO
+require 'csv'
+
+class StaffingLoader
+  def self.load_data(filepath:)
+    schools = []
+    respondents = []
+    CSV.parse(File.read(filepath), headers: true) do |row|
+      row = StaffingRowValues.new(row:)
+      next unless row.school.present? && row.academic_year.present?
+
+      schools << row.school
+
+      respondents << create_staffing_entry(row:)
+    end
+
+    Respondent.import respondents, batch_size: 1000, on_duplicate_key_update: [:total_teachers]
+    Respondent.where.not(school: schools).destroy_all
+  end
+
+  def self.clone_previous_year_data
+    years = AcademicYear.order(:range).last(2)
+    previous_year = years.first
+    current_year = years.last
+    respondents = []
+    School.all.each do |school|
+      Respondent.where(school:, academic_year: previous_year).each do |respondent|
+        current_respondent = Respondent.find_or_initialize_by(school:, academic_year: current_year)
+        current_respondent.total_teachers = respondent.total_teachers
+        respondents << current_respondent
+      end
+    end
+    Respondent.import respondents, batch_size: 1000, on_duplicate_key_update: [:total_teachers]
+  end
+
+  private
+
+  def self.create_staffing_entry(row:)
+    respondent = Respondent.find_or_initialize_by(school: row.school, academic_year: row.academic_year)
+    respondent.total_teachers = row.fte_count
+    respondent
+  end
+
+  private_class_method :create_staffing_entry
+end
+
+class StaffingRowValues
+  attr_reader :row
+
+  def initialize(row:)
+    @row = row
+  end
+
+  def school
+    @school ||= begin
+      dese_id = row['DESE ID'].strip.to_i
+      School.find_by_dese_id(dese_id)
+    end
+  end
+
+  def academic_year
+    @academic_year ||= begin
+      year = row['Academic Year']
+      AcademicYear.find_by_range(year)
+    end
+  end
+
+  def fte_count
+    row['FTE Count']
+  end
+end