Scrape enrollment and staffing information. Seed enrollment and staffing information. Update DatabaseCleaner so it cleans up leftover information in the database. Remove old admin csvs from codebase.

app/lib/seeder.rb

@@ -138,6 +138,14 @@ class Seeder
     EnrollmentLoader.load_data(filepath: csv_file)
   end
 
+  def seed_staffing(csv_file)
+    StaffingLoader.load_data(filepath: csv_file)
+    missing_staffing_for_current_year = Respondent.where(academic_year: AcademicYear.order(:range).last).none? do |respondent|
+      respondent.total_teachers.present?
+    end
+    StaffingLoader.clone_previous_year_data if missing_staffing_for_current_year
+  end
+
   private
 
   def marked?(mark)

app/services/dese/one_a_three.rb

@@ -42,7 +42,7 @@ module Dese
     def run_a_pcom_i3
       filepath = filepaths[1]
       headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
-                 'African American (%)', 'Asian (%)', 'Hispanic (%)', 'White (%)', 'Native Hawaiian, Pacific Islander (%)',
+                 'African American (%)', 'Asian (%)', 'Hispanic (%)', 'White (%)', 'Native Amertican (%)', 'Native Hawaiian, Pacific Islander (%)',
                  'Multi-Race,Non-Hispanic (%)', 'Females (%)', 'Males (%)', 'FTE Count']
       write_headers(filepath:, headers:)
 

app/services/dese/scraper.rb

@@ -1,6 +1,6 @@
 module Dese
   module Scraper
-    DELAY = 20
+    DELAY = 20 # The dese site will block you if you hit it too many times in a short period of time
 
     Prerequisites = Struct.new('Prerequisites', :filepath, :url, :selectors, :submit_id, :admin_data_item_id,
                                :calculation)

app/services/enrollment_loader.rb

@@ -15,7 +15,7 @@ class EnrollmentLoader
       create_enrollment_entry(row:)
     end
 
-    # Respondent.where.not(school: schools).destroy_all
+    Respondent.where.not(school: schools).destroy_all
   end
 
   private
@@ -51,7 +51,7 @@ class EnrollmentRowValues
 
   def school
     @school ||= begin
-      dese_id = row['School Code'].try(:strip).to_i
+      dese_id = row['DESE ID'].try(:strip).to_i
       School.find_by_dese_id(dese_id)
     end
   end

app/services/staffing_loader.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+class StaffingLoader
+  def self.load_data(filepath:)
+    schools = []
+    CSV.parse(File.read(filepath), headers: true) do |row|
+      row = StaffingRowValues.new(row:)
+      next unless row.school.present? && row.academic_year.present?
+
+      schools << row.school
+
+      create_staffing_entry(row:)
+    end
+
+    Respondent.where.not(school: schools).destroy_all
+  end
+
+  def self.clone_previous_year_data
+    years = AcademicYear.order(:range).last(2)
+    previous_year = years.first
+    current_year = years.last
+    School.all.each do |school|
+      Respondent.where(school:, academic_year: previous_year).each do |respondent|
+        current_respondent = Respondent.find_or_initialize_by(school:, academic_year: current_year)
+        current_respondent.total_teachers = respondent.total_teachers
+        current_respondent.save
+      end
+    end
+  end
+
+  private
+
+  def self.create_staffing_entry(row:)
+    respondent = Respondent.find_or_initialize_by(school: row.school, academic_year: row.academic_year)
+    respondent.total_teachers = row.fte_count
+    respondent.save
+  end
+
+  private_class_method :create_staffing_entry
+end
+
+class StaffingRowValues
+  attr_reader :row
+
+  def initialize(row:)
+    @row = row
+  end
+
+  def school
+    @school ||= begin
+      dese_id = row['DESE ID'].strip.to_i
+      School.find_by_dese_id(dese_id)
+    end
+  end
+
+  def academic_year
+    @academic_year ||= begin
+      year = row['Academic Year']
+      AcademicYear.find_by_range(year)
+    end
+  end
+
+  def fte_count
+    row['FTE Count']
+  end
+end