chore: seed enrollment

app/lib/dashboard/seeder.rb

@@ -101,14 +101,14 @@ module Dashboard
       DemographicLoader.load_data(filepath: csv_file)
     end
 
-    #     def seed_enrollment(csv_file)
+    def seed_enrollment(csv_file)
-    #       EnrollmentLoader.load_data(filepath: csv_file)
+      EnrollmentLoader.new.load_data(filepath: csv_file)
-    #       missing_enrollment_for_current_year = Respondent.where(academic_year: AcademicYear.order(:range).last).none? do |respondent|
+      missing_enrollment_for_current_year = Respondent.where(academic_year: AcademicYear.order(:range).last).none? do |respondent|
-    #         respondent&.total_students&.zero?
+        respondent&.total_students&.zero?
-    #       end
+      end
 
-    #       EnrollmentLoader.clone_previous_year_data if missing_enrollment_for_current_year
+      EnrollmentLoader.new.clone_previous_year_data if missing_enrollment_for_current_year
-    #     end
+    end
 
     #     def seed_staffing(csv_file)
     #       StaffingLoader.load_data(filepath: csv_file)

app/models/dashboard/academic_year.rb

@@ -1,5 +1,7 @@
 module Dashboard
   class AcademicYear < ApplicationRecord
+    scope :by_range, -> { all.map { |ay| [ay.range, ay] }.to_h }
+
     def self.find_by_date(date)
       year = parse_year_range(date:)
       range = "#{year.start}-#{year.end.to_s[2, 3]}"

app/models/dashboard/school.rb

@@ -10,7 +10,7 @@ module Dashboard
     validates :name, presence: true
 
     scope :alphabetic, -> { order(name: :asc) }
-    scope :school_hash, -> { all.map { |school| [school.dese_id, school] }.to_h }
+    scope :by_dese_id, -> { all.map { |school| [school.dese_id, school] }.to_h }
 
     def self.find_by_district_code_and_school_code(district_code, school_code)
       School

app/services/dashboard/cleaner.rb

@@ -121,7 +121,7 @@ module Dashboard
     end
 
     def schools
-      @schools ||= School.school_hash
+      @schools ||= School.by_dese_id
     end
 
     def genders

app/services/dashboard/enrollment_loader.rb

@@ -4,11 +4,11 @@ require "csv"
 
 module Dashboard
   class EnrollmentLoader
-    def self.load_data(filepath:)
+    def load_data(filepath:)
       schools = []
       enrollments = []
       CSV.parse(File.read(filepath), headers: true) do |row|
-        row = EnrollmentRowValues.new(row:)
+        row = EnrollmentRowValues.new(row:, schools: school_hash, academic_years: academic_year_hash)
 
         next unless row.school.present? && row.academic_year.present?
 
@@ -17,113 +17,148 @@ module Dashboard
         enrollments << create_enrollment_entry(row:)
       end
 
-      # It's possible that instead of updating all columns on duplicate key, we could just update the student columns and leave total_teachers alone. Right now enrollment data loads before staffing data so it works correctly.
+      Respondent.upsert_all(enrollments, unique_by: %i[dashboard_school_id dashboard_academic_year_id])
-      Respondent.import enrollments, batch_size: 1000,
+    end
-                                     on_duplicate_key_update: %i[pk k one two three four five six seven eight nine ten eleven twelve total_students]
 
-      Respondent.where.not(school: schools).destroy_all
+    def clone_previous_year_data
+      years = AcademicYear.order(:range).last(2)
+      previous_year = years.first
+      current_year = years.last
+      respondents = []
+      School.all.each do |school|
+        Respondent.where(school:, academic_year: previous_year).each do |respondent|
+          respondents << { dashboard_school_id: respondent.school.id,
+                           dashboard_academic_year_id: current_year.id,
+                           pk: respondent.pk,
+                           k: respondent.k,
+                           one: respondent.one,
+                           two: respondent.two,
+                           three: respondent.three,
+                           four: respondent.four,
+                           five: respondent.five,
+                           six: respondent.six,
+                           seven: respondent.seven,
+                           eight: respondent.eight,
+                           nine: respondent.nine,
+                           ten: respondent.ten,
+                           eleven: respondent.eleven,
+                           twelve: respondent.twelve,
+                           total_students: respondent.total_students }
+        end
+      end
+      Respondent.upsert_all(respondents, unique_by: %i[dashboard_school_id dashboard_academic_year_id])
     end
 
     private
 
-    def self.create_enrollment_entry(row:)
+    def school_hash
-      respondent = Respondent.find_or_initialize_by(school: row.school, academic_year: row.academic_year)
+      @school_hash ||= School.by_dese_id
-      respondent.pk = row.pk
-      respondent.k = row.k
-      respondent.one = row.one
-      respondent.two = row.two
-      respondent.three = row.three
-      respondent.four = row.four
-      respondent.five = row.five
-      respondent.six = row.six
-      respondent.seven = row.seven
-      respondent.eight = row.eight
-      respondent.nine = row.nine
-      respondent.ten = row.ten
-      respondent.eleven = row.eleven
-      respondent.twelve = row.twelve
-      respondent.total_students = row.total_students
-      respondent
     end
 
-    private_class_method :create_enrollment_entry
+    def academic_year_hash
+      @academic_year_hash ||= AcademicYear.by_range
+    end
+
+    def create_enrollment_entry(row:)
+      { dashboard_school_id: row.school.id,
+        dashboard_academic_year_id: row.academic_year.id,
+        pk: row.pk,
+        k: row.k,
+        one: row.one,
+        two: row.two,
+        three: row.three,
+        four: row.four,
+        five: row.five,
+        six: row.six,
+        seven: row.seven,
+        eight: row.eight,
+        nine: row.nine,
+        ten: row.ten,
+        eleven: row.eleven,
+        twelve: row.twelve,
+        total_students: row.total_students }
+    end
   end
 
   class EnrollmentRowValues
-    attr_reader :row
+    attr_reader :row, :schools, :academic_years
 
-    def initialize(row:)
+    def initialize(row:, schools:, academic_years:)
       @row = row
+      @schools = schools
+      @academic_years = academic_years
     end
 
     def school
       @school ||= begin
         dese_id = row["DESE ID"].try(:strip).to_i
-        School.find_by_dese_id(dese_id)
+        schools[dese_id]
       end
     end
 
     def academic_year
       @academic_year ||= begin
         year = row["Academic Year"]
-        AcademicYear.find_by_range(year)
+        academic_years[year]
       end
     end
 
     def pk
-      row["PK"] || row["pk"]
+      output = row["PK"] || row["pk"]
+      output&.strip&.to_i
     end
 
     def k
-      row["K"] || row["k"]
+      output = row["K"] || row["k"]
+      output&.strip&.to_i
     end
 
     def one
-      row["1"]
+      row["1"]&.strip&.to_i
     end
 
     def two
-      row["2"]
+      row["2"]&.strip&.to_i
     end
 
     def three
-      row["3"]
+      row["3"]&.strip&.to_i
     end
 
     def four
-      row["4"]
+      row["4"]&.strip&.to_i
     end
 
     def five
-      row["5"]
+      row["5"]&.strip&.to_i
     end
 
     def six
-      row["6"]
+      row["6"]&.strip&.to_i
     end
 
     def seven
-      row["7"]
+      row["7"]&.strip&.to_i
     end
 
     def eight
-      row["8"]
+      row["8"]&.strip&.to_i
     end
 
     def nine
-      row["9"]
+      row["9"]&.strip&.to_i
     end
 
     def ten
-      row["10"]
+      row["10"]&.strip&.to_i
     end
 
     def eleven
-      row["11"]
+      row["11"]&.strip&.to_i
     end
 
     def twelve
-      row["12"]
+      row["12"]&.strip&.to_i
     end
 
     def total_students

app/services/dashboard/survey_responses_data_loader.rb

@@ -49,7 +49,7 @@ module Dashboard
     private
 
     def schools
-      @schools = School.school_hash
+      @schools = School.by_dese_id
     end
 
     def genders

db/migrate/20240104181617_create_dashboard_respondents.rb

@@ -22,5 +22,7 @@ class CreateDashboardRespondents < ActiveRecord::Migration[7.1]
 
       t.timestamps
     end
+
+    add_index :dashboard_respondents, %i[dashboard_school_id dashboard_academic_year_id], unique: true
   end
 end

lib/tasks/db.rake

@@ -10,6 +10,10 @@ namespace :dashboard do
                                                                     "master_list_of_schools_and_districts.csv")
       seeder.seed_sqm_framework Dashboard::Engine.root.join("data", "dashboard", "sqm_framework.csv")
       seeder.seed_demographics Dashboard::Engine.root.join("data", "dashboard", "demographics.csv")
+
+      Dir.glob("#{Dashboard::Engine.root}/data/dashboard/enrollment/*.csv").each do |file|
+        seeder.seed_enrollment file
+      end
       # seeder.seed_enrollment Rails.root.join("data", "enrollment", "enrollment.csv")
       # seeder.seed_enrollment Rails.root.join("data", "enrollment", "nj_enrollment.csv")
       # seeder.seed_enrollment Rails.root.join("data", "enrollment", "wi_enrollment.csv")

spec/dummy/db/schema.rb

@@ -126,6 +126,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_04_192128) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["dashboard_academic_year_id"], name: "index_dashboard_respondents_on_dashboard_academic_year_id"
+    t.index ["dashboard_school_id", "dashboard_academic_year_id"], name: "idx_on_dashboard_school_id_dashboard_academic_year__17920cd0dd", unique: true
     t.index ["dashboard_school_id"], name: "index_dashboard_respondents_on_dashboard_school_id"
   end