From 0af7d42e5206e0ff881ea14dc2287aab10cd9fbd Mon Sep 17 00:00:00 2001 From: Nelson Jovel Date: Fri, 12 Jan 2024 21:04:42 -0800 Subject: [PATCH] chore: seed enrollment --- app/lib/dashboard/seeder.rb | 14 +- app/models/dashboard/academic_year.rb | 2 + app/models/dashboard/school.rb | 2 +- app/services/dashboard/cleaner.rb | 2 +- app/services/dashboard/enrollment_loader.rb | 127 +++++++++++------- .../dashboard/survey_responses_data_loader.rb | 2 +- ...0104181617_create_dashboard_respondents.rb | 2 + lib/tasks/db.rake | 4 + spec/dummy/db/schema.rb | 1 + 9 files changed, 100 insertions(+), 56 deletions(-) diff --git a/app/lib/dashboard/seeder.rb b/app/lib/dashboard/seeder.rb index 9bca426..3fa9b3e 100644 --- a/app/lib/dashboard/seeder.rb +++ b/app/lib/dashboard/seeder.rb @@ -101,14 +101,14 @@ module Dashboard DemographicLoader.load_data(filepath: csv_file) end - # def seed_enrollment(csv_file) - # EnrollmentLoader.load_data(filepath: csv_file) - # missing_enrollment_for_current_year = Respondent.where(academic_year: AcademicYear.order(:range).last).none? do |respondent| - # respondent&.total_students&.zero? - # end + def seed_enrollment(csv_file) + EnrollmentLoader.new.load_data(filepath: csv_file) + missing_enrollment_for_current_year = Respondent.where(academic_year: AcademicYear.order(:range).last).none? do |respondent| + respondent&.total_students&.zero? + end - # EnrollmentLoader.clone_previous_year_data if missing_enrollment_for_current_year - # end + EnrollmentLoader.new.clone_previous_year_data if missing_enrollment_for_current_year + end # def seed_staffing(csv_file) # StaffingLoader.load_data(filepath: csv_file) diff --git a/app/models/dashboard/academic_year.rb b/app/models/dashboard/academic_year.rb index 7b56652..a87b4a2 100644 --- a/app/models/dashboard/academic_year.rb +++ b/app/models/dashboard/academic_year.rb @@ -1,5 +1,7 @@ module Dashboard class AcademicYear < ApplicationRecord + scope :by_range, -> { all.map { |ay| [ay.range, ay] }.to_h } + def self.find_by_date(date) year = parse_year_range(date:) range = "#{year.start}-#{year.end.to_s[2, 3]}" diff --git a/app/models/dashboard/school.rb b/app/models/dashboard/school.rb index 2ba8e5d..37883a3 100644 --- a/app/models/dashboard/school.rb +++ b/app/models/dashboard/school.rb @@ -10,7 +10,7 @@ module Dashboard validates :name, presence: true scope :alphabetic, -> { order(name: :asc) } - scope :school_hash, -> { all.map { |school| [school.dese_id, school] }.to_h } + scope :by_dese_id, -> { all.map { |school| [school.dese_id, school] }.to_h } def self.find_by_district_code_and_school_code(district_code, school_code) School diff --git a/app/services/dashboard/cleaner.rb b/app/services/dashboard/cleaner.rb index 4b11a66..d1d2969 100644 --- a/app/services/dashboard/cleaner.rb +++ b/app/services/dashboard/cleaner.rb @@ -121,7 +121,7 @@ module Dashboard end def schools - @schools ||= School.school_hash + @schools ||= School.by_dese_id end def genders diff --git a/app/services/dashboard/enrollment_loader.rb b/app/services/dashboard/enrollment_loader.rb index 7859446..e24c29e 100644 --- a/app/services/dashboard/enrollment_loader.rb +++ b/app/services/dashboard/enrollment_loader.rb @@ -4,11 +4,11 @@ require "csv" module Dashboard class EnrollmentLoader - def self.load_data(filepath:) + def load_data(filepath:) schools = [] enrollments = [] CSV.parse(File.read(filepath), headers: true) do |row| - row = EnrollmentRowValues.new(row:) + row = EnrollmentRowValues.new(row:, schools: school_hash, academic_years: academic_year_hash) next unless row.school.present? && row.academic_year.present? @@ -17,113 +17,148 @@ module Dashboard enrollments << create_enrollment_entry(row:) end - # It's possible that instead of updating all columns on duplicate key, we could just update the student columns and leave total_teachers alone. Right now enrollment data loads before staffing data so it works correctly. - Respondent.import enrollments, batch_size: 1000, - on_duplicate_key_update: %i[pk k one two three four five six seven eight nine ten eleven twelve total_students] - - Respondent.where.not(school: schools).destroy_all + Respondent.upsert_all(enrollments, unique_by: %i[dashboard_school_id dashboard_academic_year_id]) + end + + def clone_previous_year_data + years = AcademicYear.order(:range).last(2) + previous_year = years.first + current_year = years.last + respondents = [] + School.all.each do |school| + Respondent.where(school:, academic_year: previous_year).each do |respondent| + respondents << { dashboard_school_id: respondent.school.id, + dashboard_academic_year_id: current_year.id, + pk: respondent.pk, + k: respondent.k, + one: respondent.one, + two: respondent.two, + three: respondent.three, + four: respondent.four, + five: respondent.five, + six: respondent.six, + seven: respondent.seven, + eight: respondent.eight, + nine: respondent.nine, + ten: respondent.ten, + eleven: respondent.eleven, + twelve: respondent.twelve, + total_students: respondent.total_students } + end + end + Respondent.upsert_all(respondents, unique_by: %i[dashboard_school_id dashboard_academic_year_id]) end private - def self.create_enrollment_entry(row:) - respondent = Respondent.find_or_initialize_by(school: row.school, academic_year: row.academic_year) - respondent.pk = row.pk - respondent.k = row.k - respondent.one = row.one - respondent.two = row.two - respondent.three = row.three - respondent.four = row.four - respondent.five = row.five - respondent.six = row.six - respondent.seven = row.seven - respondent.eight = row.eight - respondent.nine = row.nine - respondent.ten = row.ten - respondent.eleven = row.eleven - respondent.twelve = row.twelve - respondent.total_students = row.total_students - respondent - end - - private_class_method :create_enrollment_entry + def school_hash + @school_hash ||= School.by_dese_id + end + + def academic_year_hash + @academic_year_hash ||= AcademicYear.by_range + end + + def create_enrollment_entry(row:) + { dashboard_school_id: row.school.id, + dashboard_academic_year_id: row.academic_year.id, + pk: row.pk, + k: row.k, + one: row.one, + two: row.two, + three: row.three, + four: row.four, + five: row.five, + six: row.six, + seven: row.seven, + eight: row.eight, + nine: row.nine, + ten: row.ten, + eleven: row.eleven, + twelve: row.twelve, + total_students: row.total_students } + end end class EnrollmentRowValues - attr_reader :row + attr_reader :row, :schools, :academic_years - def initialize(row:) + def initialize(row:, schools:, academic_years:) @row = row + @schools = schools + @academic_years = academic_years end def school @school ||= begin dese_id = row["DESE ID"].try(:strip).to_i - School.find_by_dese_id(dese_id) + schools[dese_id] end end def academic_year @academic_year ||= begin year = row["Academic Year"] - AcademicYear.find_by_range(year) + academic_years[year] end end def pk - row["PK"] || row["pk"] + output = row["PK"] || row["pk"] + output&.strip&.to_i end def k - row["K"] || row["k"] + output = row["K"] || row["k"] + output&.strip&.to_i end def one - row["1"] + row["1"]&.strip&.to_i end def two - row["2"] + row["2"]&.strip&.to_i end def three - row["3"] + row["3"]&.strip&.to_i end def four - row["4"] + row["4"]&.strip&.to_i end def five - row["5"] + row["5"]&.strip&.to_i end def six - row["6"] + row["6"]&.strip&.to_i end def seven - row["7"] + row["7"]&.strip&.to_i end def eight - row["8"] + row["8"]&.strip&.to_i end def nine - row["9"] + row["9"]&.strip&.to_i end def ten - row["10"] + row["10"]&.strip&.to_i end def eleven - row["11"] + row["11"]&.strip&.to_i end def twelve - row["12"] + row["12"]&.strip&.to_i end def total_students diff --git a/app/services/dashboard/survey_responses_data_loader.rb b/app/services/dashboard/survey_responses_data_loader.rb index 26c6256..e813953 100644 --- a/app/services/dashboard/survey_responses_data_loader.rb +++ b/app/services/dashboard/survey_responses_data_loader.rb @@ -49,7 +49,7 @@ module Dashboard private def schools - @schools = School.school_hash + @schools = School.by_dese_id end def genders diff --git a/db/migrate/20240104181617_create_dashboard_respondents.rb b/db/migrate/20240104181617_create_dashboard_respondents.rb index 5c54156..8cced02 100644 --- a/db/migrate/20240104181617_create_dashboard_respondents.rb +++ b/db/migrate/20240104181617_create_dashboard_respondents.rb @@ -22,5 +22,7 @@ class CreateDashboardRespondents < ActiveRecord::Migration[7.1] t.timestamps end + + add_index :dashboard_respondents, %i[dashboard_school_id dashboard_academic_year_id], unique: true end end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index b6973fe..a67af87 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -10,6 +10,10 @@ namespace :dashboard do "master_list_of_schools_and_districts.csv") seeder.seed_sqm_framework Dashboard::Engine.root.join("data", "dashboard", "sqm_framework.csv") seeder.seed_demographics Dashboard::Engine.root.join("data", "dashboard", "demographics.csv") + + Dir.glob("#{Dashboard::Engine.root}/data/dashboard/enrollment/*.csv").each do |file| + seeder.seed_enrollment file + end # seeder.seed_enrollment Rails.root.join("data", "enrollment", "enrollment.csv") # seeder.seed_enrollment Rails.root.join("data", "enrollment", "nj_enrollment.csv") # seeder.seed_enrollment Rails.root.join("data", "enrollment", "wi_enrollment.csv") diff --git a/spec/dummy/db/schema.rb b/spec/dummy/db/schema.rb index 82ed2cf..415a2c8 100644 --- a/spec/dummy/db/schema.rb +++ b/spec/dummy/db/schema.rb @@ -126,6 +126,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_04_192128) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["dashboard_academic_year_id"], name: "index_dashboard_respondents_on_dashboard_academic_year_id" + t.index ["dashboard_school_id", "dashboard_academic_year_id"], name: "idx_on_dashboard_school_id_dashboard_academic_year__17920cd0dd", unique: true t.index ["dashboard_school_id"], name: "index_dashboard_respondents_on_dashboard_school_id" end