chore: get admin data loader working

app/services/dashboard/dese/loader.rb

@@ -1,5 +1,6 @@
 require "csv"
 
+module Dashboard
   module Dese
     class Loader
       def self.load_data(filepath:)
@@ -11,7 +12,9 @@ module Dese
           admin_data_values << create_admin_data_value(row:, score:)
         end
 
-      AdminDataValue.import(admin_data_values.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
+        AdminDataValue.upsert_all(admin_data_values.flatten.compact,
+                                  unique_by: %i[dashboard_school_id dashboard_admin_data_item_id
+                                                dashboard_academic_year_id])
       end
 
       private
@@ -44,21 +47,12 @@ module Dese
         return if school.nil?
         return if admin_data_item_id.nil? || admin_data_item_id.blank?
 
-      admin_data_value = AdminDataValue.find_by(academic_year: AcademicYear.find_by_range(ay(row:)),
-        school:,
-        admin_data_item: AdminDataItem.find_by_admin_data_item_id(admin_data_item_id))
-      if admin_data_value.present?
-        admin_data_value.likert_score = score
-        admin_data_value.save
-        nil
-      else
-        AdminDataValue.new(
+        {
           likert_score: score,
-          academic_year: AcademicYear.find_by_range(ay(row:)),
-          school:,
-          admin_data_item: AdminDataItem.find_by_admin_data_item_id(admin_data_item(row:))
-        )
-      end
+          dashboard_school_id: school.id,
+          dashboard_admin_data_item_id: AdminDataItem.find_by_admin_data_item_id(admin_data_item(row:)).id,
+          dashboard_academic_year_id: AcademicYear.find_by_range(ay(row:)).id
+        }
       end
 
       private_class_method :valid_likert_score
@@ -69,3 +63,4 @@ module Dese
       private_class_method :create_admin_data_value
     end
   end
+end

app/services/dashboard/survey_responses_data_loader.rb

@@ -3,7 +3,6 @@
 module Dashboard
   class SurveyResponsesDataLoader
     def load_data(filepath:)
-      byebug
       File.open(filepath) do |file|
         headers = file.first
         headers_array = CSV.parse(headers).first
@@ -14,7 +13,9 @@ module Dashboard
             process_row(row: SurveyItemValues.new(row:, headers: headers_array, survey_items: all_survey_items,
                                                   schools:))
           end
-          SurveyItemResponse.upsert_all(survey_item_responses, unique_by: :response_id)
+          SurveyItemResponse.upsert_all(survey_item_responses.flatten.compact,
+                                        unique_by: %i[response_id dashboard_academic_year_id dashboard_school_id
+                                                      dashboard_survey_item_id])
         end
       end
     end

db/migrate/20240104174331_create_dashboard_admin_data_values.rb

@@ -8,5 +8,8 @@ class CreateDashboardAdminDataValues < ActiveRecord::Migration[7.1]
 
       t.timestamps
     end
+
+    add_index :dashboard_admin_data_values,
+              %i[dashboard_admin_data_item_id dashboard_school_id dashboard_academic_year_id], unique: true
   end
 end

lib/tasks/dashboard_tasks.rake

@@ -31,14 +31,15 @@ namespace :dashboard do
     desc "load admin_data"
     task load_admin_data: :environment do
       original_count = AdminDataValue.count
-      Dir.glob(Rails.root.join("data", "admin_data", "dese", "*.csv")).each do |filepath|
+      Dir.glob(Dashboard::Engine.root.join("data", "dashboard", "admin_data", "dese", "*.csv")).each do |filepath|
         puts "=====================> Loading data from csv at path: #{filepath}"
-        Dese::Loader.load_data filepath:
+        Dashboard::Dese::Loader.load_data filepath:
       end
 
-      Dir.glob(Rails.root.join("data", "admin_data", "out_of_state", "*.csv")).each do |filepath|
+      Dir.glob(Dashboard::Engine.root.join("data", "dashboard", "admin_data", "out_of_state",
+                                           "*.csv")).each do |filepath|
         puts "=====================> Loading data from csv at path: #{filepath}"
-        Dese::Loader.load_data filepath:
+        Dashboard::Dese::Loader.load_data filepath:
       end
       puts "=====================> Completed loading #{AdminDataValue.count - original_count} admin data values"
     end

spec/dummy/db/schema.rb

@@ -43,6 +43,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_04_192128) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["dashboard_academic_year_id"], name: "idx_on_dashboard_academic_year_id_1de27231d5"
+    t.index ["dashboard_admin_data_item_id", "dashboard_school_id", "dashboard_academic_year_id"], name: "idx_on_dashboard_admin_data_item_id_dashboard_schoo_4a9c27f1d0", unique: true
     t.index ["dashboard_admin_data_item_id"], name: "idx_on_dashboard_admin_data_item_id_edae2faad3"
     t.index ["dashboard_school_id"], name: "index_dashboard_admin_data_values_on_dashboard_school_id"
   end