chore: get admin data loader working

app/services/dashboard/dese/loader.rb

@@ -1,71 +1,66 @@
 require "csv"
 
-module Dese
+module Dashboard
-  class Loader
+  module Dese
-    def self.load_data(filepath:)
+    class Loader
-      admin_data_values = []
+      def self.load_data(filepath:)
-      CSV.parse(File.read(filepath), headers: true) do |row|
+        admin_data_values = []
-        score = likert_score(row:)
+        CSV.parse(File.read(filepath), headers: true) do |row|
-        next unless valid_likert_score(likert_score: score)
+          score = likert_score(row:)
+          next unless valid_likert_score(likert_score: score)
 
-        admin_data_values << create_admin_data_value(row:, score:)
+          admin_data_values << create_admin_data_value(row:, score:)
-      end
+        end
 
-      AdminDataValue.import(admin_data_values.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
+        AdminDataValue.upsert_all(admin_data_values.flatten.compact,
-    end
+                                  unique_by: %i[dashboard_school_id dashboard_admin_data_item_id
+                                                dashboard_academic_year_id])
+      end
 
-    private
+      private
 
-    def self.valid_likert_score(likert_score:)
+      def self.valid_likert_score(likert_score:)
-      likert_score >= 1 && likert_score <= 5
+        likert_score >= 1 && likert_score <= 5
-    end
+      end
 
-    def self.likert_score(row:)
+      def self.likert_score(row:)
-      likert_score = (row["Likert Score"] || row["LikertScore"] || row["Likert_Score"]).to_f
+        likert_score = (row["Likert Score"] || row["LikertScore"] || row["Likert_Score"]).to_f
-      likert_score.round_up_to_one.round_down_to_five
+        likert_score.round_up_to_one.round_down_to_five
-    end
+      end
 
-    def self.ay(row:)
+      def self.ay(row:)
-      row["Academic Year"] || row["AcademicYear"]
+        row["Academic Year"] || row["AcademicYear"]
-    end
+      end
 
-    def self.dese_id(row:)
+      def self.dese_id(row:)
-      row["DESE ID"] || row["Dese ID"] || row["Dese Id"] || row["School ID"]
+        row["DESE ID"] || row["Dese ID"] || row["Dese Id"] || row["School ID"]
-    end
+      end
 
-    def self.admin_data_item(row:)
+      def self.admin_data_item(row:)
-      row["Admin Data Item"] || row["Item ID"] || row["Item Id"] || row["Item  ID"]
+        row["Admin Data Item"] || row["Item ID"] || row["Item Id"] || row["Item  ID"]
-    end
+      end
 
-    def self.create_admin_data_value(row:, score:)
+      def self.create_admin_data_value(row:, score:)
-      school = School.find_by_dese_id(dese_id(row:).to_i)
+        school = School.find_by_dese_id(dese_id(row:).to_i)
-      admin_data_item_id = admin_data_item(row:)
+        admin_data_item_id = admin_data_item(row:)
 
-      return if school.nil?
+        return if school.nil?
-      return if admin_data_item_id.nil? || admin_data_item_id.blank?
+        return if admin_data_item_id.nil? || admin_data_item_id.blank?
 
-      admin_data_value = AdminDataValue.find_by(academic_year: AcademicYear.find_by_range(ay(row:)),
+        {
-        school:,
-        admin_data_item: AdminDataItem.find_by_admin_data_item_id(admin_data_item_id))
-      if admin_data_value.present?
-        admin_data_value.likert_score = score
-        admin_data_value.save
-        nil
-      else
-        AdminDataValue.new(
           likert_score: score,
-          academic_year: AcademicYear.find_by_range(ay(row:)),
+          dashboard_school_id: school.id,
-          school:,
+          dashboard_admin_data_item_id: AdminDataItem.find_by_admin_data_item_id(admin_data_item(row:)).id,
-          admin_data_item: AdminDataItem.find_by_admin_data_item_id(admin_data_item(row:))
+          dashboard_academic_year_id: AcademicYear.find_by_range(ay(row:)).id
-        )
+        }
       end
-    end
 
-    private_class_method :valid_likert_score
+      private_class_method :valid_likert_score
-    private_class_method :likert_score
+      private_class_method :likert_score
-    private_class_method :ay
+      private_class_method :ay
-    private_class_method :dese_id
+      private_class_method :dese_id
-    private_class_method :admin_data_item
+      private_class_method :admin_data_item
-    private_class_method :create_admin_data_value
+      private_class_method :create_admin_data_value
+    end
   end
 end

app/services/dashboard/survey_responses_data_loader.rb

@@ -3,7 +3,6 @@
 module Dashboard
   class SurveyResponsesDataLoader
     def load_data(filepath:)
-      byebug
       File.open(filepath) do |file|
         headers = file.first
         headers_array = CSV.parse(headers).first
@@ -14,7 +13,9 @@ module Dashboard
             process_row(row: SurveyItemValues.new(row:, headers: headers_array, survey_items: all_survey_items,
                                                   schools:))
           end
-          SurveyItemResponse.upsert_all(survey_item_responses, unique_by: :response_id)
+          SurveyItemResponse.upsert_all(survey_item_responses.flatten.compact,
+                                        unique_by: %i[response_id dashboard_academic_year_id dashboard_school_id
+                                                      dashboard_survey_item_id])
         end
       end
     end

db/migrate/20240104174331_create_dashboard_admin_data_values.rb

@@ -8,5 +8,8 @@ class CreateDashboardAdminDataValues < ActiveRecord::Migration[7.1]
 
       t.timestamps
     end
+
+    add_index :dashboard_admin_data_values,
+              %i[dashboard_admin_data_item_id dashboard_school_id dashboard_academic_year_id], unique: true
   end
 end

lib/tasks/dashboard_tasks.rake

@@ -31,14 +31,15 @@ namespace :dashboard do
     desc "load admin_data"
     task load_admin_data: :environment do
       original_count = AdminDataValue.count
-      Dir.glob(Rails.root.join("data", "admin_data", "dese", "*.csv")).each do |filepath|
+      Dir.glob(Dashboard::Engine.root.join("data", "dashboard", "admin_data", "dese", "*.csv")).each do |filepath|
         puts "=====================> Loading data from csv at path: #{filepath}"
-        Dese::Loader.load_data filepath:
+        Dashboard::Dese::Loader.load_data filepath:
       end
 
-      Dir.glob(Rails.root.join("data", "admin_data", "out_of_state", "*.csv")).each do |filepath|
+      Dir.glob(Dashboard::Engine.root.join("data", "dashboard", "admin_data", "out_of_state",
+                                           "*.csv")).each do |filepath|
         puts "=====================> Loading data from csv at path: #{filepath}"
-        Dese::Loader.load_data filepath:
+        Dashboard::Dese::Loader.load_data filepath:
       end
       puts "=====================> Completed loading #{AdminDataValue.count - original_count} admin data values"
     end

spec/dummy/db/schema.rb

@@ -43,6 +43,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_04_192128) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["dashboard_academic_year_id"], name: "idx_on_dashboard_academic_year_id_1de27231d5"
+    t.index ["dashboard_admin_data_item_id", "dashboard_school_id", "dashboard_academic_year_id"], name: "idx_on_dashboard_admin_data_item_id_dashboard_schoo_4a9c27f1d0", unique: true
     t.index ["dashboard_admin_data_item_id"], name: "idx_on_dashboard_admin_data_item_id_edae2faad3"
     t.index ["dashboard_school_id"], name: "index_dashboard_admin_data_values_on_dashboard_school_id"
   end