From 725348bf95733831e32a0ae012e282afbf90b974 Mon Sep 17 00:00:00 2001 From: Nelson Jovel Date: Mon, 5 Feb 2024 15:00:39 -0800 Subject: [PATCH] chore: get admin data loader working --- app/services/dashboard/dese/loader.rb | 101 +++++++++--------- .../dashboard/survey_responses_data_loader.rb | 5 +- ...4331_create_dashboard_admin_data_values.rb | 3 + lib/tasks/dashboard_tasks.rake | 9 +- spec/dummy/db/schema.rb | 1 + 5 files changed, 60 insertions(+), 59 deletions(-) diff --git a/app/services/dashboard/dese/loader.rb b/app/services/dashboard/dese/loader.rb index 04200cc..61ea979 100644 --- a/app/services/dashboard/dese/loader.rb +++ b/app/services/dashboard/dese/loader.rb @@ -1,71 +1,66 @@ require "csv" -module Dese - class Loader - def self.load_data(filepath:) - admin_data_values = [] - CSV.parse(File.read(filepath), headers: true) do |row| - score = likert_score(row:) - next unless valid_likert_score(likert_score: score) +module Dashboard + module Dese + class Loader + def self.load_data(filepath:) + admin_data_values = [] + CSV.parse(File.read(filepath), headers: true) do |row| + score = likert_score(row:) + next unless valid_likert_score(likert_score: score) - admin_data_values << create_admin_data_value(row:, score:) - end + admin_data_values << create_admin_data_value(row:, score:) + end - AdminDataValue.import(admin_data_values.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all) - end + AdminDataValue.upsert_all(admin_data_values.flatten.compact, + unique_by: %i[dashboard_school_id dashboard_admin_data_item_id + dashboard_academic_year_id]) + end - private + private - def self.valid_likert_score(likert_score:) - likert_score >= 1 && likert_score <= 5 - end + def self.valid_likert_score(likert_score:) + likert_score >= 1 && likert_score <= 5 + end - def self.likert_score(row:) - likert_score = (row["Likert Score"] || row["LikertScore"] || row["Likert_Score"]).to_f - likert_score.round_up_to_one.round_down_to_five - end + def self.likert_score(row:) + likert_score = (row["Likert Score"] || row["LikertScore"] || row["Likert_Score"]).to_f + likert_score.round_up_to_one.round_down_to_five + end - def self.ay(row:) - row["Academic Year"] || row["AcademicYear"] - end + def self.ay(row:) + row["Academic Year"] || row["AcademicYear"] + end - def self.dese_id(row:) - row["DESE ID"] || row["Dese ID"] || row["Dese Id"] || row["School ID"] - end + def self.dese_id(row:) + row["DESE ID"] || row["Dese ID"] || row["Dese Id"] || row["School ID"] + end - def self.admin_data_item(row:) - row["Admin Data Item"] || row["Item ID"] || row["Item Id"] || row["Item ID"] - end + def self.admin_data_item(row:) + row["Admin Data Item"] || row["Item ID"] || row["Item Id"] || row["Item ID"] + end - def self.create_admin_data_value(row:, score:) - school = School.find_by_dese_id(dese_id(row:).to_i) - admin_data_item_id = admin_data_item(row:) + def self.create_admin_data_value(row:, score:) + school = School.find_by_dese_id(dese_id(row:).to_i) + admin_data_item_id = admin_data_item(row:) - return if school.nil? - return if admin_data_item_id.nil? || admin_data_item_id.blank? + return if school.nil? + return if admin_data_item_id.nil? || admin_data_item_id.blank? - admin_data_value = AdminDataValue.find_by(academic_year: AcademicYear.find_by_range(ay(row:)), - school:, - admin_data_item: AdminDataItem.find_by_admin_data_item_id(admin_data_item_id)) - if admin_data_value.present? - admin_data_value.likert_score = score - admin_data_value.save - nil - else - AdminDataValue.new( + { likert_score: score, - academic_year: AcademicYear.find_by_range(ay(row:)), - school:, - admin_data_item: AdminDataItem.find_by_admin_data_item_id(admin_data_item(row:)) - ) + dashboard_school_id: school.id, + dashboard_admin_data_item_id: AdminDataItem.find_by_admin_data_item_id(admin_data_item(row:)).id, + dashboard_academic_year_id: AcademicYear.find_by_range(ay(row:)).id + } end - end - private_class_method :valid_likert_score - private_class_method :likert_score - private_class_method :ay - private_class_method :dese_id - private_class_method :admin_data_item - private_class_method :create_admin_data_value + private_class_method :valid_likert_score + private_class_method :likert_score + private_class_method :ay + private_class_method :dese_id + private_class_method :admin_data_item + private_class_method :create_admin_data_value + end end end diff --git a/app/services/dashboard/survey_responses_data_loader.rb b/app/services/dashboard/survey_responses_data_loader.rb index 7271459..ac65f52 100644 --- a/app/services/dashboard/survey_responses_data_loader.rb +++ b/app/services/dashboard/survey_responses_data_loader.rb @@ -3,7 +3,6 @@ module Dashboard class SurveyResponsesDataLoader def load_data(filepath:) - byebug File.open(filepath) do |file| headers = file.first headers_array = CSV.parse(headers).first @@ -14,7 +13,9 @@ module Dashboard process_row(row: SurveyItemValues.new(row:, headers: headers_array, survey_items: all_survey_items, schools:)) end - SurveyItemResponse.upsert_all(survey_item_responses, unique_by: :response_id) + SurveyItemResponse.upsert_all(survey_item_responses.flatten.compact, + unique_by: %i[response_id dashboard_academic_year_id dashboard_school_id + dashboard_survey_item_id]) end end end diff --git a/db/migrate/20240104174331_create_dashboard_admin_data_values.rb b/db/migrate/20240104174331_create_dashboard_admin_data_values.rb index f9ec684..64d73a6 100644 --- a/db/migrate/20240104174331_create_dashboard_admin_data_values.rb +++ b/db/migrate/20240104174331_create_dashboard_admin_data_values.rb @@ -8,5 +8,8 @@ class CreateDashboardAdminDataValues < ActiveRecord::Migration[7.1] t.timestamps end + + add_index :dashboard_admin_data_values, + %i[dashboard_admin_data_item_id dashboard_school_id dashboard_academic_year_id], unique: true end end diff --git a/lib/tasks/dashboard_tasks.rake b/lib/tasks/dashboard_tasks.rake index e054df7..8facda8 100644 --- a/lib/tasks/dashboard_tasks.rake +++ b/lib/tasks/dashboard_tasks.rake @@ -31,14 +31,15 @@ namespace :dashboard do desc "load admin_data" task load_admin_data: :environment do original_count = AdminDataValue.count - Dir.glob(Rails.root.join("data", "admin_data", "dese", "*.csv")).each do |filepath| + Dir.glob(Dashboard::Engine.root.join("data", "dashboard", "admin_data", "dese", "*.csv")).each do |filepath| puts "=====================> Loading data from csv at path: #{filepath}" - Dese::Loader.load_data filepath: + Dashboard::Dese::Loader.load_data filepath: end - Dir.glob(Rails.root.join("data", "admin_data", "out_of_state", "*.csv")).each do |filepath| + Dir.glob(Dashboard::Engine.root.join("data", "dashboard", "admin_data", "out_of_state", + "*.csv")).each do |filepath| puts "=====================> Loading data from csv at path: #{filepath}" - Dese::Loader.load_data filepath: + Dashboard::Dese::Loader.load_data filepath: end puts "=====================> Completed loading #{AdminDataValue.count - original_count} admin data values" end diff --git a/spec/dummy/db/schema.rb b/spec/dummy/db/schema.rb index 9991353..1d09e93 100644 --- a/spec/dummy/db/schema.rb +++ b/spec/dummy/db/schema.rb @@ -43,6 +43,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_04_192128) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["dashboard_academic_year_id"], name: "idx_on_dashboard_academic_year_id_1de27231d5" + t.index ["dashboard_admin_data_item_id", "dashboard_school_id", "dashboard_academic_year_id"], name: "idx_on_dashboard_admin_data_item_id_dashboard_schoo_4a9c27f1d0", unique: true t.index ["dashboard_admin_data_item_id"], name: "idx_on_dashboard_admin_data_item_id_edae2faad3" t.index ["dashboard_school_id"], name: "index_dashboard_admin_data_values_on_dashboard_school_id" end