diff --git a/app/services/dese/loader.rb b/app/services/dese/loader.rb index 04200cc0..5f318f55 100644 --- a/app/services/dese/loader.rb +++ b/app/services/dese/loader.rb @@ -2,8 +2,10 @@ require "csv" module Dese class Loader + @memo = Hash.new def self.load_data(filepath:) admin_data_values = [] + @memo = Hash.new CSV.parse(File.read(filepath), headers: true) do |row| score = likert_score(row:) next unless valid_likert_score(likert_score: score) @@ -37,16 +39,31 @@ module Dese row["Admin Data Item"] || row["Item ID"] || row["Item Id"] || row["Item ID"] end + # these three methods do the memoization + def self.find_school(dese_id:) + return @memo["school"+dese_id] if @memo.key? "school"+dese_id + @memo["school"+dese_id] ||= School.find_by_dese_id(dese_id.to_i) + end + def self.find_admin_data_item(admin_data_item_id:) + return @memo["admin"+admin_data_item_id] if @memo.key? "admin"+admin_data_item_id + @memo["admin"+admin_data_item_id] ||= AdminDataItem.find_by_admin_data_item_id(admin_data_item_id) + end + def self.find_ay(ay:) + return @memo["year"+ay] if @memo.key? "year"+ay + @memo["year"+ay] ||= AcademicYear.find_by_range(ay) + end + def self.create_admin_data_value(row:, score:) - school = School.find_by_dese_id(dese_id(row:).to_i) + school = find_school(dese_id: dese_id(row:)) admin_data_item_id = admin_data_item(row:) + admin_data_item = find_admin_data_item(admin_data_item_id:) + academic_year = find_ay(ay: ay(row:)) return if school.nil? return if admin_data_item_id.nil? || admin_data_item_id.blank? - admin_data_value = AdminDataValue.find_by(academic_year: AcademicYear.find_by_range(ay(row:)), - school:, - admin_data_item: AdminDataItem.find_by_admin_data_item_id(admin_data_item_id)) + admin_data_value = AdminDataValue.find_by(academic_year:, school:, admin_data_item:) + if admin_data_value.present? admin_data_value.likert_score = score admin_data_value.save @@ -54,9 +71,9 @@ module Dese else AdminDataValue.new( likert_score: score, - academic_year: AcademicYear.find_by_range(ay(row:)), + academic_year:, school:, - admin_data_item: AdminDataItem.find_by_admin_data_item_id(admin_data_item(row:)) + admin_data_item:, ) end end diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake index 495b4a2d..9d187892 100644 --- a/lib/tasks/data.rake +++ b/lib/tasks/data.rake @@ -28,15 +28,14 @@ namespace :data do desc "load admin_data" task load_admin_data: :environment do original_count = AdminDataValue.count - Dir.glob(Rails.root.join("data", "admin_data", "dese", "*.csv")).each do |filepath| - puts "=====================> Loading data from csv at path: #{filepath}" - Dese::Loader.load_data filepath: - end - - Dir.glob(Rails.root.join("data", "admin_data", "out_of_state", "*.csv")).each do |filepath| + jobs = Queue.new + Dir.glob(Rails.root.join("data", "admin_data", "dese", "*.csv")).each { |filepath| jobs << filepath } + Dir.glob(Rails.root.join("data", "admin_data", "out_of_state", "*.csv")).each { |filepath| jobs << filepath } + while filepath = jobs.pop(true) puts "=====================> Loading data from csv at path: #{filepath}" Dese::Loader.load_data filepath: end + rescue ThreadError puts "=====================> Completed loading #{AdminDataValue.count - original_count} admin data values" end