gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 21:48:16 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Faster admin data loader + rename School.school_hash

Browse source
This commit is contained in:
Gabe Farrell 2024-04-22 14:46:37 -04:00
parent f58cd413aa
commit 5789ebf564
7 changed files with 119 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

26
app/services/dese/loader.rb
View file

@ -4,11 +4,12 @@ module Dese
def self.load_data(filepath:)
admin_data_values = []
@memo = Hash.new
schools = School.school_by_dese_id
CSV.parse(File.read(filepath), headers: true) do |row|
score = likert_score(row:)
next unless valid_likert_score(likert_score: score)
admin_data_values << create_admin_data_value(row:, score:)
admin_data_values << create_admin_data_value(row:, score:, schools:)
end
AdminDataValue.import(admin_data_values.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
@ -38,21 +39,18 @@ module Dese
end
# these three methods do the memoization
def self.find_school(dese_id:)
return @memo["school"+dese_id] if @memo.key? "school"+dese_id
@memo["school"+dese_id] ||= School.find_by_dese_id(dese_id.to_i)
end
def self.find_admin_data_item(admin_data_item_id:)
return @memo["admin"+admin_data_item_id] if @memo.key? "admin"+admin_data_item_id
@memo["admin"+admin_data_item_id] ||= AdminDataItem.find_by_admin_data_item_id(admin_data_item_id)
end
def self.find_ay(ay:)
return @memo["year"+ay] if @memo.key? "year"+ay
@memo["year"+ay] ||= AcademicYear.find_by_range(ay)
return @memo["admin" + admin_data_item_id] if @memo.key?("admin" + admin_data_item_id)
@memo["admin" + admin_data_item_id] ||= AdminDataItem.find_by_admin_data_item_id(admin_data_item_id)
end
def self.create_admin_data_value(row:, score:)
school = find_school(dese_id: dese_id(row:))
def self.find_ay(ay:)
return @memo["year" + ay] if @memo.key?("year" + ay)
@memo["year" + ay] ||= AcademicYear.find_by_range(ay)
end
def self.create_admin_data_value(row:, score:, schools:)
school = schools[dese_id(row:).to_i]
admin_data_item_id = admin_data_item(row:)
admin_data_item = find_admin_data_item(admin_data_item_id:)
academic_year = find_ay(ay: ay(row:))
@ -71,7 +69,7 @@ module Dese
likert_score: score,
academic_year:,
school:,
admin_data_item:,
admin_data_item:
)
end
end