Fix problem with dese scraper lumping in 2021-22 data as 2022-23 data.

Deleted unused csvs.  Turned off puts statements in admin loader.
Remove old, now unused admin data loader class.

app/services/dese/five_d_two.rb

@@ -7,7 +7,7 @@ module Dese
     include Dese::Enrollments
     attr_reader :filepaths
 
-    def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', '5D_2_enrollments.csv'),
+    def initialize(filepaths: [Rails.root.join('data', 'admin_data', 'dese', 'enrollments.csv'),
                                Rails.root.join('data', 'admin_data', 'dese', '5D_2_age_staffing.csv')])
       @filepaths = filepaths
     end

app/services/dese/loader.rb

@@ -1,20 +1,24 @@
-require 'csv'
+require "csv"
 
 module Dese
   class Loader
     def self.load_data(filepath:)
+      admin_data_values = []
       CSV.parse(File.read(filepath), headers: true) do |row|
         score = likert_score(row:)
         unless valid_likert_score(likert_score: score)
-          school = School.find_by_dese_id(row['DESE ID']) || School.new(name: 'Non-MCIEA school',
+          # school = School.find_by_dese_id(row['DESE ID']) || School.new(name: 'School not in consortium',
-                                                                        dese_id: row['DESE ID'])
+          #                                                               dese_id: row['DESE ID'])
-          puts "Invalid score: #{score}
+          # puts "Invalid score: #{score}
-        for school: #{school.name}
+          # for school: #{school.name}
-        admin data item #{admin_data_item(row:)} "
+          # admin data item #{admin_data_item(row:)} "
           next
         end
-        create_admin_data_value(row:, score:)
+
+        admin_data_values << create_admin_data_value(row:, score:)
       end
+
+      AdminDataValue.import(admin_data_values.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
     end
 
     private
@@ -24,7 +28,7 @@ module Dese
     end
 
     def self.likert_score(row:)
-      likert_score = (row['Likert Score'] || row['LikertScore'] || row['Likert_Score']).to_f
+      likert_score = (row["Likert Score"] || row["LikertScore"] || row["Likert_Score"]).to_f
       round_up_to_one(likert_score:)
     end
 
@@ -34,15 +38,15 @@ module Dese
     end
 
     def self.ay(row:)
-      row['Academic Year'] || row['AcademicYear']
+      row["Academic Year"] || row["AcademicYear"]
     end
 
     def self.dese_id(row:)
-      row['DESE ID'] || row['Dese ID'] || row['Dese Id']
+      row["DESE ID"] || row["Dese ID"] || row["Dese Id"]
     end
 
     def self.admin_data_item(row:)
-      row['Admin Data Item'] || row['Item ID'] || row['Item Id']
+      row["Admin Data Item"] || row["Item ID"] || row["Item Id"]
     end
 
     def self.create_admin_data_value(row:, score:)
@@ -53,13 +57,14 @@ module Dese
       return if admin_data_item_id.nil? || admin_data_item_id.blank?
 
       admin_data_value = AdminDataValue.find_by(academic_year: AcademicYear.find_by_range(ay(row:)),
-                                                school:,
+        school:,
-                                                admin_data_item: AdminDataItem.find_by_admin_data_item_id(admin_data_item_id))
+        admin_data_item: AdminDataItem.find_by_admin_data_item_id(admin_data_item_id))
       if admin_data_value.present?
         admin_data_value.likert_score = score
         admin_data_value.save
+        nil
       else
-        AdminDataValue.create!(
+        AdminDataValue.new(
           likert_score: score,
           academic_year: AcademicYear.find_by_range(ay(row:)),
           school:,

app/services/dese/scraper.rb

@@ -35,7 +35,7 @@ module Dese
       browser.goto(url)
 
       selectors.each do |key, value|
-        next unless browser.option(text: value).present?
+        return unless browser.option(text: value).present?
 
         browser.select(id: key).select(text: value)
       end