feat: if admin data value is above 5, round down to 5

app/services/dese/loader.rb

@@ -1,18 +1,15 @@
 module Dese
   class Loader
     def self.load_data(filepath:)
+      admin_data_values = []
       CSV.parse(File.read(filepath), headers: true) do |row|
         score = likert_score(row:)
-        unless valid_likert_score(likert_score: score)
-          school = School.find_by_dese_id(row['DESE ID']) || School.new(name: 'School not in consortium',
-                                                                        dese_id: row['DESE ID'])
-          puts "Invalid score: #{score}
-        for school: #{school.name}
-        admin data item #{admin_data_item(row:)} "
-          next
-        end
-        create_admin_data_value(row:, score:)
+        next unless valid_likert_score(likert_score: score)
+
+        admin_data_values << create_admin_data_value(row:, score:)
       end
+
+      AdminDataValue.import(admin_data_values.flatten.compact, batch_size: 1_000, on_duplicate_key_update: :all)
     end
 
     private
@@ -23,12 +20,7 @@ module Dese
 
     def self.likert_score(row:)
       likert_score = (row['Likert Score'] || row['LikertScore'] || row['Likert_Score']).to_f
-      round_up_to_one(likert_score:)
-    end
-
-    def self.round_up_to_one(likert_score:)
-      likert_score = 1 if likert_score.positive? && likert_score < 1
-      likert_score
+      likert_score.round_up_to_one.round_down_to_five
     end
 
     def self.ay(row:)
@@ -36,11 +28,11 @@ module Dese
     end
 
     def self.dese_id(row:)
-      row['DESE ID'] || row['Dese ID'] || row['Dese Id']
+      row['DESE ID'] || row['Dese ID'] || row['Dese Id'] || row['School ID']
     end
 
     def self.admin_data_item(row:)
-      row['Admin Data Item'] || row['Item ID'] || row['Item Id']
+      row['Admin Data Item'] || row['Item ID'] || row['Item Id'] || row['Item  ID']
     end
 
     def self.create_admin_data_value(row:, score:)
@@ -56,8 +48,9 @@ module Dese
       if admin_data_value.present?
         admin_data_value.likert_score = score
         admin_data_value.save
+        nil
       else
-        AdminDataValue.create!(
+        AdminDataValue.new(
           likert_score: score,
           academic_year: AcademicYear.find_by_range(ay(row:)),
           school:,
@@ -68,7 +61,6 @@ module Dese
 
     private_class_method :valid_likert_score
     private_class_method :likert_score
-    private_class_method :round_up_to_one
     private_class_method :ay
     private_class_method :dese_id
     private_class_method :admin_data_item

config/initializers/float_monkey_patches.rb

@@ -0,0 +1,19 @@
+module FloatMonkeyPatches
+  def round_up_to_one
+    if positive? && self < 1
+      1.0
+    else
+      self
+    end
+  end
+
+  def round_down_to_five
+    if positive? && self > 5
+      5.0
+    else
+      self
+    end
+  end
+end
+
+Float.include FloatMonkeyPatches

spec/fixtures/sample_four_d_data.csv

@@ -17,7 +17,7 @@ Assabet Valley Regional Vocational Technical - Assabet Valley Vocational High Sc
 Athol-Royalston - Athol High,6150505,13,14.5,0,21.7,11.6,0,37.7,0,0,1.4,69,5.25333333333333,5,a-cgpr-i1,2020-21
 Atlantis Charter (District) - Atlantis Charter School,4910550,19,38.1,0,22.6,0,4.8,8.3,2.4,0,4.8,84,4.94933333333333,4.95,a-cgpr-i1,2020-21
 Attleboro - Attleboro Community Academy,160515,0,1.9,0,20.4,14.8,1.9,55.6,5.6,0,0,54,5.04533333333333,5,a-cgpr-i1,2020-21
-Attleboro - Attleboro High,160505,27,28.3,0,18,1.3,0,20.3,3.6,0.3,1.3,389,5.06133333333333,5,a-cgpr-i1,2020-21
+Attleboro - Attleboro High,160505,27,28.3,0,18,1.3,0,20.3,3.6,0.3,1.3,389,6.06133333333333,6.06133333333333,a-cgpr-i1,2020-21
 Auburn - Auburn Senior High,170505,24.4,48.1,0,10,3.1,0,11.9,1.9,0,0.6,160,5.2,5,a-cgpr-i1,2020-21
 Avon - Avon Middle High School,180510,26.3,42.1,0,18.4,0,0,7.9,2.6,0,2.6,38,5.05066666666667,5,a-cgpr-i1,2020-21
 Ayer Shirley School District - Ayer Shirley Regional High School,6160505,26.6,38,0,10.1,1.3,0,6.3,8.9,0,8.9,79,4.38933333333333,4.39,a-cgpr-i1,2020-21
@@ -413,7 +413,7 @@ Assabet Valley Regional Vocational Technical - Assabet Valley Vocational High Sc
 Athol-Royalston - Athol High,6150505,14.1,15.4,0,30.8,5.1,1.3,25.6,5.1,1.3,1.3,78,4.92266666666667,4.92,a-cgpr-i1,2019-20
 Atlantis Charter (District) - Atlantis Charter School,4910550,25.9,37.9,0,27.6,0,0,6.9,1.7,0,0,58,5.24266666666667,5,a-cgpr-i1,2019-20
 Attleboro - Attleboro Community Academy,160515,0,3,0,27.3,12.1,3,45.5,0,9.1,0,33,4.848,4.85,a-cgpr-i1,2019-20
-Attleboro - Attleboro High,160505,30.9,29.4,0,18.6,3.1,0.3,13.4,3.1,0.8,0.5,388,5.104,5,a-cgpr-i1,2019-20
+Attleboro - Attleboro High,160505,30.9,29.4,0,18.6,3.1,0.3,13.4,3.1,0.8,0.5,388,0.1,0.1,a-cgpr-i1,2019-20
 Auburn - Auburn Senior High,170505,27.4,33.5,0.6,15.1,1.7,0,9.5,2.8,0,9.5,179,4.68266666666667,4.68,a-cgpr-i1,2019-20
 Avon - Avon Middle High School,180510,23.4,46.8,0,10.6,2.1,0,12.8,0,0,4.3,47,5.104,5,a-cgpr-i1,2019-20
 Ayer Shirley School District - Ayer Shirley Regional High School,6160505,27,40,0,13,2,0,9,2,3,4,100,4.85333333333333,4.85,a-cgpr-i1,2019-20
@@ -1206,7 +1206,7 @@ Assabet Valley Regional Vocational Technical - Assabet Valley Vocational High Sc
 Athol-Royalston - Athol High,6150505,17.4,14,0,34.9,5.8,0,16.3,3.5,5.8,2.3,86,4.71466666666667,4.71,a-cgpr-i1,2017-18
 Atlantis Charter (District) - Atlantis Charter School,4910550,13.2,44.7,0,42.1,0,0,0,0,0,0,38,5.33333333333333,5,a-cgpr-i1,2017-18
 Attleboro - Attleboro Community Academy,160515,0,2,0,28,4,2,52,0,4,8,50,4.69333333333333,4.69,a-cgpr-i1,2017-18
-Attleboro - Attleboro High,160505,29.6,31.8,0.3,19.5,3.3,0,9,5.2,0.8,0.5,365,4.98666666666667,4.99,a-cgpr-i1,2017-18
+Attleboro - Attleboro High,160505,29.6,31.8,0.3,19.5,3.3,0,9,5.2,0.8,0.5,365,0,0,a-cgpr-i1,2017-18
 Auburn - Auburn Senior High,170505,20.4,42.6,0.6,24.7,1.9,0,6.2,3.7,0,0,162,5.14133333333333,5,a-cgpr-i1,2017-18
 Avon - Avon Middle High School,180510,25,47.2,0,16.7,5.6,0,5.6,0,0,0,36,5.33866666666667,5,a-cgpr-i1,2017-18
 Ayer Shirley School District - Ayer Shirley Regional High School,6160505,16.5,48.2,0,20,2.4,0,10.6,1.2,0,1.2,85,5.21066666666667,5,a-cgpr-i1,2017-18
@@ -1602,7 +1602,7 @@ Ashland - Ashland High,140505,33.5,50.5,0,8.8,1.6,,4.9,0,0,0.5,182,5.296,5,a-cgp
 Assabet Valley Regional Vocational Technical - Assabet Valley Vocational High School,8010605,15.7,19.5,0.4,15.3,6.1,,39.8,2.7,0,0.4,261,5.16266666666667,5,a-cgpr-i1,2016-17
 Athol-Royalston - Athol High,6150505,19.4,11.9,1.5,46.3,4.5,,11.9,3,1.5,0,67,5.09333333333333,5,a-cgpr-i1,2016-17
 Attleboro - Attleboro Community Academy,160515,0,0,3.3,30,6.7,,36.7,6.7,6.7,10,30,4.09066666666667,4.09,a-cgpr-i1,2016-17
-Attleboro - Attleboro High,160505,26.7,31.1,0,21.3,1.6,,12,5.4,0.3,1.6,367,4.944,4.94,a-cgpr-i1,2016-17
+Attleboro - Attleboro High,160505,26.7,31.1,0,21.3,1.6,,12,5.4,0.3,1.6,367,,,a-cgpr-i1,2016-17
 Auburn - Auburn Senior High,170505,35.1,42.4,0,11.9,0.7,,7.9,2,0,0,151,5.22666666666667,5,a-cgpr-i1,2016-17
 Avon - Avon Middle High School,180510,20.8,54.2,0,4.2,12.5,,2.1,0,4.2,2.1,48,5.00266666666667,5,a-cgpr-i1,2016-17
 Ayer Shirley School District - Ayer Shirley Regional High School,6160505,20.3,36.7,0,25.3,0,,7.6,2.5,0,7.6,79,4.79466666666667,4.79,a-cgpr-i1,2016-17

spec/services/dese/loader_spec.rb

@@ -1,15 +1,15 @@
-require 'rails_helper'
+require "rails_helper"
 RSpec.describe Dese::Loader do
-  let(:path_to_admin_data) { Rails.root.join('spec', 'fixtures', 'sample_four_d_data.csv') }
+  let(:path_to_admin_data) { Rails.root.join("spec", "fixtures", "sample_four_d_data.csv") }
 
-  let(:ay_2022_23) { create(:academic_year, range: '2022-23') }
-  let(:ay_2021_22) { create(:academic_year, range: '2021-22') }
-  let(:ay_2020_21) { create(:academic_year, range: '2020-21') }
-  let(:ay_2019_20) { create(:academic_year, range: '2019-20') }
-  let(:ay_2018_19) { create(:academic_year, range: '2018-19') }
-  let(:ay_2017_18) { create(:academic_year, range: '2017-18') }
-  let(:ay_2016_17) { create(:academic_year, range: '2016-17') }
-  let(:four_d) { create(:admin_data_item, admin_data_item_id: 'a-cgpr-i1') }
+  let(:ay_2022_23) { create(:academic_year, range: "2022-23") }
+  let(:ay_2021_22) { create(:academic_year, range: "2021-22") }
+  let(:ay_2020_21) { create(:academic_year, range: "2020-21") }
+  let(:ay_2019_20) { create(:academic_year, range: "2019-20") }
+  let(:ay_2018_19) { create(:academic_year, range: "2018-19") }
+  let(:ay_2017_18) { create(:academic_year, range: "2017-18") }
+  let(:ay_2016_17) { create(:academic_year, range: "2016-17") }
+  let(:four_d) { create(:admin_data_item, admin_data_item_id: "a-cgpr-i1") }
   let(:attleboro) { create(:school, dese_id: 160_505) }
   let(:winchester) { create(:school, dese_id: 3_440_505) }
   let(:milford) { create(:school, dese_id: 1_850_505) }
@@ -35,12 +35,12 @@ RSpec.describe Dese::Loader do
   after :each do
     # DatabaseCleaner.clean
   end
-  context 'when running the loader' do
+  context "when running the loader" do
     before :each do
       Dese::Loader.load_data filepath: path_to_admin_data
     end
 
-    it 'load the correct admin data values' do
+    it "load the correct admin data values" do
       expect(AdminDataValue.find_by(school: winchester, admin_data_item: four_d,
                                     academic_year: ay_2016_17).likert_score).to eq 5
       expect(AdminDataValue.find_by(school: attleboro, admin_data_item: four_d,
@@ -53,14 +53,31 @@ RSpec.describe Dese::Loader do
                                     academic_year: ay_2020_21).likert_score).to eq 4.8
     end
 
-    it 'loads the correct number of items' do
-      expect(AdminDataValue.count).to eq 25
+    it "loads the correct number of items" do
+      expect(AdminDataValue.count).to eq 23
     end
 
-    it 'is idempotent' do
+    it "cap maximum likert score to 5" do
+      expect(AdminDataValue.find_by(school: attleboro, admin_data_item: four_d,
+                                    academic_year: ay_2020_21).likert_score).to eq 5
+    end
+
+    it "any number between 0 and 1 is rounded to 1" do
+      expect(AdminDataValue.find_by(school: attleboro, admin_data_item: four_d,
+                                    academic_year: ay_2019_20).likert_score).to eq 1
+    end
+
+    it "ignores blank and zero values" do
+      expect(AdminDataValue.find_by(school: attleboro, admin_data_item: four_d,
+                                    academic_year: ay_2017_18)).to eq nil
+      expect(AdminDataValue.find_by(school: attleboro, admin_data_item: four_d,
+                                    academic_year: ay_2016_17)).to eq nil
+    end
+
+    it "is idempotent" do
       Dese::Loader.load_data filepath: path_to_admin_data
 
-      expect(AdminDataValue.count).to eq 25
+      expect(AdminDataValue.count).to eq 23
     end
   end
 end