Merge branch 'rpp-response-rate' to bring in changes to test files

app/lib/seeder.rb

@@ -6,13 +6,18 @@ class Seeder
   end
 
   def seed_academic_years(*academic_year_ranges)
+    academic_years = []
     academic_year_ranges.each do |range|
-      AcademicYear.find_or_create_by! range:
+      academic_year = AcademicYear.find_or_initialize_by(range:)
+      academic_years << academic_year
     end
+
+    AcademicYear.import academic_years, on_duplicate_key_update: :all
   end
 
   def seed_districts_and_schools(csv_file)
     dese_ids = []
+    schools = []
     CSV.parse(File.read(csv_file), headers: true) do |row|
       district_name = row['District'].strip
       next if rules.any? do |rule|
@@ -27,21 +32,26 @@ class Seeder
       hs = row['HS?']
 
       district = District.find_or_create_by! name: district_name
-      district.update! slug: district_name.parameterize, qualtrics_code: district_code
+      district.slug = district_name.parameterize
+      district.qualtrics_code = district_code
+      district.save
 
       school = School.find_or_initialize_by(dese_id:, district:)
       school.name = school_name
       school.qualtrics_code = school_code
       school.is_hs = marked? hs
-      school.save!
+      schools << school
     end
 
+    School.import schools, on_duplicate_key_update: :all
+
     Respondent.joins(:school).where.not("school.dese_id": dese_ids).destroy_all
     Survey.joins(:school).where.not("school.dese_id": dese_ids).destroy_all
     School.where.not(dese_id: dese_ids).destroy_all
   end
 
   def seed_surveys(csv_file)
+    surveys = []
     CSV.parse(File.read(csv_file), headers: true) do |row|
       district_name = row['District'].strip
       next if rules.any? do |rule|
@@ -57,41 +67,11 @@ class Seeder
         survey = Survey.find_or_initialize_by(school:, academic_year:)
         is_short_form_school = marked?(short_form)
         survey.form = is_short_form_school ? Survey.forms[:short] : Survey.forms[:normal]
-        survey.save!
-      end
-    end
-  end
-
-  def seed_respondents(csv_file)
-    schools = []
-    CSV.parse(File.read(csv_file), headers: true) do |row|
-      dese_id = row['DESE School ID'].strip.to_i
-
-      district_name = row['District'].strip
-      next if rules.any? do |rule|
-                rule.new(row:).skip_row?
-              end
-
-      district = District.find_or_create_by! name: district_name
-
-      school = School.find_by(dese_id:, district:)
-      schools << school
-
-      academic_years = AcademicYear.all
-      academic_years.each do |academic_year|
-        total_students = row["Total Students for Response Rate (#{academic_year.range})"]
-        total_teachers = row["Total Teachers for Response Rate (#{academic_year.range})"]
-        total_students = remove_commas(total_students)
-        total_teachers = remove_commas(total_teachers)
-        respondent = Respondent.find_or_initialize_by(school:, academic_year:)
-        respondent.total_students = total_students
-        respondent.total_teachers = total_teachers
-        respondent.academic_year = academic_year
-        respondent.save
+        surveys << survey
       end
     end
 
-    Respondent.where.not(school: schools).destroy_all
+    Survey.import surveys, on_duplicate_key_update: :all
   end
 
   def seed_sqm_framework(csv_file)
@@ -164,6 +144,18 @@ class Seeder
     DemographicLoader.load_data(filepath: csv_file)
   end
 
+  def seed_enrollment(csv_file)
+    EnrollmentLoader.load_data(filepath: csv_file)
+  end
+
+  def seed_staffing(csv_file)
+    StaffingLoader.load_data(filepath: csv_file)
+    missing_staffing_for_current_year = Respondent.where(academic_year: AcademicYear.order(:range).last).none? do |respondent|
+      respondent.total_teachers.present?
+    end
+    StaffingLoader.clone_previous_year_data if missing_staffing_for_current_year
+  end
+
   private
 
   def marked?(mark)

app/models/district.rb

@@ -14,4 +14,8 @@ class District < ApplicationRecord
   before_save do
     self.slug ||= name.parameterize
   end
+
+  def self.boston
+    District.find_by_name('Boston')
+  end
 end

app/models/respondent.rb

@@ -3,4 +3,6 @@
 class Respondent < ApplicationRecord
   belongs_to :school
   belongs_to :academic_year
+
+  validates :school, uniqueness: { scope: :academic_year }
 end

app/models/response_rate_calculator.rb

@@ -39,10 +39,6 @@ class ResponseRateCalculator
     Survey.find_by(school:, academic_year:)
   end
 
-  def raw_response_rate
-    (average_responses_per_survey_item / total_possible_responses.to_f * 100).round
-  end
-
   def average_responses_per_survey_item
     response_count / survey_item_count.to_f
   end

app/models/student_response_rate_calculator.rb

@@ -3,6 +3,27 @@
 class StudentResponseRateCalculator < ResponseRateCalculator
   private
 
+  def raw_response_rate
+    # def rate
+    # check to see if enrollment data is available
+    #   if not, run the dese loader to get the data
+    #   then upload the enrollment data into the db
+    #
+    # if you still don't see enrollment for the school, raise an error and return 100 from this method
+    #
+    #  Get the enrollment information from the db
+    #  Get the list of all grades
+    #  For each grade, get the survey items with data
+    #
+    #
+    #  All methods below will need to specify a grade
+
+    grades_with_sufficient_responses.map do |grade|
+      puts "Grade: #{grade}"
+    end
+    (average_responses_per_survey_item / total_possible_responses.to_f * 100).round
+  end
+
   def survey_item_count
     @survey_item_count ||= begin
       survey_items = SurveyItem.includes(%i[scale
@@ -34,4 +55,15 @@ class StudentResponseRateCalculator < ResponseRateCalculator
       total_responses.total_students
     end
   end
+
+  def grades_with_sufficient_responses
+    SurveyItemResponse.where(school:, academic_year:,
+                             survey_item: subcategory.survey_items.student_survey_items).where.not(grade: nil)
+                      .group(:grade)
+                      .select(:response_id)
+                      .distinct(:response_id)
+                      .count.reject do |_key, value|
+      value < 10
+    end.keys
+  end
 end

app/models/subcategory.rb

@@ -4,6 +4,7 @@ class Subcategory < ActiveRecord::Base
   belongs_to :category, counter_cache: true
 
   has_many :measures
+  has_many :survey_items, through: :measures
 
   def score(school:, academic_year:)
     scores = measures.map do |measure|

app/models/survey_item.rb

@@ -24,7 +24,41 @@ class SurveyItem < ActiveRecord::Base
   scope :short_form_items, lambda {
     where(on_short_form: true)
   }
+  scope :early_education_surveys, lambda {
+    where("survey_item_id LIKE '%-%-es%'")
+  }
 
+  scope :survey_items_for_grade, lambda { |school, academic_year, grade|
+    includes(:survey_item_responses)
+      .where("survey_item_responses.grade": grade,
+             "survey_item_responses.school": school,
+             "survey_item_responses.academic_year": academic_year).distinct
+  }
+
+  scope :survey_item_ids_for_grade, lambda { |school, academic_year, grade|
+    survey_items_for_grade(school, academic_year, grade).pluck(:survey_item_id)
+  }
+
+  scope :survey_items_for_grade_and_subcategory, lambda { |school, academic_year, grade, subcategory|
+    includes(:survey_item_responses)
+      .where(
+        survey_item_id: subcategory.survey_items.pluck(:survey_item_id),
+        "survey_item_responses.school": school,
+        "survey_item_responses.academic_year": academic_year,
+        "survey_item_responses.grade": grade
+      )
+  }
+
+  scope :survey_type_for_grade, lambda { |school, academic_year, grade|
+    survey_items_set_by_grade = survey_items_for_grade(school, academic_year, grade).pluck(:survey_item_id).to_set
+    if survey_items_set_by_grade.size > 0 && survey_items_set_by_grade.subset?(early_education_surveys.pluck(:survey_item_id).to_set)
+      return :early_education
+    end
+
+    :regular
+  }
+
+  # TODO: rename this to Summary
   def description
     DataAvailability.new(survey_item_id, prompt, true)
   end

app/models/survey_item_response.rb

@@ -13,17 +13,16 @@ class SurveyItemResponse < ActiveRecord::Base
   has_one :measure, through: :survey_item
 
   scope :exclude_boston, lambda {
-                           boston = District.find_by_name('Boston')
-                           where.not(school: boston.schools) if boston.present?
+                           where.not(school: District.boston.schools) if District.boston.present?
                          }
 
-  scope :averages_for_grade, ->(survey_items, school, academic_year, grade) {
-            SurveyItemResponse.where(survey_item: survey_items, school:,
-                                                academic_year: , grade:).group(:survey_item).average(:likert_score)
+  scope :averages_for_grade, lambda { |survey_items, school, academic_year, grade|
+    SurveyItemResponse.where(survey_item: survey_items, school:,
+                             academic_year:, grade:).group(:survey_item).average(:likert_score)
   }
 
-  scope :averages_for_gender, ->(survey_items, school, academic_year, gender) {
-            SurveyItemResponse.where(survey_item: survey_items, school:,
-                                                academic_year: , gender:).group(:survey_item).average(:likert_score)
+  scope :averages_for_gender, lambda { |survey_items, school, academic_year, gender|
+    SurveyItemResponse.where(survey_item: survey_items, school:,
+                             academic_year:, gender:).group(:survey_item).average(:likert_score)
   }
 end

app/models/teacher_response_rate_calculator.rb

@@ -26,4 +26,8 @@ class TeacherResponseRateCalculator < ResponseRateCalculator
       total_responses.total_teachers
     end
   end
+
+  def raw_response_rate
+    (average_responses_per_survey_item / total_possible_responses.to_f * 100).round
+  end
 end

app/services/admin_data_loader.rb

@@ -43,6 +43,7 @@ class AdminDataLoader
   end
 
   def self.create_admin_data_value(row:, score:)
+    # byebug unless %w[a-vale-i1 a-sust-i3].include? admin_data_item(row:)
     AdminDataValue.create!(likert_score: score,
                            academic_year: AcademicYear.find_by_range(ay(row:)),
                            school: School.find_by_dese_id(dese_id(row:).to_i),

app/services/dese/one_a_three.rb

@@ -41,7 +41,7 @@ module Dese
     def run_a_pcom_i3
       filepath = filepaths[1]
       headers = ['Raw likert calculation', 'Likert Score', 'Admin Data Item', 'Academic Year', 'School Name', 'DESE ID',
-                 'African American (%)', 'Asian (%)', 'Hispanic (%)', 'White (%)', 'Native Hawaiian, Pacific Islander (%)',
+                 'African American (%)', 'Asian (%)', 'Hispanic (%)', 'White (%)', 'Native Amertican (%)', 'Native Hawaiian, Pacific Islander (%)',
                  'Multi-Race,Non-Hispanic (%)', 'Females (%)', 'Males (%)', 'FTE Count']
       write_headers(filepath:, headers:)
 

app/services/dese/scraper.rb

@@ -1,6 +1,6 @@
 module Dese
   module Scraper
-    DELAY = 20
+    DELAY = 20 # The dese site will block you if you hit it too many times in a short period of time
 
     Prerequisites = Struct.new('Prerequisites', :filepath, :url, :selectors, :submit_id, :admin_data_item_id,
                                :calculation)

app/services/enrollment_loader.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+class EnrollmentLoader
+  def self.load_data(filepath:)
+    schools = []
+    enrollments = []
+    CSV.parse(File.read(filepath), headers: true) do |row|
+      row = EnrollmentRowValues.new(row:)
+
+      next unless row.school.present? && row.academic_year.present?
+
+      schools << row.school
+
+      enrollments << create_enrollment_entry(row:)
+    end
+
+    # It's possible that instead of updating all columns on duplicate key, we could just update the student columns and leave total_teachers alone. Right now enrollment data loads before staffing data so it works correctly.
+    Respondent.import enrollments, batch_size: 1000,
+                                   on_duplicate_key_update: %i[pk k one two three four five six seven eight nine ten eleven twelve total_students]
+
+    Respondent.where.not(school: schools).destroy_all
+  end
+
+  private
+
+  def self.create_enrollment_entry(row:)
+    respondent = Respondent.find_or_initialize_by(school: row.school, academic_year: row.academic_year)
+    respondent.pk = row.pk
+    respondent.k = row.k
+    respondent.one = row.one
+    respondent.two = row.two
+    respondent.three = row.three
+    respondent.four = row.four
+    respondent.five = row.five
+    respondent.six = row.six
+    respondent.seven = row.seven
+    respondent.eight = row.eight
+    respondent.nine = row.nine
+    respondent.ten = row.ten
+    respondent.eleven = row.eleven
+    respondent.twelve = row.twelve
+    respondent.total_students = row.total_students
+    respondent
+  end
+
+  private_class_method :create_enrollment_entry
+end
+
+class EnrollmentRowValues
+  attr_reader :row
+
+  def initialize(row:)
+    @row = row
+  end
+
+  def school
+    @school ||= begin
+      dese_id = row['DESE ID'].try(:strip).to_i
+      School.find_by_dese_id(dese_id)
+    end
+  end
+
+  def academic_year
+    @academic_year ||= begin
+      year = row['Academic Year']
+      AcademicYear.find_by_range(year)
+    end
+  end
+
+  def pk
+    row['PK'] || row['pk']
+  end
+
+  def k
+    row['K'] || row['k']
+  end
+
+  def one
+    row['1']
+  end
+
+  def two
+    row['2']
+  end
+
+  def three
+    row['3']
+  end
+
+  def four
+    row['4']
+  end
+
+  def five
+    row['5']
+  end
+
+  def six
+    row['6']
+  end
+
+  def seven
+    row['7']
+  end
+
+  def eight
+    row['8']
+  end
+
+  def nine
+    row['9']
+  end
+
+  def ten
+    row['10']
+  end
+
+  def eleven
+    row['11']
+  end
+
+  def twelve
+    row['12']
+  end
+
+  def total_students
+    row['Total'].gsub(',', '').to_i
+  end
+end

app/services/response_rate_loader.rb

@@ -4,11 +4,7 @@ class ResponseRateLoader
   def self.reset(schools: School.all, academic_years: AcademicYear.all, subcategories: Subcategory.all)
     subcategories.each do |subcategory|
       schools.each do |school|
-        next if test_env? && (school != milford)
-
         academic_years.each do |academic_year|
-          next if test_env? && (academic_year != test_year)
-
           process_response_rate(subcategory:, school:, academic_year:)
         end
       end
@@ -17,18 +13,6 @@ class ResponseRateLoader
 
   private
 
-  def self.milford
-    School.find_by_slug 'milford-high-school'
-  end
-
-  def self.test_year
-    AcademicYear.find_by_range '2020-21'
-  end
-
-  def self.rails_env
-    @rails_env ||= ENV['RAILS_ENV']
-  end
-
   def self.process_response_rate(subcategory:, school:, academic_year:)
     student = StudentResponseRateCalculator.new(subcategory:, school:, academic_year:)
     teacher = TeacherResponseRateCalculator.new(subcategory:, school:, academic_year:)
@@ -41,13 +25,5 @@ class ResponseRateLoader
                           meets_teacher_threshold: teacher.meets_teacher_threshold?)
   end
 
-  def self.test_env?
-    rails_env == 'test'
-  end
-
-  private_class_method :milford
-  private_class_method :test_year
-  private_class_method :rails_env
   private_class_method :process_response_rate
-  private_class_method :test_env?
 end

app/services/staffing_loader.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# TODO
+require 'csv'
+
+class StaffingLoader
+  def self.load_data(filepath:)
+    schools = []
+    respondents = []
+    CSV.parse(File.read(filepath), headers: true) do |row|
+      row = StaffingRowValues.new(row:)
+      next unless row.school.present? && row.academic_year.present?
+
+      schools << row.school
+
+      respondents << create_staffing_entry(row:)
+    end
+
+    Respondent.import respondents, batch_size: 1000, on_duplicate_key_update: [:total_teachers]
+    Respondent.where.not(school: schools).destroy_all
+  end
+
+  def self.clone_previous_year_data
+    years = AcademicYear.order(:range).last(2)
+    previous_year = years.first
+    current_year = years.last
+    respondents = []
+    School.all.each do |school|
+      Respondent.where(school:, academic_year: previous_year).each do |respondent|
+        current_respondent = Respondent.find_or_initialize_by(school:, academic_year: current_year)
+        current_respondent.total_teachers = respondent.total_teachers
+        respondents << current_respondent
+      end
+    end
+    Respondent.import respondents, batch_size: 1000, on_duplicate_key_update: [:total_teachers]
+  end
+
+  private
+
+  def self.create_staffing_entry(row:)
+    respondent = Respondent.find_or_initialize_by(school: row.school, academic_year: row.academic_year)
+    respondent.total_teachers = row.fte_count
+    respondent
+  end
+
+  private_class_method :create_staffing_entry
+end
+
+class StaffingRowValues
+  attr_reader :row
+
+  def initialize(row:)
+    @row = row
+  end
+
+  def school
+    @school ||= begin
+      dese_id = row['DESE ID'].strip.to_i
+      School.find_by_dese_id(dese_id)
+    end
+  end
+
+  def academic_year
+    @academic_year ||= begin
+      year = row['Academic Year']
+      AcademicYear.find_by_range(year)
+    end
+  end
+
+  def fte_count
+    row['FTE Count']
+  end
+end