Add automated data cleaning. Modify SurveyItemValues class to use regex

instead of hard coded values.  Produce a clean csv and a csv with all
the removed values and columns with reason for removal. Add script for
running cleaning for each project

app/models/gender.rb

@@ -1,2 +1,5 @@
 class Gender < ApplicationRecord
+  scope :gender_hash, lambda {
+    all.map { |gender| [gender.qualtrics_code, gender] }.to_h
+  }
 end

app/models/school.rb

@@ -8,6 +8,7 @@ class School < ApplicationRecord
   validates :name, presence: true
 
   scope :alphabetic, -> { order(name: :asc) }
+  scope :school_hash, -> { all.map { |school| [school.dese_id, school] }.to_h }
 
   include FriendlyId
   friendly_id :name, use: [:slugged]

app/models/survey_item.rb

@@ -18,17 +18,61 @@ class SurveyItem < ActiveRecord::Base
   scope :student_survey_items, lambda {
     where("survey_items.survey_item_id LIKE 's-%'")
   }
+  scope :standard_survey_items, lambda {
+    where("survey_items.survey_item_id LIKE 's-%-q%'")
+  }
   scope :teacher_survey_items, lambda {
     where("survey_items.survey_item_id LIKE 't-%'")
   }
-  scope :short_form_items, lambda {
+  scope :short_form_survey_items, lambda {
     where(on_short_form: true)
   }
-  scope :early_education_surveys, lambda {
+  scope :early_education_survey_items, lambda {
     where("survey_items.survey_item_id LIKE '%-%-es%'")
   }
 
+  scope :survey_items_for_grade, lambda { |school, academic_year, grade|
+    includes(:survey_item_responses)
+      .where("survey_item_responses.grade": grade,
+             "survey_item_responses.school": school,
+             "survey_item_responses.academic_year": academic_year).distinct
+  }
+
+  scope :survey_item_ids_for_grade, lambda { |school, academic_year, grade|
+    survey_items_for_grade(school, academic_year, grade).pluck(:survey_item_id)
+  }
+
+  scope :survey_items_for_grade_and_subcategory, lambda { |school, academic_year, grade, subcategory|
+    includes(:survey_item_responses)
+      .where(
+        survey_item_id: subcategory.survey_items.pluck(:survey_item_id),
+        "survey_item_responses.school": school,
+        "survey_item_responses.academic_year": academic_year,
+        "survey_item_responses.grade": grade
+      )
+  }
+
+  scope :survey_type_for_grade, lambda { |school, academic_year, grade|
+    survey_items_set_by_grade = survey_items_for_grade(school, academic_year, grade).pluck(:survey_item_id).to_set
+    if survey_items_set_by_grade.size > 0 && survey_items_set_by_grade.subset?(early_education_survey_items.pluck(:survey_item_id).to_set)
+      return :early_education
+    end
+
+    :standard
+  }
+
+  # TODO: rename this to Summary
   def description
     Summary.new(survey_item_id, prompt, true)
   end
+
+  def self.survey_type(survey_item_ids:)
+    survey_item_ids = survey_item_ids.to_set
+    return :short_form if survey_item_ids.subset? short_form_survey_items.map(&:survey_item_id).to_set
+    return :early_education if survey_item_ids.subset? early_education_survey_items.map(&:survey_item_id).to_set
+    return :teacher if survey_item_ids.subset? teacher_survey_items.map(&:survey_item_id).to_set
+    return :standard if survey_item_ids.subset? standard_survey_items.map(&:survey_item_id).to_set
+
+    :unknown
+  end
 end