Load teacher survey responses

app/services/survey_response_aggregator.rb

@@ -7,9 +7,8 @@ end
 class SurveyResponseAggregator
   def self.score(academic_year:, school:, measure:)
     SurveyItemResponse
-      .where(academic_year: academic_year)
-      .where(school: school)
-      .filter { |survey_response| survey_response.survey_item.measure == measure }
+      .where(academic_year: academic_year, school: school)
+      .filter { |survey_item_response| survey_item_response.survey_item.measure == measure }
       .map { |survey_response| survey_response.likert_score }
       .average
   end

app/services/survey_responses_data_loader.rb

@@ -0,0 +1,68 @@
+require 'csv'
+
+class SurveyResponsesDataLoader
+  def self.load_data(filepath:)
+    csv_file = File.read(filepath)
+
+    parsed_csv_file = CSV.parse(csv_file, headers: true)
+    survey_items = parsed_csv_file.headers
+                         .filter { |header| !header.nil? }
+                         .filter { |header| header.start_with? 't-' }
+                         .map { |survey_item_id| SurveyItem.find_by_survey_item_id survey_item_id }
+
+    parsed_csv_file.each do |row|
+      process_row row: row, survey_items: survey_items
+    end
+  end
+
+  private
+
+  def self.process_row(row:, survey_items:)
+    response_date = Date.parse(row['Recorded Date'])
+    academic_year = academic_year date: response_date
+
+    response_id = row['Response ID']
+
+    school_code = row['school_code']
+    return if school_code.nil?
+
+    school = school(row: row)
+    return if school.nil?
+
+    return unless SurveyItemResponse.find_by_response_id(response_id).nil?
+
+    survey_items.each do |survey_item|
+      likert_score = row[survey_item.survey_item_id]
+      next if likert_score.nil?
+      SurveyItemResponse.create(
+        response_id: response_id,
+        academic_year: academic_year,
+        school: school,
+        survey_item: survey_item,
+        likert_score: likert_score
+      )
+    end
+
+  end
+
+  def self.school(row:)
+    district_code = row['district_code']
+    school_code = row['school_code']
+    return nil if school_code.nil?
+
+    School
+       .where({district: District.find_by_qualtrics_code(district_code), qualtrics_code: school_code})
+       .first
+  end
+
+  def self.academic_year(date:)
+    if date.month > 7
+      ay_range_start = date.year
+      ay_range_end = date.year + 1
+    else
+      ay_range_start = date.year - 1
+      ay_range_end = date.year
+    end
+    AcademicYear.find_by_range("#{ay_range_start}-#{ay_range_end.to_s[2, 3]}")
+  end
+end

db/seeds.rb

@@ -19,6 +19,7 @@ CSV.parse(qualtrics_district_and_school_code_key, headers: true).each do |row|
   else
     district.update(qualtrics_code: district_code) if district.qualtrics_code.nil?
   end
+  district = District.find_by_name(district_name)
 
   school = School.find_by_slug(school_slug)
   if school.nil?

lib/tasks/data.rake

@@ -36,6 +36,14 @@ require 'csv'
 namespace :data do
   @year = 2019
 
+  desc "load survey responses"
+  task load_survey_responses: :environment do
+    filepath = Rails.root.join('data', 'survey_results', '2020-21_teacher_survey_responses.csv')
+    puts "=====================> Loading data from csv at path: #{filepath}"
+    SurveyResponsesDataLoader.load_data filepath: filepath
+    puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses"
+  end
+
   desc "Load in all data"
   task load: :environment do
     # return if School.count > 0

spec/fixtures/test_2020-21_teacher_survey_responses.csv

@@ -0,0 +1,6 @@
+Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,Recorded Date,Response ID,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district_code,school_code,,,t-prep-q1,t-prep-q2,t-prep-q3,t-pcom-q1,t-pcom-q2,t-pcom-q3,t-pcom-q4,t-pcom-q5,t-ieff-q1,t-ieff-q2,t-ieff-q3,t-ieff-q4,t-qupd-q3,t-qupd-q2,t-qupd-q1,t-qupd-q4,t-coll-q1,t-coll-q2,t-coll-q3,t-prtr-q1,t-prtr-q2,t-prtr-q3,t-inle-q1,"t-inle-q2",t-inle-q3,t-pvic-q1,t-pvic-q2,t-pvic-q3,t-psup-q1,t-psup-q2,t-psup-q3,t-psup-q4,t-acch-q1,t-acch-q2,t-acch-q3,t-reso-q1,t-reso-q2,t-reso-q3,t-reso-q4,t-reso-q5,t-sust-q1,t-sust-q2,t-sust-q3,t-sust-q4,t-curv-q1,t-curv-q2,t-curv-q3,t-curv-q4,t-cure-q1,t-cure-q2,t-cure-q3,t-cure-q4,t-peng-q1,t-peng-q2,t-peng-q3,t-peng-q4,t-ceng-q1,t-ceng-q2,t-ceng-q3,t-ceng-q4,#N/A,t-sach-q2,t-sach-q3,t-psol-q1,t-psol-q2,t-psol-q3,t-expa-q2,t-expa-q3,t-phya-q2,t-phya-q3,,,,
+2020-10-09 9:42:45,2020-10-09 11:08:57,0,50.204.125.194,93,5172,0,2020-10-16 11:09:03,survey_response_1,,,,,,,anonymous,EN,1,1,4,,,,,,,3,,,,,,,,,,,3,3,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,,,,,,,,,,,,
+2020-10-09 9:21:30,2020-10-09 14:28:09,0,50.204.125.194,24,18398,0,2020-10-16 14:28:18,survey_response_2,,,,,,,anonymous,EN,1,1,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+2020-12-06 8:28:00,2020-12-06 8:36:52,0,73.61.140.32,100,532,1,2020-12-06 8:36:52,survey_response_3,,,,,41.92610168,-71.30110168,anonymous,EN,1,2,2,,5,5,5,5,5,5,5,1,5,5,5,5,3,2,1,5,4,5,5,4,4,5,5,5,5,3,3,3,3,1,3,3,4,3,2,4,4,4,4,3,3,2,2,2,4,4,4,3,2,3,2,5,4,5,5,3,4,4,3,5,2,1,3,4,3,3,4,4,3,3,,9,9,
+2020-12-06 8:41:41,2020-12-06 8:51:25,0,96.230.183.162,100,584,1,2020-12-06 8:51:25,survey_response_4,,,,,41.98530579,-71.5184021,anonymous,EN,1,2,2,,5,5,5,4,4,4,4,1,4,4,4,4,3,3,3,2,5,2,4,5,4,5,4,4,4,3,3,3,3,4,4,3,3,3,3,3,4,3,3,4,4,3,2,4,4,4,4,4,3,2,3,4,5,3,2,2,4,4,3,3,4,3,3,3,3,3,3,4,3,3,,9,6,
+2020-12-06 8:44:42,2020-12-06 8:55:58,0,216.41.82.178,100,675,1,2020-12-06 8:55:58,survey_response_5,,,,,41.92610168,-71.30110168,anonymous,EN,1,2,2,,5,4,4,2,2,4,1,2,4,4,4,4,3,3,3,2,4,3,5,4,3,4,3,3,4,2,2,4,4,2,4,3,4,4,3,3,2,3,2,3,3,2,4,4,3,4,4,3,2,2,4,4,5,2,3,1,4,4,3,3,3,2,2,4,4,3,3,3,3,3,,7,5,"These questions are very difficult to address during a PANDEMIC!  So it is not your average typical year.  Our principal may not be as concerned about academics, because they are too busy navigating the complexities COVID has caused.  I am honestly unsure why we would get surveyed during this time. "

spec/services/survey_response_aggregator_spec.rb

@@ -1,6 +1,6 @@
 require 'rails_helper'
 
-describe SurveyResponseAggregator, type: :model do
+describe SurveyResponseAggregator do
   let(:category) { SqmCategory.create }
   let(:subcategory) { Subcategory.create sqm_category: category }
 

spec/services/survey_responses_data_loader_spec.rb

@@ -0,0 +1,66 @@
+require 'rails_helper'
+
+describe SurveyResponsesDataLoader do
+  let(:path_to_csv) { Rails.root.join('spec', 'fixtures', 'test_2020-21_teacher_survey_responses.csv') }
+
+  let(:ay_2020_21) { AcademicYear.find_by_range '2020-21' }
+
+  let(:attleboro_high_school) { School.find_by_slug 'attleboro-high-school' }
+
+  let(:t_pcom_q3) { SurveyItem.find_by_survey_item_id 't-pcom-q3' }
+  let(:t_pcom_q2) { SurveyItem.find_by_survey_item_id 't-pcom-q2' }
+
+  describe 'self.load_data' do
+    before :each do
+      SurveyResponsesDataLoader.load_data filepath: path_to_csv
+    end
+
+    it 'loads the csv data into the database' do
+      expect(SurveyItemResponse.count).to be > 0
+    end
+
+    it 'assigns the academic year to the survey item responses' do
+      expect(SurveyItemResponse.first.academic_year).to eq ay_2020_21
+    end
+
+    it 'assigns the school to the survey item responses' do
+      expect(SurveyItemResponse.first.school).to eq attleboro_high_school
+    end
+
+    it 'loads all the survey item responses for a given survey response' do
+      expect(SurveyItemResponse.where(response_id: 'survey_response_1').count).to eq 5
+      expect(SurveyItemResponse.where(response_id: 'survey_response_2').count).to eq 0
+      expect(SurveyItemResponse.where(response_id: 'survey_response_3').count).to eq 69
+    end
+
+    it 'loads all the survey item responses for a given survey item' do
+      expect(SurveyItemResponse.where(survey_item: t_pcom_q2).count).to eq 3
+      expect(SurveyItemResponse.where(survey_item: t_pcom_q3).count).to eq 4
+    end
+
+    it 'captures the likert scores for the survey item responses' do
+      expect(SurveyItemResponse.where(response_id: 'survey_response_1').where(survey_item: t_pcom_q2)).to be_empty
+      expect(SurveyItemResponse.where(response_id: 'survey_response_1').where(survey_item: t_pcom_q3).first.likert_score).to eq 3
+
+      expect(SurveyItemResponse.where(response_id: 'survey_response_2').where(survey_item: t_pcom_q2)).to be_empty
+      expect(SurveyItemResponse.where(response_id: 'survey_response_2').where(survey_item: t_pcom_q3)).to be_empty
+
+      expect(SurveyItemResponse.where(response_id: 'survey_response_3').where(survey_item: t_pcom_q2).first.likert_score).to eq 5
+      expect(SurveyItemResponse.where(response_id: 'survey_response_3').where(survey_item: t_pcom_q3).first.likert_score).to eq 5
+
+      expect(SurveyItemResponse.where(response_id: 'survey_response_4').where(survey_item: t_pcom_q2).first.likert_score).to eq 4
+      expect(SurveyItemResponse.where(response_id: 'survey_response_4').where(survey_item: t_pcom_q3).first.likert_score).to eq 4
+
+      expect(SurveyItemResponse.where(response_id: 'survey_response_5').where(survey_item: t_pcom_q2).first.likert_score).to eq 2
+      expect(SurveyItemResponse.where(response_id: 'survey_response_5').where(survey_item: t_pcom_q3).first.likert_score).to eq 4
+    end
+
+    it 'is idempotent, i.e. loading the data a second time does not duplicate survey item responses' do
+      number_of_survey_item_responses = SurveyItemResponse.count
+
+      SurveyResponsesDataLoader.load_data filepath: path_to_csv
+
+      expect(SurveyItemResponse.count).to eq number_of_survey_item_responses
+    end
+  end
+end