fix: make sure all likert scores get counted even when the survey item id has different capitalization. Add tests for uploading parent data. Change the parent response rate calcuation to count all students in the school instead of just for the grades that were given the student survey

app/lib/seeder.rb

@@ -51,6 +51,7 @@ class Seeder
 
   def seed_sqm_framework(csv_file)
     admin_data_item_ids = []
+    survey_item_ids = []
     CSV.parse(File.read(csv_file), headers: true) do |row|
       next if row["Source"] == "No source"
 
@@ -105,6 +106,7 @@ class Seeder
         survey_item.ideal_low_benchmark = ideal_low if ideal_low
         survey_item.on_short_form = marked? on_short_form
         survey_item.update! prompt: row["Question/item (23-24)"].strip
+        survey_item_ids << survey_item.id
       end
 
       active_admin = row["Active admin & survey items"]
@@ -122,6 +124,7 @@ class Seeder
     end
     AdminDataValue.where.not(admin_data_item_id: admin_data_item_ids).delete_all
     AdminDataItem.where.not(id: admin_data_item_ids).delete_all
+    SurveyItem.where.not(id: survey_item_ids).delete_all
   end
 
   def seed_demographics(csv_file)

app/presenters/overview/overview_presenter.rb

@@ -36,15 +36,15 @@ class Overview::OverviewPresenter
   end
 
   def student_response_rate_presenter
-    ResponseRatePresenter.new(focus: :student, school: @school, academic_year: @academic_year)
+    StudentResponseRatePresenter.new(focus: :student, school: @school, academic_year: @academic_year)
   end
 
   def teacher_response_rate_presenter
-    ResponseRatePresenter.new(focus: :teacher, school: @school, academic_year: @academic_year)
+    TeacherResponseRatePresenter.new(focus: :teacher, school: @school, academic_year: @academic_year)
   end
 
   def parent_response_rate_presenter
-    ResponseRatePresenter.new(focus: :parent, school: @school, academic_year: @academic_year)
+    ParentResponseRatePresenter.new(focus: :parent, school: @school, academic_year: @academic_year)
   end
 
   def presenter_for_measure(measure)

app/presenters/parent_response_rate_presenter.rb

@@ -0,0 +1,20 @@
+class ParentResponseRatePresenter < ResponseRatePresenter
+  def initialize(focus:, academic_year:, school:)
+    super(focus:, academic_year:, school:)
+    @survey_items = SurveyItem.parent_survey_items if focus == :parent
+  end
+
+  def actual_count
+    SurveyItemResponse.includes(:parent).where(school:, academic_year:).where.not(parent_id: nil)
+                      .select(:parent_id)
+                      .distinct
+                      .map { |response| response.parent&.number_of_children }
+                      .compact.sum
+  end
+
+  def respondents_count
+    return 0 if respondents.nil?
+
+    respondents.total_students
+  end
+end

app/presenters/response_rate_presenter.rb

@@ -5,13 +5,6 @@ class ResponseRatePresenter
     @focus = focus
     @academic_year = academic_year
     @school = school
-    if focus == :student
-      @survey_items = Measure.all.flat_map do |measure|
-        measure.student_survey_items_with_sufficient_responses(school:, academic_year:)
-      end
-    end
-    @survey_items = SurveyItem.teacher_survey_items if focus == :teacher
-    @survey_items = SurveyItem.parent_survey_items if focus == :parent
   end
 
   def date
@@ -48,36 +41,7 @@ class ResponseRatePresenter
   end
 
   def actual_count
-    if focus == :teacher
-      response_count_for_survey_items(survey_items:)
-    elsif focus == :parent
-      SurveyItemResponse.includes(:parent).where(school:, academic_year:).where.not(parent_id: nil)
-                        .select(:parent_id)
-                        .distinct
-                        .map { |response| response.parent&.number_of_children }
-                        .compact.sum
-    else
-      non_early_ed_items = survey_items - SurveyItem.early_education_survey_items
-      non_early_ed_count = response_count_for_survey_items(survey_items: non_early_ed_items)
-
-      early_ed_items = survey_items & SurveyItem.early_education_survey_items
-      early_ed_count = SurveyItemResponse.where(school:, academic_year:,
-                                                survey_item: early_ed_items)
-                                         .group(:survey_item)
-                                         .select(:response_id)
-                                         .distinct
-                                         .count
-                                         .reduce(0) do |largest, row|
-        count = row[1]
-        if count > largest
-          count
-        else
-          largest
-        end
-      end
-
-      non_early_ed_count + early_ed_count
-    end
+    raise "please implement the method: actual_count"
   end
 
   def response_count_for_survey_items(survey_items:)
@@ -86,11 +50,7 @@ class ResponseRatePresenter
   end
 
   def respondents_count
-    return 0 if respondents.nil?
-
-    count = enrollment if focus == :student || focus == :parent
-    count = respondents.total_teachers if focus == :teacher
-    count
+    raise "please implement the method: respondents_count"
   end
 
   def enrollment

app/presenters/student_response_rate_presenter.rb

@@ -0,0 +1,38 @@
+class StudentResponseRatePresenter < ResponseRatePresenter
+  def initialize(focus:, academic_year:, school:)
+    super(focus:, academic_year:, school:)
+    @survey_items = Measure.all.flat_map do |measure|
+      measure.student_survey_items_with_sufficient_responses(school:, academic_year:)
+    end
+  end
+
+  def actual_count
+    # Early ed surveys are given in batches so they have to be counted separately because we have to account for the same student having a different response id per batch
+    non_early_ed_items = survey_items - SurveyItem.early_education_survey_items
+    non_early_ed_count = response_count_for_survey_items(survey_items: non_early_ed_items)
+
+    early_ed_items = SurveyItem.early_education_survey_items
+    early_ed_count = SurveyItemResponse.where(school:, academic_year:,
+                                              survey_item: early_ed_items)
+                                       .group(:survey_item)
+                                       .select(:response_id)
+                                       .distinct
+                                       .count
+                                       .reduce(0) do |largest, row|
+      count = row[1]
+      if count > largest
+        count
+      else
+        largest
+      end
+    end
+
+    non_early_ed_count + early_ed_count
+  end
+
+  def respondents_count
+    return 0 if respondents.nil?
+
+    enrollment
+  end
+end

app/presenters/teacher_response_rate_presenter.rb

@@ -0,0 +1,16 @@
+class TeacherResponseRatePresenter < ResponseRatePresenter
+  def initialize(focus:, academic_year:, school:)
+    super(focus:, academic_year:, school:)
+    @survey_items = SurveyItem.teacher_survey_items
+  end
+
+  def actual_count
+    response_count_for_survey_items(survey_items:)
+  end
+
+  def respondents_count
+    return 0 if respondents.nil?
+
+    respondents.total_teachers
+  end
+end

app/services/survey_item_values.rb

@@ -3,8 +3,8 @@ class SurveyItemValues
 
   def initialize(row:, headers:, survey_items:, schools:, academic_years: AcademicYear.all)
     @row = row
-    # Remove any newlines in headers
-    headers = headers.map { |item| item.delete("\n") if item.present? }
+    # Remove any newlines in headers and
+    @headers = normalize_headers(headers:)
     @headers = include_all_headers(headers:)
     @survey_items = survey_items
     @schools = schools
@@ -25,6 +25,13 @@ class SurveyItemValues
     copy_data_to_main_column(main: /Gender/i, secondary: /Gender Secondary|Gender-1/i)
   end
 
+  def normalize_headers(headers:)
+    headers
+      .select(&:present?)
+      .map { |item| item.strip }
+      .map { |item| item.downcase if item.match(/[stp]-/i) }
+  end
+
   def copy_data_to_main_column(main:, secondary:)
     main_column = headers.find { |header| main.match(header) }
     row[main_column] = value_from(pattern: secondary) if row[main_column].nil?
@@ -97,7 +104,7 @@ class SurveyItemValues
   end
 
   def likert_score(survey_item_id:)
-    row[survey_item_id] || row["#{survey_item_id}-1"]
+    row[survey_item_id] || row["#{survey_item_id}-1"] || value_from(pattern: /#{survey_item_id}/i)
   end
 
   def school
@@ -196,7 +203,7 @@ class SurveyItemValues
     output = nil
     matches = headers.select do |header|
       pattern.match(header)
-    end.map { |item| item.delete("\n") }
+    end
 
     matches.each do |match|
       output ||= row[match]&.strip

app/services/survey_responses_data_loader.rb

@@ -165,6 +165,7 @@ class SurveyResponsesDataLoader
       .parse(headers)
       .first
       .filter(&:present?)
+      .map(&:downcase)
       .filter { |header| header.start_with?("t-", "s-", "p-") }
   end
 end

data/sqm_framework.csv

@@ -93,6 +93,7 @@ Community & Wellbeing,5,"Measures the development of traits relevant for student
 Community & Wellbeing,5,"Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives. It considers factors like perseverance and determination, participation in arts and literature, and social and emotional health.","Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives.",Civic Engagement,5A,Seeks to determine the degree to which students are prepared to thoughtfully interact with others in a diverse society and work towards social equity. It includes the measures of appreciation for diversity and civic participation. ,Appreciation For Diversity,Draws on anonymous student reports about their level of comfort in considering different perspectives and conflicting opinions.,5A-i,,,Students,,TRUE,TRUE,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,s-sper-q3,,,,Q61,,,,2.48,2.49,2.99,3,3.49,3.5,4.4,4.41,
 Community & Wellbeing,5,"Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives. It considers factors like perseverance and determination, participation in arts and literature, and social and emotional health.","Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives.",Civic Engagement,5A,Seeks to determine the degree to which students are prepared to thoughtfully interact with others in a diverse society and work towards social equity. It includes the measures of appreciation for diversity and civic participation. ,Appreciation For Diversity,Draws on anonymous student reports about their level of comfort in considering different perspectives and conflicting opinions.,5A-i,,,Students,,TRUE,TRUE,"Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?",s-sper-q2,,,,Q60,,,,2.48,2.49,2.99,3,3.49,3.5,4.4,4.41,
 Community & Wellbeing,5,"Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives. It considers factors like perseverance and determination, participation in arts and literature, and social and emotional health.","Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives.",Civic Engagement,5A,Seeks to determine the degree to which students are prepared to thoughtfully interact with others in a diverse society and work towards social equity. It includes the measures of appreciation for diversity and civic participation. ,Appreciation For Diversity,Draws on anonymous student reports about their level of comfort in considering different perspectives and conflicting opinions.,5A-i,,,Students,,TRUE,TRUE,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,s-sper-q1,,,,Q59,,,,2.48,2.49,2.99,3,3.49,3.5,4.4,4.41,
+School Culture,2,"Measures the degree to which the school environment is safe, caring, and academically-oriented.  It considers factors like bullying, student-teacher relationships, and student valuing of learning.","Measures the degree to which the school environment is safe, caring, and academically-oriented.",Relationships,2B,Seeks to determine the degree to which school climate includes supportive student relationships. It includes measures of student sense of belonging and student-teacher relationships. ,Student Sense of Belonging,"Draws on anonymous student and teacher reports about the degree to which they feel understood, supported, and accepted by students and adults at the school.",2B-i,,,Students,,TRUE,TRUE,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,s-sbel-q5,,,,Q49,,,,2.23,2.24,2.74,2.75,3.24,3.25,4.5,4.51,
 School Culture,2,"Measures the degree to which the school environment is safe, caring, and academically-oriented.  It considers factors like bullying, student-teacher relationships, and student valuing of learning.","Measures the degree to which the school environment is safe, caring, and academically-oriented.",Relationships,2B,Seeks to determine the degree to which school climate includes supportive student relationships. It includes measures of student sense of belonging and student-teacher relationships. ,Student Sense of Belonging,"Draws on anonymous student and teacher reports about the degree to which they feel understood, supported, and accepted by students and adults at the school.",2B-i,,,Students,,TRUE,TRUE,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,s-sbel-q4,,,,Q48,,,,2.23,2.24,2.74,2.75,3.24,3.25,4.5,4.51,
 School Culture,2,"Measures the degree to which the school environment is safe, caring, and academically-oriented.  It considers factors like bullying, student-teacher relationships, and student valuing of learning.","Measures the degree to which the school environment is safe, caring, and academically-oriented.",Relationships,2B,Seeks to determine the degree to which school climate includes supportive student relationships. It includes measures of student sense of belonging and student-teacher relationships. ,Student Sense of Belonging,"Draws on anonymous student and teacher reports about the degree to which they feel understood, supported, and accepted by students and adults at the school.",2B-i,,,Students,,TRUE,TRUE,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,s-sbel-q3,,,,Q46,,,,2.23,2.24,2.74,2.75,3.24,3.25,4.5,4.51,
 School Culture,2,"Measures the degree to which the school environment is safe, caring, and academically-oriented.  It considers factors like bullying, student-teacher relationships, and student valuing of learning.","Measures the degree to which the school environment is safe, caring, and academically-oriented.",Relationships,2B,Seeks to determine the degree to which school climate includes supportive student relationships. It includes measures of student sense of belonging and student-teacher relationships. ,Student Sense of Belonging,"Draws on anonymous student and teacher reports about the degree to which they feel understood, supported, and accepted by students and adults at the school.",2B-i,,,Students,,TRUE,TRUE,"At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?",s-sbel-q2,,,X,Q45,,,,2.23,2.24,2.74,2.75,3.24,3.25,4.5,4.51,
@@ -165,7 +166,7 @@ Resources,3,"Measures the adequacy of a school's facility, personnel, and curric
 Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Family-School Relationships,Draws on anonymous teacher reports about their engagement with parents and leadership opportunities for parents at the school.,3C-i,,,Teachers,,TRUE,TRUE,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,t-peng-q4,,,,#N/A,,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41,
 Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Family-School Relationships,Draws on anonymous teacher reports about their engagement with parents and leadership opportunities for parents at the school.,3C-i,Teacher Communication - Parent,Draws on anonymous parent reports about whether there are adequate opportunities to meet with their child(ren)'s teachers. ,Parents,New scale - teacher communication,TRUE,TRUE,"There are adequate opportunities to meet with my
 child(ren)'s teacher(s), e.g., parent-teacher conferences?",,,,,,,,p-tcom-q1,,,,Q20,Update survey item ID in qualtrics,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41,
-Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Family-School Relationships,Draws on anonymous teacher reports about their engagement with parents and leadership opportunities for parents at the school.,3C-i,Teacher Communication - Parent,Draws on anonymous parent reports about whether there are adequate opportunities to meet with their child(ren)'s teachers. ,Parents,New scale - teacher communication,TRUE,TRUE,There is an opportunity to meet with my child's teacher if I would like to (either in person or virtually)?,,,,,,,,p-tcom-q2,,,,Q20,Update survey item ID in qualtrics,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41,
+Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Family-School Relationships,Draws on anonymous teacher reports about their engagement with parents and leadership opportunities for parents at the school.,3C-i,Teacher Communication - Parent,Draws on anonymous parent reports about whether there are adequate opportunities to meet with their child(ren)'s teachers. ,No source,New scale - teacher communication,TRUE,FALSE,There is an opportunity to meet with my child's teacher if I would like to (either in person or virtually)?,,,,,,,,p-tcom-q2,,,,Q20,Update survey item ID in qualtrics,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41,
 Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Family-School Relationships,Draws on anonymous teacher reports about their engagement with parents and leadership opportunities for parents at the school.,3C-i,Teacher Communication - Parent,Draws on anonymous parent reports about whether there are adequate opportunities to meet with their child(ren)'s teachers. ,Parents,New scale - teacher communication,TRUE,TRUE,"I feel comfortable requesting a meeting with my child(ren)'s teacher(s), if I would like to.",,,,,,,,p-tcom-q3,,,,Q100,Update survey item ID in qualtrics,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41,
 Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Community Involvement & External Partners,Draws on anonymous teacher reports about the degree to which the school is responsive to equity issues in its community. ,3C-ii,,,Teachers,,TRUE,TRUE,"How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?",t-ceng-q1,,,,#N/A,,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41,
 Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Community Involvement & External Partners,Draws on anonymous teacher reports about the degree to which the school is responsive to equity issues in its community. ,3C-ii,,,Teachers,,TRUE,TRUE,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,t-ceng-q2,,,,#N/A,,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41,

spec/fixtures/test_2020-21_parent_survey_responses.csv

@@ -0,0 +1,8 @@
+StartDate,EndDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,DistributionChannel,UserLanguage,DESE ID,Number of Children,Gender-1,Gender-1_7_TEXT,Race-1,Race-1_7_TEXT,Gender-2,Gender-2_7_TEXT,Race-2,Race-2_7_TEXT,Gender-3,Gender-3_7_TEXT,Race-3,Race-3_7_TEXT,Gender-4,Gender-4_7_TEXT,Race-4,Race-4_7_TEXT,Gender-5,Gender-5_7_TEXT,Race-5,Race-5_7_TEXT,Gender,Gender_7_TEXT,p-scrp-q3,p-scrp-q2,p-valm-q1,p-valm-q2,p-valm-q3,p-valm-q4,p-comm-q1,p-comm-q2,p-comm-q3,p-tcom-q1,P-tcom-q2,p-tcom-q3,p-evnt-q4,p-comm-q4,p-evnt-q3,p-evnt-q1,p-evnt-q2,p-socx-q3,p-socx-q4,p-scrp-q1,p-socx-q1,p-sosu-q1,p-sosu-q2,p-sosu-q3,p-socx-q2,p-sosu-q4,p-phys-q3,p-acpr-q1,p-acpr-q2,p-acpr-q3,p-acpr-q4,p-cure-q1,p-cure-q2,p-cure-q3,p-cure-q4,Housing,Housing_100_TEXT,Employment,Employment_100_TEXT,Caregivers,Caregivers_100_TEXT,Education,Education_100_TEXT,Benefits,Benefits_100_TEXT,Language,Language_100_TEXT,Raw Income,Income,Raw ELL,ELL,Raw SpEd,SpEd,Progress Count,Race
+5/1/2024 10:04:34,5/1/2024 10:10:49,0,72.93.86.98,100,374,1,2021-03-31T10:01:36,parent_survey_response_1,email,EN,1500025,1,,,,,,,,,,,,,,,,,,,,,2,,4,5,5,4,5,5,5,5,5,4,4,5,4,5,3,4,5,4,4,5,5,5,5,5,5,5,1,2,2,2,1,4,5,5,5,1,,1,,2,,5,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,99
+5/1/2024 10:03:52,5/1/2024 10:14:42,0,73.69.182.58,100,649,1,2021-04-01T10:01:36,parent_survey_response_2,email,EN,1500025,1,,,,,,,,,,,,,,,,,,,,,1,,4,4,5,5,5,5,5,5,5,5,5,5,3,5,4,5,5,5,5,5,4,4,4,4,5,5,1,5,4,5,5,5,5,5,5,1,,99,,2,,3,,1,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,99
+5/1/2024 10:06:44,5/1/2024 10:15:41,0,50.235.109.170,100,537,1,2021-04-02T10:01:36,parent_survey_response_3,email,EN,1500025,2,2,,5,,2,,5,,,,,,,,,,,,,,2,,5,5,5,4,5,5,5,5,5,4,4,5,4,4,3,4,4,4,4,5,4,4,5,5,2,5,3,4,4,4,4,5,5,5,5,1,,1,,3,,6,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,5
+5/1/2024 10:14:23,5/1/2024 10:22:22,0,73.38.238.192,100,478,1,2021-04-03T10:01:36,parent_survey_response_4,email,EN,1500025,1,,,,,,,,,,,,,,,,,,,,,1,,5,5,5,5,5,5,5,4,5,4,4,4,2,5,4,5,4,5,5,5,3,5,5,5,2,5,1,5,5,5,5,5,5,5,5,1,,1,,2,,5,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,99
+5/1/2024 10:18:39,5/1/2024 10:23:49,0,73.69.158.255,100,310,1,2021-04-04T10:01:36,parent_survey_response_5,email,EN,1500025,2,2,,5,,1,,5,,,,,,,,,,,,,,2,,5,4,5,5,5,5,1,1,1,1,1,1,3,1,4,4,5,1,1,1,4,1,1,1,4,5,1,5,5,5,5,1,5,1,1,1,,"2,4",,3,,5,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,5
+5/1/2024 10:20:30,5/1/2024 10:25:16,0,73.182.146.201,100,285,1,2021-04-05T10:01:36,parent_survey_response_6,email,EN,1500025,1,,,,,,,,,,,,,,,,,,,,,1,,3,3,3,1,3,2,4,2,4,1,1,3,3,4,3,4,1,5,5,4,3,5,4,3,3,1,3,5,5,4,5,4,4,5,4,2,,2,,2,,5,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,99
+5/1/2024 10:14:01,5/1/2024 10:27:19,0,209.107.182.203,100,798,1,2021-04-06T10:01:36,parent_survey_response_7,email,EN,1500025,2,1,,5,,1,,5,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,1,,2,,5,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,5

spec/services/survey_item_values_spec.rb

@@ -107,6 +107,14 @@ RSpec.describe SurveyItemValues, type: :model do
     (survey_item_ids << common_headers).flatten
   end
 
+  context ".normalize_headers" do
+    it "normalizes the headers to remove invisible newlines and lowercase survey item ids" do
+    headers = [ " p-tcom-q1\n",	" P-tcom-q2\r\n ", 	" P-tcom-q3 " ]
+    normalized_headers = SurveyItemValues.new(row: {}, headers:, survey_items:, schools:, academic_years:).normalize_headers(headers:)
+    expect(normalized_headers).to eq  ["p-tcom-q1", "p-tcom-q2", "p-tcom-q3"]
+    end
+  end
+
   context ".recorded_date" do
     it "returns the recorded date" do
       row = { "RecordedDate" => "2017-01-01T12:12:121" }

spec/services/survey_responses_data_loader_spec.rb

@@ -3,6 +3,7 @@ require "rails_helper"
 describe SurveyResponsesDataLoader do
   let(:path_to_teacher_responses) { Rails.root.join("spec", "fixtures", "test_2020-21_teacher_survey_responses.csv") }
   let(:path_to_student_responses) { Rails.root.join("spec", "fixtures", "test_2020-21_student_survey_responses.csv") }
+  let(:path_to_parent_responses) { Rails.root.join("spec", "fixtures", "test_2020-21_parent_survey_responses.csv") }
   let(:path_to_butler_student_responses) do
     Rails.root.join("spec", "fixtures", "test_2022-23_butler_student_survey_responses.csv")
   end
@@ -41,6 +42,54 @@ describe SurveyResponsesDataLoader do
       create(:survey_item, survey_item_id: id)
     end
   end
+  let(:parent_survey_items) do
+    ids = %w[
+      p-socx-q1
+      p-socx-q2
+      p-socx-q3
+      p-socx-q4
+      p-sosu-q1
+      p-sosu-q2
+      p-sosu-q3
+      p-sosu-q4
+      p-tcom-q1
+      p-tcom-q2
+      p-tcom-q3
+      p-comm-q1
+      p-comm-q2
+      p-comm-q3
+      p-comm-q4
+      p-valm-q1
+      p-valm-q2
+      p-valm-q3
+      p-valm-q4
+      p-acpr-q1
+      p-acpr-q2
+      p-acpr-q3
+      p-acpr-q4
+    ]
+
+    ids.each do |id|
+      create(:survey_item, survey_item_id: id)
+    end
+  end
+
+  let(:vestigial_parent_ids) do
+    %w[
+      p-scrp-q3
+      p-cure-q1
+      p-cure-q2
+      p-cure-q3
+      p-cure-q4
+      p-evnt-q1
+      p-evnt-q2
+      p-evnt-q3
+      p-evnt-q4
+      p-phys-q3
+      p-scrp-q1
+      p-scrp-q2
+    ]
+  end
 
   let(:t_pcom_q3) { create(:survey_item, survey_item_id: "t-pcom-q3") }
   let(:t_pcom_q2) { create(:survey_item, survey_item_id: "t-pcom-q2") }
@@ -174,6 +223,41 @@ describe SurveyResponsesDataLoader do
       end
     end
   end
+
+  describe "parent survey responses" do
+    before do
+      school
+      ay_2020_21
+      parent_survey_items
+      SurveyResponsesDataLoader.new.load_data filepath: path_to_parent_responses
+    end
+
+    it "adds only the surveyitems that exist in source of truth" do
+      si = (SurveyItemResponse.where(school:, response_id: "parent_survey_response_1").map do |response|
+        response.survey_item.survey_item_id
+      end)
+      response_ids = %w[
+        parent_survey_response_1
+        parent_survey_response_2
+        parent_survey_response_3
+        parent_survey_response_4
+        parent_survey_response_5
+        parent_survey_response_6
+      ]
+      response_ids.each do |id|
+        expect(SurveyItemResponse.where(response_id: id,
+                                        survey_item: SurveyItem.parent_survey_items).count).to eq 23
+      end
+
+      expect(SurveyItemResponse.where(response_id: "parent_survey_response_7").count).to eq 0
+    end
+
+    it "does not add surveyitems from questions that have been disabled" do
+      vestigial_parent_ids.each do |id|
+        expect(SurveyItemResponse.where(school:, survey_item: id).count).to eq 0
+      end
+    end
+  end
 end
 
 def assigns_academic_year_to_survey_item_responses