From beb93aa8e89392cbc7c46d4d6f595cf09be205d7 Mon Sep 17 00:00:00 2001 From: Nelson Jovel Date: Fri, 8 Nov 2024 15:36:34 -0800 Subject: [PATCH] fix: make sure all likert scores get counted even when the survey item id has different capitalization. Add tests for uploading parent data. Change the parent response rate calcuation to count all students in the school instead of just for the grades that were given the student survey --- app/lib/seeder.rb | 3 + app/presenters/overview/overview_presenter.rb | 6 +- .../parent_response_rate_presenter.rb | 20 +++++ app/presenters/response_rate_presenter.rb | 44 +--------- .../student_response_rate_presenter.rb | 38 +++++++++ .../teacher_response_rate_presenter.rb | 16 ++++ app/services/survey_item_values.rb | 15 +++- app/services/survey_responses_data_loader.rb | 1 + data/sqm_framework.csv | 3 +- .../test_2020-21_parent_survey_responses.csv | 8 ++ spec/services/survey_item_values_spec.rb | 8 ++ .../survey_responses_data_loader_spec.rb | 84 +++++++++++++++++++ 12 files changed, 196 insertions(+), 50 deletions(-) create mode 100644 app/presenters/parent_response_rate_presenter.rb create mode 100644 app/presenters/student_response_rate_presenter.rb create mode 100644 app/presenters/teacher_response_rate_presenter.rb create mode 100644 spec/fixtures/test_2020-21_parent_survey_responses.csv diff --git a/app/lib/seeder.rb b/app/lib/seeder.rb index ff0e6536..f781c366 100644 --- a/app/lib/seeder.rb +++ b/app/lib/seeder.rb @@ -51,6 +51,7 @@ class Seeder def seed_sqm_framework(csv_file) admin_data_item_ids = [] + survey_item_ids = [] CSV.parse(File.read(csv_file), headers: true) do |row| next if row["Source"] == "No source" @@ -105,6 +106,7 @@ class Seeder survey_item.ideal_low_benchmark = ideal_low if ideal_low survey_item.on_short_form = marked? on_short_form survey_item.update! prompt: row["Question/item (23-24)"].strip + survey_item_ids << survey_item.id end active_admin = row["Active admin & survey items"] @@ -122,6 +124,7 @@ class Seeder end AdminDataValue.where.not(admin_data_item_id: admin_data_item_ids).delete_all AdminDataItem.where.not(id: admin_data_item_ids).delete_all + SurveyItem.where.not(id: survey_item_ids).delete_all end def seed_demographics(csv_file) diff --git a/app/presenters/overview/overview_presenter.rb b/app/presenters/overview/overview_presenter.rb index 02e97361..c30aa93e 100644 --- a/app/presenters/overview/overview_presenter.rb +++ b/app/presenters/overview/overview_presenter.rb @@ -36,15 +36,15 @@ class Overview::OverviewPresenter end def student_response_rate_presenter - ResponseRatePresenter.new(focus: :student, school: @school, academic_year: @academic_year) + StudentResponseRatePresenter.new(focus: :student, school: @school, academic_year: @academic_year) end def teacher_response_rate_presenter - ResponseRatePresenter.new(focus: :teacher, school: @school, academic_year: @academic_year) + TeacherResponseRatePresenter.new(focus: :teacher, school: @school, academic_year: @academic_year) end def parent_response_rate_presenter - ResponseRatePresenter.new(focus: :parent, school: @school, academic_year: @academic_year) + ParentResponseRatePresenter.new(focus: :parent, school: @school, academic_year: @academic_year) end def presenter_for_measure(measure) diff --git a/app/presenters/parent_response_rate_presenter.rb b/app/presenters/parent_response_rate_presenter.rb new file mode 100644 index 00000000..65a41785 --- /dev/null +++ b/app/presenters/parent_response_rate_presenter.rb @@ -0,0 +1,20 @@ +class ParentResponseRatePresenter < ResponseRatePresenter + def initialize(focus:, academic_year:, school:) + super(focus:, academic_year:, school:) + @survey_items = SurveyItem.parent_survey_items if focus == :parent + end + + def actual_count + SurveyItemResponse.includes(:parent).where(school:, academic_year:).where.not(parent_id: nil) + .select(:parent_id) + .distinct + .map { |response| response.parent&.number_of_children } + .compact.sum + end + + def respondents_count + return 0 if respondents.nil? + + respondents.total_students + end +end diff --git a/app/presenters/response_rate_presenter.rb b/app/presenters/response_rate_presenter.rb index c5ab79c3..fb3426e1 100644 --- a/app/presenters/response_rate_presenter.rb +++ b/app/presenters/response_rate_presenter.rb @@ -5,13 +5,6 @@ class ResponseRatePresenter @focus = focus @academic_year = academic_year @school = school - if focus == :student - @survey_items = Measure.all.flat_map do |measure| - measure.student_survey_items_with_sufficient_responses(school:, academic_year:) - end - end - @survey_items = SurveyItem.teacher_survey_items if focus == :teacher - @survey_items = SurveyItem.parent_survey_items if focus == :parent end def date @@ -48,36 +41,7 @@ class ResponseRatePresenter end def actual_count - if focus == :teacher - response_count_for_survey_items(survey_items:) - elsif focus == :parent - SurveyItemResponse.includes(:parent).where(school:, academic_year:).where.not(parent_id: nil) - .select(:parent_id) - .distinct - .map { |response| response.parent&.number_of_children } - .compact.sum - else - non_early_ed_items = survey_items - SurveyItem.early_education_survey_items - non_early_ed_count = response_count_for_survey_items(survey_items: non_early_ed_items) - - early_ed_items = survey_items & SurveyItem.early_education_survey_items - early_ed_count = SurveyItemResponse.where(school:, academic_year:, - survey_item: early_ed_items) - .group(:survey_item) - .select(:response_id) - .distinct - .count - .reduce(0) do |largest, row| - count = row[1] - if count > largest - count - else - largest - end - end - - non_early_ed_count + early_ed_count - end + raise "please implement the method: actual_count" end def response_count_for_survey_items(survey_items:) @@ -86,11 +50,7 @@ class ResponseRatePresenter end def respondents_count - return 0 if respondents.nil? - - count = enrollment if focus == :student || focus == :parent - count = respondents.total_teachers if focus == :teacher - count + raise "please implement the method: respondents_count" end def enrollment diff --git a/app/presenters/student_response_rate_presenter.rb b/app/presenters/student_response_rate_presenter.rb new file mode 100644 index 00000000..35bc6fcc --- /dev/null +++ b/app/presenters/student_response_rate_presenter.rb @@ -0,0 +1,38 @@ +class StudentResponseRatePresenter < ResponseRatePresenter + def initialize(focus:, academic_year:, school:) + super(focus:, academic_year:, school:) + @survey_items = Measure.all.flat_map do |measure| + measure.student_survey_items_with_sufficient_responses(school:, academic_year:) + end + end + + def actual_count + # Early ed surveys are given in batches so they have to be counted separately because we have to account for the same student having a different response id per batch + non_early_ed_items = survey_items - SurveyItem.early_education_survey_items + non_early_ed_count = response_count_for_survey_items(survey_items: non_early_ed_items) + + early_ed_items = SurveyItem.early_education_survey_items + early_ed_count = SurveyItemResponse.where(school:, academic_year:, + survey_item: early_ed_items) + .group(:survey_item) + .select(:response_id) + .distinct + .count + .reduce(0) do |largest, row| + count = row[1] + if count > largest + count + else + largest + end + end + + non_early_ed_count + early_ed_count + end + + def respondents_count + return 0 if respondents.nil? + + enrollment + end +end diff --git a/app/presenters/teacher_response_rate_presenter.rb b/app/presenters/teacher_response_rate_presenter.rb new file mode 100644 index 00000000..f82f26f6 --- /dev/null +++ b/app/presenters/teacher_response_rate_presenter.rb @@ -0,0 +1,16 @@ +class TeacherResponseRatePresenter < ResponseRatePresenter + def initialize(focus:, academic_year:, school:) + super(focus:, academic_year:, school:) + @survey_items = SurveyItem.teacher_survey_items + end + + def actual_count + response_count_for_survey_items(survey_items:) + end + + def respondents_count + return 0 if respondents.nil? + + respondents.total_teachers + end +end diff --git a/app/services/survey_item_values.rb b/app/services/survey_item_values.rb index 17d3026d..ab97c11d 100644 --- a/app/services/survey_item_values.rb +++ b/app/services/survey_item_values.rb @@ -3,8 +3,8 @@ class SurveyItemValues def initialize(row:, headers:, survey_items:, schools:, academic_years: AcademicYear.all) @row = row - # Remove any newlines in headers - headers = headers.map { |item| item.delete("\n") if item.present? } + # Remove any newlines in headers and + @headers = normalize_headers(headers:) @headers = include_all_headers(headers:) @survey_items = survey_items @schools = schools @@ -25,6 +25,13 @@ class SurveyItemValues copy_data_to_main_column(main: /Gender/i, secondary: /Gender Secondary|Gender-1/i) end + def normalize_headers(headers:) + headers + .select(&:present?) + .map { |item| item.strip } + .map { |item| item.downcase if item.match(/[stp]-/i) } + end + def copy_data_to_main_column(main:, secondary:) main_column = headers.find { |header| main.match(header) } row[main_column] = value_from(pattern: secondary) if row[main_column].nil? @@ -97,7 +104,7 @@ class SurveyItemValues end def likert_score(survey_item_id:) - row[survey_item_id] || row["#{survey_item_id}-1"] + row[survey_item_id] || row["#{survey_item_id}-1"] || value_from(pattern: /#{survey_item_id}/i) end def school @@ -196,7 +203,7 @@ class SurveyItemValues output = nil matches = headers.select do |header| pattern.match(header) - end.map { |item| item.delete("\n") } + end matches.each do |match| output ||= row[match]&.strip diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index a80154f8..6da9ef71 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -165,6 +165,7 @@ class SurveyResponsesDataLoader .parse(headers) .first .filter(&:present?) + .map(&:downcase) .filter { |header| header.start_with?("t-", "s-", "p-") } end end diff --git a/data/sqm_framework.csv b/data/sqm_framework.csv index 974268b3..72f9c19b 100644 --- a/data/sqm_framework.csv +++ b/data/sqm_framework.csv @@ -93,6 +93,7 @@ Community & Wellbeing,5,"Measures the development of traits relevant for student Community & Wellbeing,5,"Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives. It considers factors like perseverance and determination, participation in arts and literature, and social and emotional health.","Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives.",Civic Engagement,5A,Seeks to determine the degree to which students are prepared to thoughtfully interact with others in a diverse society and work towards social equity. It includes the measures of appreciation for diversity and civic participation. ,Appreciation For Diversity,Draws on anonymous student reports about their level of comfort in considering different perspectives and conflicting opinions.,5A-i,,,Students,,TRUE,TRUE,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,How often do you try to figure out what motivates others to behave as they do?,s-sper-q3,,,,Q61,,,,2.48,2.49,2.99,3,3.49,3.5,4.4,4.41, Community & Wellbeing,5,"Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives. It considers factors like perseverance and determination, participation in arts and literature, and social and emotional health.","Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives.",Civic Engagement,5A,Seeks to determine the degree to which students are prepared to thoughtfully interact with others in a diverse society and work towards social equity. It includes the measures of appreciation for diversity and civic participation. ,Appreciation For Diversity,Draws on anonymous student reports about their level of comfort in considering different perspectives and conflicting opinions.,5A-i,,,Students,,TRUE,TRUE,"Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?","Overall, how often do you try to understand the point of view of other people?",s-sper-q2,,,,Q60,,,,2.48,2.49,2.99,3,3.49,3.5,4.4,4.41, Community & Wellbeing,5,"Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives. It considers factors like perseverance and determination, participation in arts and literature, and social and emotional health.","Measures the development of traits relevant for students leading full and rewarding lives—in society, the workplace, and their private lives.",Civic Engagement,5A,Seeks to determine the degree to which students are prepared to thoughtfully interact with others in a diverse society and work towards social equity. It includes the measures of appreciation for diversity and civic participation. ,Appreciation For Diversity,Draws on anonymous student reports about their level of comfort in considering different perspectives and conflicting opinions.,5A-i,,,Students,,TRUE,TRUE,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,How often do you try to think of more than one explanation for why someone else acted as they did?,s-sper-q1,,,,Q59,,,,2.48,2.49,2.99,3,3.49,3.5,4.4,4.41, +School Culture,2,"Measures the degree to which the school environment is safe, caring, and academically-oriented. It considers factors like bullying, student-teacher relationships, and student valuing of learning.","Measures the degree to which the school environment is safe, caring, and academically-oriented.",Relationships,2B,Seeks to determine the degree to which school climate includes supportive student relationships. It includes measures of student sense of belonging and student-teacher relationships. ,Student Sense of Belonging,"Draws on anonymous student and teacher reports about the degree to which they feel understood, supported, and accepted by students and adults at the school.",2B-i,,,Students,,TRUE,TRUE,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,How connected do you feel to the adults at your school?,s-sbel-q5,,,,Q49,,,,2.23,2.24,2.74,2.75,3.24,3.25,4.5,4.51, School Culture,2,"Measures the degree to which the school environment is safe, caring, and academically-oriented. It considers factors like bullying, student-teacher relationships, and student valuing of learning.","Measures the degree to which the school environment is safe, caring, and academically-oriented.",Relationships,2B,Seeks to determine the degree to which school climate includes supportive student relationships. It includes measures of student sense of belonging and student-teacher relationships. ,Student Sense of Belonging,"Draws on anonymous student and teacher reports about the degree to which they feel understood, supported, and accepted by students and adults at the school.",2B-i,,,Students,,TRUE,TRUE,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,How much respect do students in your school show you?,s-sbel-q4,,,,Q48,,,,2.23,2.24,2.74,2.75,3.24,3.25,4.5,4.51, School Culture,2,"Measures the degree to which the school environment is safe, caring, and academically-oriented. It considers factors like bullying, student-teacher relationships, and student valuing of learning.","Measures the degree to which the school environment is safe, caring, and academically-oriented.",Relationships,2B,Seeks to determine the degree to which school climate includes supportive student relationships. It includes measures of student sense of belonging and student-teacher relationships. ,Student Sense of Belonging,"Draws on anonymous student and teacher reports about the degree to which they feel understood, supported, and accepted by students and adults at the school.",2B-i,,,Students,,TRUE,TRUE,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,How well do people at your school understand you?,s-sbel-q3,,,,Q46,,,,2.23,2.24,2.74,2.75,3.24,3.25,4.5,4.51, School Culture,2,"Measures the degree to which the school environment is safe, caring, and academically-oriented. It considers factors like bullying, student-teacher relationships, and student valuing of learning.","Measures the degree to which the school environment is safe, caring, and academically-oriented.",Relationships,2B,Seeks to determine the degree to which school climate includes supportive student relationships. It includes measures of student sense of belonging and student-teacher relationships. ,Student Sense of Belonging,"Draws on anonymous student and teacher reports about the degree to which they feel understood, supported, and accepted by students and adults at the school.",2B-i,,,Students,,TRUE,TRUE,"At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?","At your school, how accepted do you feel by the other students?",s-sbel-q2,,,X,Q45,,,,2.23,2.24,2.74,2.75,3.24,3.25,4.5,4.51, @@ -165,7 +166,7 @@ Resources,3,"Measures the adequacy of a school's facility, personnel, and curric Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Family-School Relationships,Draws on anonymous teacher reports about their engagement with parents and leadership opportunities for parents at the school.,3C-i,,,Teachers,,TRUE,TRUE,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,How often does the average parent help out at your school?,t-peng-q4,,,,#N/A,,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41, Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Family-School Relationships,Draws on anonymous teacher reports about their engagement with parents and leadership opportunities for parents at the school.,3C-i,Teacher Communication - Parent,Draws on anonymous parent reports about whether there are adequate opportunities to meet with their child(ren)'s teachers. ,Parents,New scale - teacher communication,TRUE,TRUE,"There are adequate opportunities to meet with my child(ren)'s teacher(s), e.g., parent-teacher conferences?",,,,,,,,p-tcom-q1,,,,Q20,Update survey item ID in qualtrics,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41, -Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Family-School Relationships,Draws on anonymous teacher reports about their engagement with parents and leadership opportunities for parents at the school.,3C-i,Teacher Communication - Parent,Draws on anonymous parent reports about whether there are adequate opportunities to meet with their child(ren)'s teachers. ,Parents,New scale - teacher communication,TRUE,TRUE,There is an opportunity to meet with my child's teacher if I would like to (either in person or virtually)?,,,,,,,,p-tcom-q2,,,,Q20,Update survey item ID in qualtrics,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41, +Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Family-School Relationships,Draws on anonymous teacher reports about their engagement with parents and leadership opportunities for parents at the school.,3C-i,Teacher Communication - Parent,Draws on anonymous parent reports about whether there are adequate opportunities to meet with their child(ren)'s teachers. ,No source,New scale - teacher communication,TRUE,FALSE,There is an opportunity to meet with my child's teacher if I would like to (either in person or virtually)?,,,,,,,,p-tcom-q2,,,,Q20,Update survey item ID in qualtrics,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41, Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Family-School Relationships,Draws on anonymous teacher reports about their engagement with parents and leadership opportunities for parents at the school.,3C-i,Teacher Communication - Parent,Draws on anonymous parent reports about whether there are adequate opportunities to meet with their child(ren)'s teachers. ,Parents,New scale - teacher communication,TRUE,TRUE,"I feel comfortable requesting a meeting with my child(ren)'s teacher(s), if I would like to.",,,,,,,,p-tcom-q3,,,,Q100,Update survey item ID in qualtrics,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41, Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Community Involvement & External Partners,Draws on anonymous teacher reports about the degree to which the school is responsive to equity issues in its community. ,3C-ii,,,Teachers,,TRUE,TRUE,"How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?","How effectively does this school connect with immigrant families, providing translation when necessary?",t-ceng-q1,,,,#N/A,,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41, Resources,3,"Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community. It considers factors like physical spaces and materials, class size, and family-school relationships.","Measures the adequacy of a school's facility, personnel, and curriculum, as well as the degree to which it is supported by the community.",Community Support,3C,"Seeks to determine the degree to which schools have supportive relationships with the surrounding community. It includes measures of family-school relationships, community involvement & external partnerships.",Community Involvement & External Partners,Draws on anonymous teacher reports about the degree to which the school is responsive to equity issues in its community. ,3C-ii,,,Teachers,,TRUE,TRUE,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,How effectively does this school respond to the needs and values of the surrounding community?,t-ceng-q2,,,,#N/A,,,,2.18,2.19,2.69,2.7,3.19,3.2,4.4,4.41, diff --git a/spec/fixtures/test_2020-21_parent_survey_responses.csv b/spec/fixtures/test_2020-21_parent_survey_responses.csv new file mode 100644 index 00000000..0e1091c8 --- /dev/null +++ b/spec/fixtures/test_2020-21_parent_survey_responses.csv @@ -0,0 +1,8 @@ +StartDate,EndDate,Status,IPAddress,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,DistributionChannel,UserLanguage,DESE ID,Number of Children,Gender-1,Gender-1_7_TEXT,Race-1,Race-1_7_TEXT,Gender-2,Gender-2_7_TEXT,Race-2,Race-2_7_TEXT,Gender-3,Gender-3_7_TEXT,Race-3,Race-3_7_TEXT,Gender-4,Gender-4_7_TEXT,Race-4,Race-4_7_TEXT,Gender-5,Gender-5_7_TEXT,Race-5,Race-5_7_TEXT,Gender,Gender_7_TEXT,p-scrp-q3,p-scrp-q2,p-valm-q1,p-valm-q2,p-valm-q3,p-valm-q4,p-comm-q1,p-comm-q2,p-comm-q3,p-tcom-q1,P-tcom-q2,p-tcom-q3,p-evnt-q4,p-comm-q4,p-evnt-q3,p-evnt-q1,p-evnt-q2,p-socx-q3,p-socx-q4,p-scrp-q1,p-socx-q1,p-sosu-q1,p-sosu-q2,p-sosu-q3,p-socx-q2,p-sosu-q4,p-phys-q3,p-acpr-q1,p-acpr-q2,p-acpr-q3,p-acpr-q4,p-cure-q1,p-cure-q2,p-cure-q3,p-cure-q4,Housing,Housing_100_TEXT,Employment,Employment_100_TEXT,Caregivers,Caregivers_100_TEXT,Education,Education_100_TEXT,Benefits,Benefits_100_TEXT,Language,Language_100_TEXT,Raw Income,Income,Raw ELL,ELL,Raw SpEd,SpEd,Progress Count,Race +5/1/2024 10:04:34,5/1/2024 10:10:49,0,72.93.86.98,100,374,1,2021-03-31T10:01:36,parent_survey_response_1,email,EN,1500025,1,,,,,,,,,,,,,,,,,,,,,2,,4,5,5,4,5,5,5,5,5,4,4,5,4,5,3,4,5,4,4,5,5,5,5,5,5,5,1,2,2,2,1,4,5,5,5,1,,1,,2,,5,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,99 +5/1/2024 10:03:52,5/1/2024 10:14:42,0,73.69.182.58,100,649,1,2021-04-01T10:01:36,parent_survey_response_2,email,EN,1500025,1,,,,,,,,,,,,,,,,,,,,,1,,4,4,5,5,5,5,5,5,5,5,5,5,3,5,4,5,5,5,5,5,4,4,4,4,5,5,1,5,4,5,5,5,5,5,5,1,,99,,2,,3,,1,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,99 +5/1/2024 10:06:44,5/1/2024 10:15:41,0,50.235.109.170,100,537,1,2021-04-02T10:01:36,parent_survey_response_3,email,EN,1500025,2,2,,5,,2,,5,,,,,,,,,,,,,,2,,5,5,5,4,5,5,5,5,5,4,4,5,4,4,3,4,4,4,4,5,4,4,5,5,2,5,3,4,4,4,4,5,5,5,5,1,,1,,3,,6,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,5 +5/1/2024 10:14:23,5/1/2024 10:22:22,0,73.38.238.192,100,478,1,2021-04-03T10:01:36,parent_survey_response_4,email,EN,1500025,1,,,,,,,,,,,,,,,,,,,,,1,,5,5,5,5,5,5,5,4,5,4,4,4,2,5,4,5,4,5,5,5,3,5,5,5,2,5,1,5,5,5,5,5,5,5,5,1,,1,,2,,5,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,99 +5/1/2024 10:18:39,5/1/2024 10:23:49,0,73.69.158.255,100,310,1,2021-04-04T10:01:36,parent_survey_response_5,email,EN,1500025,2,2,,5,,1,,5,,,,,,,,,,,,,,2,,5,4,5,5,5,5,1,1,1,1,1,1,3,1,4,4,5,1,1,1,4,1,1,1,4,5,1,5,5,5,5,1,5,1,1,1,,"2,4",,3,,5,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,5 +5/1/2024 10:20:30,5/1/2024 10:25:16,0,73.182.146.201,100,285,1,2021-04-05T10:01:36,parent_survey_response_6,email,EN,1500025,1,,,,,,,,,,,,,,,,,,,,,1,,3,3,3,1,3,2,4,2,4,1,1,3,3,4,3,4,1,5,5,4,3,5,4,3,3,1,3,5,5,4,5,4,4,5,4,2,,2,,2,,5,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,99 +5/1/2024 10:14:01,5/1/2024 10:27:19,0,209.107.182.203,100,798,1,2021-04-06T10:01:36,parent_survey_response_7,email,EN,1500025,2,1,,5,,1,,5,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,1,,2,,5,,2,,1,,,Economically Disadvantaged - N,,Not ELL,,Not Special Education,34,5 diff --git a/spec/services/survey_item_values_spec.rb b/spec/services/survey_item_values_spec.rb index 7e70d462..5aaa5a56 100644 --- a/spec/services/survey_item_values_spec.rb +++ b/spec/services/survey_item_values_spec.rb @@ -107,6 +107,14 @@ RSpec.describe SurveyItemValues, type: :model do (survey_item_ids << common_headers).flatten end + context ".normalize_headers" do + it "normalizes the headers to remove invisible newlines and lowercase survey item ids" do + headers = [ " p-tcom-q1\n", " P-tcom-q2\r\n ", " P-tcom-q3 " ] + normalized_headers = SurveyItemValues.new(row: {}, headers:, survey_items:, schools:, academic_years:).normalize_headers(headers:) + expect(normalized_headers).to eq ["p-tcom-q1", "p-tcom-q2", "p-tcom-q3"] + end + end + context ".recorded_date" do it "returns the recorded date" do row = { "RecordedDate" => "2017-01-01T12:12:121" } diff --git a/spec/services/survey_responses_data_loader_spec.rb b/spec/services/survey_responses_data_loader_spec.rb index 199f7cc1..2f2cf747 100644 --- a/spec/services/survey_responses_data_loader_spec.rb +++ b/spec/services/survey_responses_data_loader_spec.rb @@ -3,6 +3,7 @@ require "rails_helper" describe SurveyResponsesDataLoader do let(:path_to_teacher_responses) { Rails.root.join("spec", "fixtures", "test_2020-21_teacher_survey_responses.csv") } let(:path_to_student_responses) { Rails.root.join("spec", "fixtures", "test_2020-21_student_survey_responses.csv") } + let(:path_to_parent_responses) { Rails.root.join("spec", "fixtures", "test_2020-21_parent_survey_responses.csv") } let(:path_to_butler_student_responses) do Rails.root.join("spec", "fixtures", "test_2022-23_butler_student_survey_responses.csv") end @@ -41,6 +42,54 @@ describe SurveyResponsesDataLoader do create(:survey_item, survey_item_id: id) end end + let(:parent_survey_items) do + ids = %w[ + p-socx-q1 + p-socx-q2 + p-socx-q3 + p-socx-q4 + p-sosu-q1 + p-sosu-q2 + p-sosu-q3 + p-sosu-q4 + p-tcom-q1 + p-tcom-q2 + p-tcom-q3 + p-comm-q1 + p-comm-q2 + p-comm-q3 + p-comm-q4 + p-valm-q1 + p-valm-q2 + p-valm-q3 + p-valm-q4 + p-acpr-q1 + p-acpr-q2 + p-acpr-q3 + p-acpr-q4 + ] + + ids.each do |id| + create(:survey_item, survey_item_id: id) + end + end + + let(:vestigial_parent_ids) do + %w[ + p-scrp-q3 + p-cure-q1 + p-cure-q2 + p-cure-q3 + p-cure-q4 + p-evnt-q1 + p-evnt-q2 + p-evnt-q3 + p-evnt-q4 + p-phys-q3 + p-scrp-q1 + p-scrp-q2 + ] + end let(:t_pcom_q3) { create(:survey_item, survey_item_id: "t-pcom-q3") } let(:t_pcom_q2) { create(:survey_item, survey_item_id: "t-pcom-q2") } @@ -174,6 +223,41 @@ describe SurveyResponsesDataLoader do end end end + + describe "parent survey responses" do + before do + school + ay_2020_21 + parent_survey_items + SurveyResponsesDataLoader.new.load_data filepath: path_to_parent_responses + end + + it "adds only the surveyitems that exist in source of truth" do + si = (SurveyItemResponse.where(school:, response_id: "parent_survey_response_1").map do |response| + response.survey_item.survey_item_id + end) + response_ids = %w[ + parent_survey_response_1 + parent_survey_response_2 + parent_survey_response_3 + parent_survey_response_4 + parent_survey_response_5 + parent_survey_response_6 + ] + response_ids.each do |id| + expect(SurveyItemResponse.where(response_id: id, + survey_item: SurveyItem.parent_survey_items).count).to eq 23 + end + + expect(SurveyItemResponse.where(response_id: "parent_survey_response_7").count).to eq 0 + end + + it "does not add surveyitems from questions that have been disabled" do + vestigial_parent_ids.each do |id| + expect(SurveyItemResponse.where(school:, survey_item: id).count).to eq 0 + end + end + end end def assigns_academic_year_to_survey_item_responses