gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-09 07:28:41 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add automated data cleaning. Modify SurveyItemValues class to use regex

Browse source 
instead of hard coded values.  Produce a clean csv and a csv with all
the removed values and columns with reason for removal. Add script for
running cleaning for each project
This commit is contained in:
rebuilt 2023-05-02 20:08:34 -07:00
parent b5dc933187
commit 0dfc9726d0
21 changed files with 1214 additions and 149 deletions
Download patch file Download diff file Expand all files Collapse all files

162
spec/services/cleaner_spec.rb Normal file
View file

@ -0,0 +1,162 @@
require 'rails_helper'
require 'fileutils'
RSpec.describe Cleaner do
let(:district) { create(:district, name: 'District1') }
let(:second_district) { create(:district, name: 'District2') }
let(:school) { create(:school, dese_id: 1_740_505, district:) }
let(:second_school) { create(:school, dese_id: 222_222, district: second_district) }
let(:academic_year) { create(:academic_year, range: '2022-23') }
let(:respondents) { create(:respondent, school:, academic_year:, nine: 40, ten: 40, eleven: 40, twelve: 40) }
let(:recorded_date) { '2023-04-01' }
let(:input_filepath) do
Rails.root.join('spec', 'fixtures', 'raw')
end
let(:output_filepath) do
Rails.root.join('tmp', 'spec', 'clean')
end
let(:log_filepath) do
Rails.root.join('tmp', 'spec', 'removed')
end
let(:common_headers) do
['Recorded Date', 'DeseID', 'ResponseID']
end
let(:standard_survey_items) do
survey_item_ids = (%w[s-peff-q1 s-peff-q2 s-peff-q3 s-peff-q4 s-peff-q5 s-peff-q6 s-phys-q1 s-phys-q2 s-phys-q3 s-phys-q4
s-emsa-q1 s-emsa-q2 s-emsa-q3 s-sbel-q1 s-sbel-q2 s-sbel-q3 s-sbel-q4 s-sbel-q5 s-tint-q1 s-tint-q2
s-tint-q3 s-tint-q4 s-tint-q5 s-vale-q1 s-vale-q2 s-vale-q3 s-vale-q4 s-acpr-q1 s-acpr-q2 s-acpr-q3
s-acpr-q4 s-sust-q1 s-sust-q2 s-cure-q1 s-cure-q2 s-cure-q3 s-cure-q4 s-sten-q1 s-sten-q2 s-sten-q3
s-sper-q1 s-sper-q2 s-sper-q3 s-sper-q4 s-civp-q1 s-civp-q2 s-civp-q3 s-civp-q4 s-grit-q1 s-grit-q2
s-grit-q3 s-grit-q4 s-grmi-q1 s-grmi-q2 s-grmi-q3 s-grmi-q4 s-expa-q1 s-appa-q1 s-appa-q2 s-appa-q3
s-acst-q1 s-acst-q2 s-acst-q3 s-poaf-q1 s-poaf-q2 s-poaf-q3 s-poaf-q4] << common_headers).flatten
survey_item_ids.map do |survey_item_id|
create(:survey_item, survey_item_id:)
end
survey_item_ids
end
let(:short_form_survey_items) do
([create(:survey_item, survey_item_id: 's-phys-q1', on_short_form: true),
create(:survey_item, survey_item_id: 's-phys-q2', on_short_form: true),
create(:survey_item, survey_item_id: 's-phys-q3',
on_short_form: true)].map(&:survey_item_id) << common_headers).flatten
end
let(:early_education_survey_items) do
([create(:survey_item, survey_item_id: 's-emsa-es1'),
create(:survey_item, survey_item_id: 's-emsa-es2'),
create(:survey_item, survey_item_id: 's-emsa-es3')].map(&:survey_item_id) << common_headers).flatten
end
let(:teacher_survey_items) do
survey_item_ids = (%w[t-prep-q1 t-prep-q2 t-prep-q3 t-ieff-q1 t-ieff-q2 t-ieff-q3 t-ieff-q4 t-pcom-q1 t-pcom-q2 t-pcom-q3
t-pcom-q4 t-pcom-q5 t-inle-q1 t-inle-q2 t-inle-q3 t-prtr-q1 t-prtr-q2 t-prtr-q3 t-coll-q1 t-coll-q2
t-coll-q3 t-qupd-q1 t-qupd-q2 t-qupd-q3 t-qupd-q4 t-pvic-q1 t-pvic-q2 t-pvic-q3 t-psup-q1 t-psup-q2
t-psup-q3 t-psup-q4 t-acch-q1 t-acch-q2 t-acch-q3 t-reso-q1 t-reso-q2 t-reso-q3 t-reso-q4 t-reso-q5
t-sust-q1 t-sust-q2 t-sust-q3 t-sust-q4 t-curv-q1 t-curv-q2 t-curv-q3 t-curv-q4 t-cure-q1 t-cure-q2
t-cure-q3 t-cure-q4 t-peng-q1 t-peng-q2 t-peng-q3 t-peng-q4 t-ceng-q1 t-ceng-q2 t-ceng-q3 t-ceng-q4
t-sach-q1 t-sach-q2 t-sach-q3 t-psol-q1 t-psol-q2 t-psol-q3 t-expa-q2 t-expa-q3 t-phya-q2 t-phya-q3] << common_headers).flatten
survey_item_ids.map do |survey_item_id|
create(:survey_item, survey_item_id:)
end
survey_item_ids
end
before :each do
school
second_school
standard_survey_items
short_form_survey_items
early_education_survey_items
teacher_survey_items
academic_year
respondents
end
context 'Creating a new Cleaner' do
it 'creates a directory for the clean data' do
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
expect(output_filepath).to exist
end
it 'creates a directory for the removed data' do
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
expect(log_filepath).to exist
end
end
context '.filename' do
context 'defines a filename in the format: [district].[early_ed/short_form/standard/teacher].[year as 2022-23]' do
context 'when the file is based on standard survey items' do
it 'adds the survey type as standard to the filename' do
survey_items = SurveyItem.where(survey_item_id: standard_survey_items)
data = [SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '1_740_505' }, headers: standard_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: standard_survey_items, data:
)
expect(filename).to eq 'District1.standard.2022-23.csv'
end
context 'when the file is based on short form survey items' do
it 'adds the survey type as short form to the filename' do
survey_items = SurveyItem.where(survey_item_id: short_form_survey_items)
data = [SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '1_740_505' }, headers: short_form_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: short_form_survey_items, data:
)
expect(filename).to eq 'District1.short_form.2022-23.csv'
end
end
context 'when the file is based on early education survey items' do
it 'adds the survey type as early education to the filename' do
survey_items = SurveyItem.where(survey_item_id: early_education_survey_items)
data = [SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '1_740_505' }, headers: early_education_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: early_education_survey_items, data:
)
expect(filename).to eq 'District1.early_education.2022-23.csv'
end
end
context 'when the file is based on teacher survey items' do
it 'adds the survey type as teacher to the filename' do
survey_items = SurveyItem.where(survey_item_id: teacher_survey_items)
data = [SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '1_740_505' }, headers: teacher_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: teacher_survey_items, data:
)
expect(filename).to eq 'District1.teacher.2022-23.csv'
end
end
context 'when there is more than one district' do
it 'adds all districts to the filename' do
survey_items = SurveyItem.where(survey_item_id: teacher_survey_items)
data = [SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '1_740_505' }, headers: teacher_survey_items, genders: nil, survey_items:, schools: School.school_hash),
SurveyItemValues.new(row: { 'Recorded Date' => recorded_date, 'Dese ID' => '222_222' },
headers: teacher_survey_items, genders: nil, survey_items:, schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: teacher_survey_items, data:
)
expect(filename).to eq 'District1.District2.teacher.2022-23.csv'
end
end
end
end
end
end

194
spec/services/survey_item_values_spec.rb Normal file
View file

@ -0,0 +1,194 @@
require "rails_helper"
RSpec.describe SurveyItemValues, type: :model do
let(:headers) do
["StartDate", "EndDate", "Status", "IPAddress", "Progress", "Duration (in seconds)", "Finished", "RecordedDate",
"ResponseId", "RecipientLastName", "RecipientFirstName", "RecipientEmail", "ExternalReference", "LocationLatitude", "LocationLongitude", "DistributionChannel", "UserLanguage", "District", "School- Lee", "School- Maynard", "LASID", "Grade", "s-emsa-q1", "s-emsa-q2", "s-emsa-q3", "s-tint-q1", "s-tint-q2", "s-tint-q3", "s-tint-q4", "s-tint-q5", "s-acpr-q1", "s-acpr-q2", "s-acpr-q3", "s-acpr-q4", "s-cure-q1", "s-cure-q2", "s-cure-q3", "s-cure-q4", "s-sten-q1", "s-sten-q2", "s-sten-q3", "s-sper-q1", "s-sper-q2", "s-sper-q3", "s-sper-q4", "s-civp-q1", "s-civp-q2", "s-civp-q3", "s-civp-q4", "s-grmi-q1", "s-grmi-q2", "s-grmi-q3", "s-grmi-q4", "s-appa-q1", "s-appa-q2", "s-appa-q3", "s-peff-q1", "s-peff-q2", "s-peff-q3", "s-peff-q4", "s-peff-q5", "s-peff-q6", "s-sbel-q1", "s-sbel-q2", "s-sbel-q3", "s-sbel-q4", "s-sbel-q5", "s-phys-q1", "s-phys-q2", "s-phys-q3", "s-phys-q4", "s-vale-q1", "s-vale-q2", "s-vale-q3", "s-vale-q4", "s-acst-q1", "s-acst-q2", "s-acst-q3", "s-sust-q1", "s-sust-q2", "s-grit-q1", "s-grit-q2", "s-grit-q3", "s-grit-q4", "s-expa-q1", "s-poaf-q1", "s-poaf-q2", "s-poaf-q3", "s-poaf-q4", "s-tint-q1-1", "s-tint-q2-1", "s-tint-q3-1", "s-tint-q4-1", "s-tint-q5-1", "s-acpr-q1-1", "s-acpr-q2-1", "s-acpr-q3-1", "s-acpr-q4-1", "s-peff-q1-1", "s-peff-q2-1", "s-peff-q3-1", "s-peff-q4-1", "s-peff-q5-1", "s-peff-q6-1", "Gender", "Race"]
end
let(:genders) do
create(:gender, qualtrics_code: 1)
gender_hash = {}
Gender.all.each do |gender|
gender_hash[gender.qualtrics_code] = gender
end
gender_hash
end
let(:survey_items) { [] }
let(:attleboro) do
create(:school, name: "Attleboro", dese_id: 1234)
end
let(:attleboro_respondents) do
create(:respondent, school: attleboro, academic_year: ay_2022_23, nine: 40, ten: 40, eleven: 40, twelve: 40)
end
let(:schools) { School.school_hash }
let(:recorded_date) { "2023-04-01" }
let(:ay_2022_23) do
create(:academic_year, range: "2022-23")
end
context ".response_date" do
it "returns the recorded date" do
row = {"RecordedDate" => "2017-01-01"}
values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
expect(values.response_date).to eq Date.parse("2017-01-01")
end
end
context ".school" do
it "returns the school that maps to the dese id provided" do
attleboro
row = {"Dese ID" => "1234"}
values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
expect(values.school).to eq attleboro
row = {"DeseID" => "1234"}
values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
expect(values.school).to eq attleboro
end
end
context ".grade" do
it "returns the grade that maps to the grade provided" do
row = {"Grade" => "1"}
values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
expect(values.grade).to eq 1
end
end
context ".gender" do
it "returns the grade that maps to the grade provided" do
row = {"Gender" => "1"}
values = SurveyItemValues.new(row:, headers:, genders:, survey_items:, schools:)
expect(values.gender.qualtrics_code).to eq 1
end
end
context ".survey_type" do
it "reads header to find the survey type" do
headers = %w[s-sbel-q5 s-phys-q2 RecordedDate]
values = SurveyItemValues.new(row: {}, headers:, genders:, survey_items:, schools:)
expect(values.survey_type).to eq :student
headers = %w[t-sbel-q5 t-phys-q2]
values = SurveyItemValues.new(row: {}, headers:, genders:, survey_items:, schools:)
expect(values.survey_type).to eq :teacher
end
end
context ".valid_duration" do
context "when duration is valid" do
it "returns true" do
headers = ["s-sbel-q5", "s-phys-q2", "RecordedDate", "Duration (in seconds)"]
values = SurveyItemValues.new(row: {"Duration (in seconds)" => "240"}, headers:, genders:, survey_items:,
schools:)
expect(values.valid_duration?).to eq true
headers = ["t-sbel-q5", "t-phys-q2", "Duration (in seconds)"]
values = SurveyItemValues.new(row: {"Duration (in seconds)" => "300"}, headers:, genders:, survey_items:,
schools:)
expect(values.valid_duration?).to eq true
end
end
context "when duration is invalid" do
it "returns false" do
headers = ["s-sbel-q5", "s-phys-q2", "RecordedDate", "Duration (in seconds)"]
values = SurveyItemValues.new(row: {"Duration (in seconds)" => "239"}, headers:, genders:, survey_items:,
schools:)
expect(values.valid_duration?).to eq false
headers = ["t-sbel-q5", "t-phys-q2", "Duration (in seconds)"]
values = SurveyItemValues.new(row: {"Duration (in seconds)" => "299"}, headers:, genders:, survey_items:,
schools:)
expect(values.valid_duration?).to eq false
end
end
end
context ".valid_progress" do
context "when progress is valid" do
it "returns true" do
headers = %w[s-sbel-q5 s-phys-q2 RecordedDate]
values = SurveyItemValues.new(row: {"Progress" => "25"}, headers:, genders:, survey_items:,
schools:)
expect(values.valid_progress?).to eq true
end
end
context "when progress is invalid" do
it "returns false" do
headers = %w[s-sbel-q5 s-phys-q2 RecordedDate]
values = SurveyItemValues.new(row: {"Progress" => "24"}, headers:, genders:, survey_items:,
schools:)
expect(values.valid_progress?).to eq false
end
end
end
context ".valid_grade?" do
context "when grade is valid" do
before :each do
attleboro
attleboro_respondents
end
it "returns true for students" do
headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate]
values = SurveyItemValues.new(row: {"grade" => "9", "RecordedDate" => recorded_date, "Dese ID" => "1234"}, headers:, genders:, survey_items:,
schools:)
expect(values.valid_grade?).to eq true
end
it "returns true for teachers" do
headers = %w[t-sbel-q5 t-phys-q2 grade RecordedDate]
values = SurveyItemValues.new(row: {"RecordedDate" => recorded_date, "Dese ID" => "1234"}, headers:, genders:, survey_items:,
schools:)
expect(values.valid_grade?).to eq true
end
end
context "when grade is invalid" do
before :each do
attleboro
attleboro_respondents
end
it "returns false" do
headers = %w[s-sbel-q5 s-phys-q2 grade RecordedDate]
values = SurveyItemValues.new(row: {"grade" => "2", "RecordedDate" => recorded_date, "Dese ID" => "1234"}, headers:, genders:, survey_items:,
schools: School.school_hash)
expect(values.valid_grade?).to eq false
end
end
end
context ".valid_sd?" do
context "when the standard deviation is valid" do
it "returns true for student questions" do
headers = %w[s-sbel-q5 s-phys-q1 s-phys-q2 RecordedDate]
values = SurveyItemValues.new(row: {"RecordedDate" => recorded_date, "Dese ID" => "1234", "s-sbel-q5" => "1", "s-phys-q1" => "", "s-phys-q2" => "5"}, headers:, genders:, survey_items:,
schools: School.school_hash)
expect(values.valid_sd?).to eq true
end
it "returns true for teacher questions" do
headers = %w[t-sbel-q5 t-phys-q2]
values = SurveyItemValues.new(row: {"RecordedDate" => recorded_date, "Dese ID" => "1234", "t-sbel-q5" => "1", "t-phys-q2" => "5"}, headers:, genders:, survey_items:,
schools: School.school_hash)
expect(values.valid_sd?).to eq true
end
end
context "when the standard deviation is invalid" do
it "returns false for student questions" do
headers = %w[s-sbel-q5 s-phys-q1 s-phys-q2 RecordedDate]
values = SurveyItemValues.new(row: {"RecordedDate" => recorded_date, "Dese ID" => "1234", "s-sbel-q5" => "1", "s-phys-q2" => "", "s-phys-q2" => "1"}, headers:, genders:, survey_items:,
schools: School.school_hash)
expect(values.valid_sd?).to eq false
end
it "returns false for teacher questions" do
headers = %w[t-sbel-q5 t-phys-q1 t-phys-q2 RecordedDate]
values = SurveyItemValues.new(row: {"RecordedDate" => recorded_date, "Dese ID" => "1234", "t-sbel-q5" => "1", "t-phys-q2" => "", "t-phys-q2" => "1"}, headers:, genders:, survey_items:,
schools: School.school_hash)
expect(values.valid_sd?).to eq false
end
end
end
end

234
spec/services/survey_responses_data_loader_spec.rb
View file

@ -1,47 +1,47 @@
require 'rails_helper'
require "rails_helper"
describe SurveyResponsesDataLoader do
let(:path_to_teacher_responses) { Rails.root.join('spec', 'fixtures', 'test_2020-21_teacher_survey_responses.csv') }
let(:path_to_student_responses) { Rails.root.join('spec', 'fixtures', 'test_2020-21_student_survey_responses.csv') }
let(:path_to_teacher_responses) { Rails.root.join("spec", "fixtures", "test_2020-21_teacher_survey_responses.csv") }
let(:path_to_student_responses) { Rails.root.join("spec", "fixtures", "test_2020-21_student_survey_responses.csv") }
let(:path_to_butler_student_responses) do
Rails.root.join('spec', 'fixtures', 'test_2022-23_butler_student_survey_responses.csv')
Rails.root.join("spec", "fixtures", "test_2022-23_butler_student_survey_responses.csv")
end
let(:ay_2020_21) { create(:academic_year, range: '2020-21') }
let(:ay_2022_23) { create(:academic_year, range: '2022-23') }
let(:ay_2020_21) { create(:academic_year, range: "2020-21") }
let(:ay_2022_23) { create(:academic_year, range: "2022-23") }
let(:school) { create(:school, slug: 'lee-elementary-school', dese_id: 1_500_025) }
let(:lowell) { create(:district, name: 'Lowell', slug: 'lowell') }
let(:second_school) { create(:school, slug: 'lee-middle-high-school', dese_id: 1_500_505, district: lowell) }
let(:school) { create(:school, name: "Lee Elementary School", slug: "lee-elementary-school", dese_id: 1_500_025) }
let(:lowell) { create(:district, name: "Lowell", slug: "lowell") }
let(:second_school) { create(:school, name: "Lee Middle High School", slug: "lee-middle-high-school", dese_id: 1_500_505, district: lowell) }
let(:butler_school) do
create(:school, name: 'Butler Elementary School', slug: 'butler-elementary-school', dese_id: 1_600_310,
district: lowell)
create(:school, name: "Butler Elementary School", slug: "butler-elementary-school", dese_id: 1_600_310,
district: lowell)
end
let(:t_pcom_q3) { create(:survey_item, survey_item_id: 't-pcom-q3') }
let(:t_pcom_q2) { create(:survey_item, survey_item_id: 't-pcom-q2') }
let(:t_coll_q1) { create(:survey_item, survey_item_id: 't-coll-q1') }
let(:t_coll_q2) { create(:survey_item, survey_item_id: 't-coll-q2') }
let(:t_coll_q3) { create(:survey_item, survey_item_id: 't-coll-q3') }
let(:t_sach_q1) { create(:survey_item, survey_item_id: 't-sach-q1') }
let(:t_sach_q2) { create(:survey_item, survey_item_id: 't-sach-q2') }
let(:t_sach_q3) { create(:survey_item, survey_item_id: 't-sach-q3') }
let(:t_pcom_q3) { create(:survey_item, survey_item_id: "t-pcom-q3") }
let(:t_pcom_q2) { create(:survey_item, survey_item_id: "t-pcom-q2") }
let(:t_coll_q1) { create(:survey_item, survey_item_id: "t-coll-q1") }
let(:t_coll_q2) { create(:survey_item, survey_item_id: "t-coll-q2") }
let(:t_coll_q3) { create(:survey_item, survey_item_id: "t-coll-q3") }
let(:t_sach_q1) { create(:survey_item, survey_item_id: "t-sach-q1") }
let(:t_sach_q2) { create(:survey_item, survey_item_id: "t-sach-q2") }
let(:t_sach_q3) { create(:survey_item, survey_item_id: "t-sach-q3") }
let(:s_phys_q1) { create(:survey_item, survey_item_id: 's-phys-q1') }
let(:s_phys_q2) { create(:survey_item, survey_item_id: 's-phys-q2') }
let(:s_phys_q3) { create(:survey_item, survey_item_id: 's-phys-q3') }
let(:s_phys_q4) { create(:survey_item, survey_item_id: 's-phys-q4') }
let(:s_vale_q1) { create(:survey_item, survey_item_id: 's-phys-q1') }
let(:s_vale_q2) { create(:survey_item, survey_item_id: 's-phys-q2') }
let(:s_vale_q3) { create(:survey_item, survey_item_id: 's-phys-q3') }
let(:s_vale_q4) { create(:survey_item, survey_item_id: 's-phys-q4') }
let(:s_acst_q1) { create(:survey_item, survey_item_id: 's-acst-q1') }
let(:s_acst_q2) { create(:survey_item, survey_item_id: 's-acst-q2') }
let(:s_acst_q3) { create(:survey_item, survey_item_id: 's-acst-q3') }
let(:s_acst_q4) { create(:survey_item, survey_item_id: 's-acst-q4') }
let(:s_emsa_q1) { create(:survey_item, survey_item_id: 's-emsa-q1') }
let(:s_emsa_q2) { create(:survey_item, survey_item_id: 's-emsa-q2') }
let(:s_emsa_q3) { create(:survey_item, survey_item_id: 's-emsa-q3') }
let(:s_phys_q1) { create(:survey_item, survey_item_id: "s-phys-q1") }
let(:s_phys_q2) { create(:survey_item, survey_item_id: "s-phys-q2") }
let(:s_phys_q3) { create(:survey_item, survey_item_id: "s-phys-q3") }
let(:s_phys_q4) { create(:survey_item, survey_item_id: "s-phys-q4") }
let(:s_vale_q1) { create(:survey_item, survey_item_id: "s-phys-q1") }
let(:s_vale_q2) { create(:survey_item, survey_item_id: "s-phys-q2") }
let(:s_vale_q3) { create(:survey_item, survey_item_id: "s-phys-q3") }
let(:s_vale_q4) { create(:survey_item, survey_item_id: "s-phys-q4") }
let(:s_acst_q1) { create(:survey_item, survey_item_id: "s-acst-q1") }
let(:s_acst_q2) { create(:survey_item, survey_item_id: "s-acst-q2") }
let(:s_acst_q3) { create(:survey_item, survey_item_id: "s-acst-q3") }
let(:s_acst_q4) { create(:survey_item, survey_item_id: "s-acst-q4") }
let(:s_emsa_q1) { create(:survey_item, survey_item_id: "s-emsa-q1") }
let(:s_emsa_q2) { create(:survey_item, survey_item_id: "s-emsa-q2") }
let(:s_emsa_q3) { create(:survey_item, survey_item_id: "s-emsa-q3") }
let(:female) { create(:gender, qualtrics_code: 1) }
let(:male) { create(:gender, qualtrics_code: 2) }
@ -86,17 +86,15 @@ describe SurveyResponsesDataLoader do
end
before :each do
AcademicYear.reset_academic_years
setup
end
describe 'loading teacher survey responses' do
describe "loading teacher survey responses" do
before do
SurveyResponsesDataLoader.load_data filepath: path_to_teacher_responses
end
it 'ensures teacher responses load correctly' do
it "ensures teacher responses load correctly" do
assigns_academic_year_to_survey_item_responses
assigns_school_to_the_survey_item_responses
loads_survey_item_responses_for_a_given_survey_response
@ -106,12 +104,12 @@ describe SurveyResponsesDataLoader do
end
end
describe 'student survey responses' do
describe "student survey responses" do
before do
SurveyResponsesDataLoader.load_data filepath: path_to_student_responses
end
it 'ensures student responses load correctly' do
it "ensures student responses load correctly" do
assigns_academic_year_to_student_survey_item_responses
assigns_school_to_student_survey_item_responses
loads_student_survey_item_response_values
@ -122,79 +120,79 @@ describe SurveyResponsesDataLoader do
is_idempotent_for_students
end
context 'when updating student survey responses from another csv file' do
context "when updating student survey responses from another csv file" do
before :each do
SurveyResponsesDataLoader.load_data filepath: Rails.root.join('spec', 'fixtures',
'secondary_test_2020-21_student_survey_responses.csv')
SurveyResponsesDataLoader.load_data filepath: Rails.root.join("spec", "fixtures",
"secondary_test_2020-21_student_survey_responses.csv")
end
it 'updates the likert score to the score on the new csv file' do
s_emsa_q1 = SurveyItem.find_by_survey_item_id 's-emsa-q1'
expect(SurveyItemResponse.where(response_id: 'student_survey_response_3',
survey_item: s_emsa_q1).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: 'student_survey_response_4',
survey_item: s_emsa_q1).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: 'student_survey_response_5',
survey_item: s_emsa_q1).first.likert_score).to eq 1
it "updates the likert score to the score on the new csv file" do
s_emsa_q1 = SurveyItem.find_by_survey_item_id "s-emsa-q1"
expect(SurveyItemResponse.where(response_id: "student_survey_response_3",
survey_item: s_emsa_q1).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: "student_survey_response_4",
survey_item: s_emsa_q1).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: "student_survey_response_5",
survey_item: s_emsa_q1).first.likert_score).to eq 1
end
end
end
# figure out why this is failing
describe 'when using Lowell rules to skip rows in the csv file' do
describe "when using Lowell rules to skip rows in the csv file" do
before :each do
SurveyResponsesDataLoader.load_data filepath: path_to_student_responses,
rules: [Rule::SkipNonLowellSchools]
rules: [Rule::SkipNonLowellSchools]
end
it 'rejects any non-lowell school' do
expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').count).to eq 0
it "rejects any non-lowell school" do
expect(SurveyItemResponse.where(response_id: "student_survey_response_1").count).to eq 0
expect(SurveyItemResponse.count).to eq 69
end
it 'loads the correct number of responses for lowell schools' do
expect(SurveyItemResponse.where(response_id: 'student_survey_response_2').count).to eq 0
expect(SurveyItemResponse.where(response_id: 'student_survey_response_3').count).to eq 12
expect(SurveyItemResponse.where(response_id: 'student_survey_response_4').count).to eq 15
expect(SurveyItemResponse.where(response_id: 'student_survey_response_5').count).to eq 14
it "loads the correct number of responses for lowell schools" do
expect(SurveyItemResponse.where(response_id: "student_survey_response_2").count).to eq 0
expect(SurveyItemResponse.where(response_id: "student_survey_response_3").count).to eq 12
expect(SurveyItemResponse.where(response_id: "student_survey_response_4").count).to eq 15
expect(SurveyItemResponse.where(response_id: "student_survey_response_5").count).to eq 14
end
context 'when loading 22-23 butler survey responses' do
context "when loading 22-23 butler survey responses" do
before :each do
SurveyResponsesDataLoader.load_data filepath: path_to_butler_student_responses,
rules: [Rule::SkipNonLowellSchools]
rules: [Rule::SkipNonLowellSchools]
end
it 'loads all the responses for Butler' do
it "loads all the responses for Butler" do
expect(SurveyItemResponse.where(school: butler_school).count).to eq 56
end
it 'blank entries for grade get loaded as nils, not zero values' do
expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_1').first.grade).to eq 7
expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_2').first.grade).to eq 7
expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_3').first.grade).to eq 7
expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_4').first.grade).to eq 5
expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_5').first.grade).to eq 7
expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_6').first.grade).to eq 6
expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_7').first.grade).to eq nil
expect(SurveyItemResponse.where(response_id: 'butler_student_survey_response_8').first.grade).to eq 0
it "blank entries for grade get loaded as nils, not zero values" do
expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_1").first.grade).to eq 7
expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_2").first.grade).to eq 7
expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_3").first.grade).to eq 7
expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_4").first.grade).to eq 5
expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_5").first.grade).to eq 7
expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_6").first.grade).to eq 6
expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_7").first.grade).to eq nil
expect(SurveyItemResponse.where(response_id: "butler_student_survey_response_8").first.grade).to eq 0
end
end
end
end
def assigns_academic_year_to_survey_item_responses
expect(SurveyItemResponse.find_by_response_id('teacher_survey_response_1').academic_year).to eq ay_2020_21
expect(SurveyItemResponse.find_by_response_id("teacher_survey_response_1").academic_year).to eq ay_2020_21
end
def assigns_school_to_the_survey_item_responses
expect(SurveyItemResponse.find_by_response_id('teacher_survey_response_1').school).to eq school
expect(SurveyItemResponse.find_by_response_id("teacher_survey_response_1").school).to eq school
end
def loads_survey_item_responses_for_a_given_survey_response
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_1').count).to eq 5
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_2').count).to eq 0
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_3').count).to eq 8
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_4').count).to eq 8
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_5').count).to eq 8
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_1").count).to eq 5
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_2").count).to eq 0
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_3").count).to eq 8
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_4").count).to eq 8
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_5").count).to eq 8
end
def loads_all_survey_item_responses_for_a_given_survey_item
@ -203,20 +201,20 @@ def loads_all_survey_item_responses_for_a_given_survey_item
end
def captures_likert_scores_for_survey_item_responses
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_1').where(survey_item: t_pcom_q2)).to be_empty
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_1').where(survey_item: t_pcom_q3).first.likert_score).to eq 3
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_1").where(survey_item: t_pcom_q2)).to be_empty
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_1").where(survey_item: t_pcom_q3).first.likert_score).to eq 3
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_2').where(survey_item: t_pcom_q2)).to be_empty
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_2').where(survey_item: t_pcom_q3)).to be_empty
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_2").where(survey_item: t_pcom_q2)).to be_empty
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_2").where(survey_item: t_pcom_q3)).to be_empty
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_3').where(survey_item: t_pcom_q2).first.likert_score).to eq 5
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_3').where(survey_item: t_pcom_q3).first.likert_score).to eq 5
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_3").where(survey_item: t_pcom_q2).first.likert_score).to eq 5
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_3").where(survey_item: t_pcom_q3).first.likert_score).to eq 5
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_4').where(survey_item: t_pcom_q2).first.likert_score).to eq 4
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_4').where(survey_item: t_pcom_q3).first.likert_score).to eq 4
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_4").where(survey_item: t_pcom_q2).first.likert_score).to eq 4
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_4").where(survey_item: t_pcom_q3).first.likert_score).to eq 4
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_5').where(survey_item: t_pcom_q2).first.likert_score).to eq 2
expect(SurveyItemResponse.where(response_id: 'teacher_survey_response_5').where(survey_item: t_pcom_q3).first.likert_score).to eq 4
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_5").where(survey_item: t_pcom_q2).first.likert_score).to eq 2
expect(SurveyItemResponse.where(response_id: "teacher_survey_response_5").where(survey_item: t_pcom_q3).first.likert_score).to eq 4
end
def is_idempotent
@ -228,19 +226,19 @@ def is_idempotent
end
def assigns_academic_year_to_student_survey_item_responses
expect(SurveyItemResponse.find_by_response_id('student_survey_response_3').academic_year).to eq ay_2020_21
expect(SurveyItemResponse.find_by_response_id("student_survey_response_3").academic_year).to eq ay_2020_21
end
def assigns_school_to_student_survey_item_responses
expect(SurveyItemResponse.find_by_response_id('student_survey_response_3').school).to eq second_school
expect(SurveyItemResponse.find_by_response_id("student_survey_response_3").school).to eq second_school
end
def loads_student_survey_item_response_values
expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').count).to eq 3
expect(SurveyItemResponse.where(response_id: 'student_survey_response_2').count).to eq 0
expect(SurveyItemResponse.where(response_id: 'student_survey_response_3').count).to eq 12
expect(SurveyItemResponse.where(response_id: 'student_survey_response_4').count).to eq 15
expect(SurveyItemResponse.where(response_id: 'student_survey_response_5').count).to eq 14
expect(SurveyItemResponse.where(response_id: "student_survey_response_1").count).to eq 3
expect(SurveyItemResponse.where(response_id: "student_survey_response_2").count).to eq 0
expect(SurveyItemResponse.where(response_id: "student_survey_response_3").count).to eq 12
expect(SurveyItemResponse.where(response_id: "student_survey_response_4").count).to eq 15
expect(SurveyItemResponse.where(response_id: "student_survey_response_5").count).to eq 14
end
def student_survey_item_response_count_matches_expected
@ -249,20 +247,20 @@ def student_survey_item_response_count_matches_expected
end
def captures_likert_scores_for_student_survey_item_responses
expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').where(survey_item: s_phys_q1).first.likert_score).to eq 3
expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').where(survey_item: s_phys_q2)).to be_empty
expect(SurveyItemResponse.where(response_id: "student_survey_response_1").where(survey_item: s_phys_q1).first.likert_score).to eq 3
expect(SurveyItemResponse.where(response_id: "student_survey_response_1").where(survey_item: s_phys_q2)).to be_empty
expect(SurveyItemResponse.where(response_id: 'student_survey_response_2').where(survey_item: s_phys_q1)).to be_empty
expect(SurveyItemResponse.where(response_id: 'student_survey_response_2').where(survey_item: s_phys_q2)).to be_empty
expect(SurveyItemResponse.where(response_id: "student_survey_response_2").where(survey_item: s_phys_q1)).to be_empty
expect(SurveyItemResponse.where(response_id: "student_survey_response_2").where(survey_item: s_phys_q2)).to be_empty
expect(SurveyItemResponse.where(response_id: 'student_survey_response_3').where(survey_item: s_phys_q1).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: 'student_survey_response_3').where(survey_item: s_phys_q2).first.likert_score).to eq 3
expect(SurveyItemResponse.where(response_id: "student_survey_response_3").where(survey_item: s_phys_q1).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: "student_survey_response_3").where(survey_item: s_phys_q2).first.likert_score).to eq 3
expect(SurveyItemResponse.where(response_id: 'student_survey_response_4').where(survey_item: s_phys_q1).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: 'student_survey_response_4').where(survey_item: s_phys_q2).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: "student_survey_response_4").where(survey_item: s_phys_q1).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: "student_survey_response_4").where(survey_item: s_phys_q2).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: 'student_survey_response_5').where(survey_item: s_phys_q1).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: 'student_survey_response_5').where(survey_item: s_phys_q2).first.likert_score).to eq 2
expect(SurveyItemResponse.where(response_id: "student_survey_response_5").where(survey_item: s_phys_q1).first.likert_score).to eq 1
expect(SurveyItemResponse.where(response_id: "student_survey_response_5").where(survey_item: s_phys_q2).first.likert_score).to eq 2
end
def is_idempotent_for_students
@ -274,12 +272,12 @@ def is_idempotent_for_students
end
def assigns_grade_level_to_responses
results = { 'student_survey_response_1' => 11,
'student_survey_response_3' => 8,
'student_survey_response_4' => 8,
'student_survey_response_5' => 7,
'student_survey_response_6' => 3,
'student_survey_response_7' => 4 }
results = {"student_survey_response_1" => 11,
"student_survey_response_3" => 8,
"student_survey_response_4" => 8,
"student_survey_response_5" => 7,
"student_survey_response_6" => 3,
"student_survey_response_7" => 4}
results.each do |key, value|
expect(SurveyItemResponse.where(response_id: key).all? do |response|
response.grade == value
@ -288,12 +286,12 @@ def assigns_grade_level_to_responses
end
def assigns_gender_to_responses
results = { 'student_survey_response_1' => female,
'student_survey_response_3' => male,
'student_survey_response_4' => non_binary,
'student_survey_response_5' => non_binary,
'student_survey_response_6' => unknown_gender,
'student_survey_response_7' => unknown_gender }
results = {"student_survey_response_1" => female,
"student_survey_response_3" => male,
"student_survey_response_4" => non_binary,
"student_survey_response_5" => non_binary,
"student_survey_response_6" => unknown_gender,
"student_survey_response_7" => unknown_gender}
results.each do |key, value|
expect(SurveyItemResponse.where(response_id: key).first.gender).to eq value