edcommonwealth
/
sqm-dashboards
mirror of https://github.com/edcommonwealth/sqm-dashboards.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: Support two date formats: ISO 8601 and the standard US date format

Browse Source 
used in google sheets
mciea-main
Nelson Jovel 2 years ago
parent 7d527b5639
commit c4d8b927af
6 changed files with 45 additions and 22 deletions
Show Stats Download Patch File Download Diff File

2
app/models/survey_item.rb
Unescape View File

@ -67,7 +67,7 @@ class SurveyItem < ActiveRecord::Base
end
def self.survey_type(survey_item_ids:)
survey_item_ids = survey_item_ids.reject { |id| id.ends_with?('-1') }.to_set
survey_item_ids = survey_item_ids.reject { |id| id.ends_with?("-1") }.to_set
return :short_form if survey_item_ids.subset? short_form_survey_items.map(&:survey_item_id).to_set
return :early_education if survey_item_ids.subset? early_education_survey_items.map(&:survey_item_id).to_set
return :teacher if survey_item_ids.subset? teacher_survey_items.map(&:survey_item_id).to_set

9
app/services/survey_item_values.rb
Unescape View File

@ -48,11 +48,10 @@ class SurveyItemValues
def recorded_date
@recorded_date ||= begin
recorded_date = value_from(pattern: /Recorded\s*Date/i)
date = nil
begin
date = Date.parse(recorded_date)
rescue StandardError => e
date = Date.strptime(recorded_date, "%m/%d/%Y")
date = if recorded_date.include?("T")
Date.parse(recorded_date)
else
Date.strptime(recorded_date, "%m/%d/%Y")
end
end
end

10
spec/fixtures/test_2020-21_teacher_survey_responses.csv vendored
Unescape View File

@ -1,6 +1,6 @@
Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,District,school,,DESE ID,t-prep-q1,t-prep-q2,t-prep-q3,t-pcom-q1,t-pcom-q2,t-pcom-q3,t-pcom-q4,t-pcom-q5,t-ieff-q1,t-ieff-q2,t-ieff-q3,t-ieff-q4,t-qupd-q3,t-qupd-q2,t-qupd-q1,t-qupd-q4,t-coll-q1,t-coll-q2,t-coll-q3,t-prtr-q1,t-prtr-q2,t-prtr-q3,t-inle-q1,t-inle-q2,t-inle-q3,t-pvic-q1,t-pvic-q2,t-pvic-q3,t-psup-q1,t-psup-q2,t-psup-q3,t-psup-q4,t-acch-q1,t-acch-q2,t-acch-q3,t-reso-q1,t-reso-q2,t-reso-q3,t-reso-q4,t-reso-q5,t-sust-q1,t-sust-q2,t-sust-q3,t-sust-q4,t-curv-q1,t-curv-q2,t-curv-q3,t-curv-q4,t-cure-q1,t-cure-q2,t-cure-q3,t-cure-q4,t-peng-q1,t-peng-q2,t-peng-q3,t-peng-q4,t-ceng-q1,t-ceng-q2,t-ceng-q3,t-ceng-q4,t-sach-q1,t-sach-q2,t-sach-q3,t-psol-q1,t-psol-q2,t-psol-q3,t-expa-q2,t-expa-q3,t-phya-q2,t-phya-q3,,,
2020-10-09 9:42:45,2020-10-09 11:08:57,0,50.204.125.194,93,5172,0,2020-10-16 11:09:03,teacher_survey_response_1,,,,,,,anonymous,EN,1,1,4,1500025,,,,,,3,,,,,,,,,,,3,3,3,NA,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,,,,,,,,,,,
2020-10-09 9:21:30,2020-10-09 14:28:09,0,50.204.125.194,24,18398,0,2020-10-16 14:28:18,teacher_survey_response_2,,,,,,,anonymous,EN,1,1,4,1500025,,,,,,,,,,,,,,,,,,,NA,,,,,NA,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
2020-12-06 8:28:00,2020-12-06 8:36:52,0,73.61.140.32,100,532,1,2020-12-06 8:36:52,teacher_survey_response_3,,,,,41.92610168,-71.30110168,anonymous,EN,1,2,2,1500505,5,5,5,5,5,5,5,1,5,5,5,5,3,2,1,5,4,5,5,4,4,5,5,5,5,3,3,3,3,1,3,3,4,3,2,4,4,4,4,3,3,2,2,2,4,4,4,3,2,3,2,5,4,5,5,3,4,4,3,5,2,1,3,4,3,3,4,4,3,3,,9,9
2020-12-06 8:41:41,2020-12-06 8:51:25,0,96.230.183.162,100,584,1,2020-12-06 8:51:25,teacher_survey_response_4,,,,,41.98530579,-71.5184021,anonymous,EN,1,2,2,1500505,5,5,5,4,4,4,4,1,4,4,4,4,3,3,3,2,5,2,4,5,4,5,4,4,4,3,3,3,3,4,4,3,3,3,3,3,4,3,3,4,4,3,2,4,4,4,4,4,3,2,3,4,5,3,2,2,4,4,3,3,4,3,3,3,3,3,3,4,3,3,,9,6
2020-12-06 8:44:42,2020-12-06 8:55:58,0,216.41.82.178,100,675,1,2020-12-06 8:55:58,teacher_survey_response_5,,,,,41.92610168,-71.30110168,anonymous,EN,1,2,2,1500505,5,4,4,2,2,4,1,2,4,4,4,4,3,3,3,2,4,3,5,4,3,4,3,3,4,2,2,4,4,2,4,3,4,4,3,3,2,3,2,3,3,2,4,4,3,4,4,3,2,2,4,4,5,2,3,1,4,4,3,3,3,2,2,4,4,3,3,3,3,3,,7,5
2020-10-09 9:42:45,2020-10-09 11:08:57,0,50.204.125.194,93,5172,0,2020-10-16T11:09:03,teacher_survey_response_1,,,,,,,anonymous,EN,1,1,4,1500025,,,,,,3,,,,,,,,,,,3,3,3,NA,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,,,,,,,,,,,
2020-10-09 9:21:30,2020-10-09 14:28:09,0,50.204.125.194,24,18398,0,2020-10-16T14:28:18,teacher_survey_response_2,,,,,,,anonymous,EN,1,1,4,1500025,,,,,,,,,,,,,,,,,,,NA,,,,,NA,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
2020-12-06 8:28:00,2020-12-06 8:36:52,0,73.61.140.32,100,532,1,2020-12-06T8:36:52,teacher_survey_response_3,,,,,41.92610168,-71.30110168,anonymous,EN,1,2,2,1500505,5,5,5,5,5,5,5,1,5,5,5,5,3,2,1,5,4,5,5,4,4,5,5,5,5,3,3,3,3,1,3,3,4,3,2,4,4,4,4,3,3,2,2,2,4,4,4,3,2,3,2,5,4,5,5,3,4,4,3,5,2,1,3,4,3,3,4,4,3,3,,9,9
2020-12-06 8:41:41,2020-12-06 8:51:25,0,96.230.183.162,100,584,1,2020-12-06T8:51:25,teacher_survey_response_4,,,,,41.98530579,-71.5184021,anonymous,EN,1,2,2,1500505,5,5,5,4,4,4,4,1,4,4,4,4,3,3,3,2,5,2,4,5,4,5,4,4,4,3,3,3,3,4,4,3,3,3,3,3,4,3,3,4,4,3,2,4,4,4,4,4,3,2,3,4,5,3,2,2,4,4,3,3,4,3,3,3,3,3,3,4,3,3,,9,6
2020-12-06 8:44:42,2020-12-06 8:55:58,0,216.41.82.178,100,675,1,2020-12-06T8:55:58,teacher_survey_response_5,,,,,41.92610168,-71.30110168,anonymous,EN,1,2,2,1500505,5,4,4,2,2,4,1,2,4,4,4,4,3,3,3,2,4,3,5,4,3,4,3,3,4,2,2,4,4,2,4,3,4,4,3,3,2,3,2,3,3,2,4,4,3,4,4,3,2,2,4,4,5,2,3,1,4,4,3,3,3,2,2,4,4,3,3,3,3,3,,7,5

1 Start Date End Date Response Type IP Address Progress Duration (in seconds) Finished RecordedDate ResponseId Recipient Last Name Recipient First Name Recipient Email External Data Reference Location Latitude Location Longitude Distribution Channel User Language District school DESE ID t-prep-q1 t-prep-q2 t-prep-q3 t-pcom-q1 t-pcom-q2 t-pcom-q3 t-pcom-q4 t-pcom-q5 t-ieff-q1 t-ieff-q2 t-ieff-q3 t-ieff-q4 t-qupd-q3 t-qupd-q2 t-qupd-q1 t-qupd-q4 t-coll-q1 t-coll-q2 t-coll-q3 t-prtr-q1 t-prtr-q2 t-prtr-q3 t-inle-q1 t-inle-q2 t-inle-q3 t-pvic-q1 t-pvic-q2 t-pvic-q3 t-psup-q1 t-psup-q2 t-psup-q3 t-psup-q4 t-acch-q1 t-acch-q2 t-acch-q3 t-reso-q1 t-reso-q2 t-reso-q3 t-reso-q4 t-reso-q5 t-sust-q1 t-sust-q2 t-sust-q3 t-sust-q4 t-curv-q1 t-curv-q2 t-curv-q3 t-curv-q4 t-cure-q1 t-cure-q2 t-cure-q3 t-cure-q4 t-peng-q1 t-peng-q2 t-peng-q3 t-peng-q4 t-ceng-q1 t-ceng-q2 t-ceng-q3 t-ceng-q4 t-sach-q1 t-sach-q2 t-sach-q3 t-psol-q1 t-psol-q2 t-psol-q3 t-expa-q2 t-expa-q3 t-phya-q2 t-phya-q3
2 2020-10-09 9:42:45 2020-10-09 11:08:57 0 50.204.125.194 93 5172 0 2020-10-16 11:09:03 2020-10-16T11:09:03 teacher_survey_response_1 anonymous EN 1 1 4 1500025 3 3 3 3 NA 5
3 2020-10-09 9:21:30 2020-10-09 14:28:09 0 50.204.125.194 24 18398 0 2020-10-16 14:28:18 2020-10-16T14:28:18 teacher_survey_response_2 anonymous EN 1 1 4 1500025 NA NA
4 2020-12-06 8:28:00 2020-12-06 8:36:52 0 73.61.140.32 100 532 1 2020-12-06 8:36:52 2020-12-06T8:36:52 teacher_survey_response_3 41.92610168 -71.30110168 anonymous EN 1 2 2 1500505 5 5 5 5 5 5 5 1 5 5 5 5 3 2 1 5 4 5 5 4 4 5 5 5 5 3 3 3 3 1 3 3 4 3 2 4 4 4 4 3 3 2 2 2 4 4 4 3 2 3 2 5 4 5 5 3 4 4 3 5 2 1 3 4 3 3 4 4 3 3 9 9
5 2020-12-06 8:41:41 2020-12-06 8:51:25 0 96.230.183.162 100 584 1 2020-12-06 8:51:25 2020-12-06T8:51:25 teacher_survey_response_4 41.98530579 -71.5184021 anonymous EN 1 2 2 1500505 5 5 5 4 4 4 4 1 4 4 4 4 3 3 3 2 5 2 4 5 4 5 4 4 4 3 3 3 3 4 4 3 3 3 3 3 4 3 3 4 4 3 2 4 4 4 4 4 3 2 3 4 5 3 2 2 4 4 3 3 4 3 3 3 3 3 3 4 3 3 9 6
6 2020-12-06 8:44:42 2020-12-06 8:55:58 0 216.41.82.178 100 675 1 2020-12-06 8:55:58 2020-12-06T8:55:58 teacher_survey_response_5 41.92610168 -71.30110168 anonymous EN 1 2 2 1500505 5 4 4 2 2 4 1 2 4 4 4 4 3 3 3 2 4 3 5 4 3 4 3 3 4 2 2 4 4 2 4 3 4 4 3 3 2 3 2 3 3 2 4 4 3 4 4 3 2 2 4 4 5 2 3 1 4 4 3 3 3 2 2 4 4 3 3 3 3 3 7 5

3
spec/services/cleaner_spec.rb
Unescape View File

@ -14,7 +14,8 @@ RSpec.describe Cleaner do
create(:respondent, school: second_school, academic_year:, one: 0, four: 40, five: 40, six: 40, seven: 40,
eight: 40)
end
let(:recorded_date) { "2023-04-01" }
let(:recorded_date) { "4/1/2023" }
let(:input_filepath) do
Rails.root.join("spec", "fixtures", "raw")
end

33
spec/services/survey_item_values_spec.rb
Unescape View File

@ -36,7 +36,7 @@ RSpec.describe SurveyItemValues, type: :model do
create(:respondent, school: attleboro, academic_year: ay_2022_23, nine: 40, ten: 40, eleven: 40, twelve: 40)
end
let(:schools) { School.school_hash }
let(:recorded_date) { "2023-04-01" }
let(:recorded_date) { "2023-04-01T12:12:12" }
let(:ay_2022_23) do
create(:academic_year, range: "2022-23")
end
@ -104,14 +104,14 @@ RSpec.describe SurveyItemValues, type: :model do
context ".recorded_date" do
it "returns the recorded date" do
row = { "RecordedDate" => "2017-01-01" }
row = { "RecordedDate" => "2017-01-01T12:12:121" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.recorded_date).to eq Date.parse("2017-01-01")
expect(values.recorded_date).to eq Date.parse("2017-01-01T12:12:12")
headers = ["Recorded Date"]
row = { "Recorded Date" => "2017-01-02" }
row = { "Recorded Date" => "2017-01-02T12:12:122" }
values = SurveyItemValues.new(row:, headers:, survey_items:, schools:)
expect(values.recorded_date).to eq Date.parse("2017-01-02")
expect(values.recorded_date).to eq Date.parse("2017-01-02T12:12:12")
end
end
@ -826,4 +826,27 @@ RSpec.describe SurveyItemValues, type: :model do
end
end
end
context ".academic_year" do
before do
create(:academic_year, range: "2020-21")
create(:academic_year, range: "2021-22")
end
it "parses the date correctly when the date is in standard date format for google sheets" do
recorded_date = "1/10/2022 14:21:45"
values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, survey_items:,
schools:)
ay_21_22 = AcademicYear.find_by_range "2021-22"
expect(values.academic_year).to eq ay_21_22
end
it "parses the date correctly when the date is in iso standard 8601" do
recorded_date = "2022-1-10T14:21:45"
values = SurveyItemValues.new(row: { "RecordedDate" => recorded_date, "DeseID" => "1234" }, headers:, survey_items:,
schools:)
ay_21_22 = AcademicYear.find_by_range "2021-22"
expect(values.academic_year).to eq ay_21_22
end
end
end

8
spec/services/survey_responses_data_loader_spec.rb
Unescape View File

@ -316,10 +316,10 @@ def assigns_recorded_date_to_student_responses
end
def assigns_recorded_date_to_teacher_responses
results = { "teacher_survey_response_1" => "2020-10-16 11:09:03",
"teacher_survey_response_3" => "2020-12-06 8:36:52",
"teacher_survey_response_4" => "2020-12-06 8:51:25",
"teacher_survey_response_5" => "2020-12-06 8:55:58" }
results = { "teacher_survey_response_1" => "2020-10-16T11:09:03",
"teacher_survey_response_3" => "2020-12-06T8:36:52",
"teacher_survey_response_4" => "2020-12-06T8:51:25",
"teacher_survey_response_5" => "2020-12-06T8:55:58" }
results.each do |key, value|
expect(SurveyItemResponse.find_by_response_id(key).recorded_date).to eq Date.parse(value)
end

Write Preview
Loading…
Cancel
Save