From a269bb0ec14a990340d16523093266cbe4c7cb06 Mon Sep 17 00:00:00 2001 From: Alex Basson Date: Mon, 27 Sep 2021 16:30:23 -0400 Subject: [PATCH] Try reporting on batch inserts --- app/services/survey_responses_data_loader.rb | 6 +++++- spec/fixtures/test_2020-21_student_survey_responses.csv | 2 +- spec/services/survey_responses_data_loader_spec.rb | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index 8f8b7b60..e498a389 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -15,7 +15,11 @@ class SurveyResponsesDataLoader process_row row: row, survey_items: survey_items end - SurveyItemResponse.import survey_item_responses.compact.flatten + batch_progress = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) { + puts "======================> Row size: #{rows_size}, Number of batches: #{num_batches}, Current batch number: #{current_batch_number}" + } + + SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 10, batch_progress: batch_progress end private diff --git a/spec/fixtures/test_2020-21_student_survey_responses.csv b/spec/fixtures/test_2020-21_student_survey_responses.csv index 07d4363f..608b0b16 100644 --- a/spec/fixtures/test_2020-21_student_survey_responses.csv +++ b/spec/fixtures/test_2020-21_student_survey_responses.csv @@ -1,5 +1,5 @@ Start Date,End Date,Response Type,IP Address,Progress,Duration (in seconds),Finished,RecordedDate,ResponseId,Recipient Last Name,Recipient First Name,Recipient Email,External Data Reference,Location Latitude,Location Longitude,Distribution Channel,User Language,district,school,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,s-emsa-q1,s-emsa-q2,s-emsa-q3,s-tint-q1,s-tint-q2,#N/A,s-tint-q4,s-tint-q5,s-acpr-q1,s-acpr-q2,s-acpr-q3,s-acpr-q4,#N/A,#N/A,s-cure-q3,s-cure-q4,#N/A,s-sten-q2,s-sten-q3,s-sper-q1,s-sper-q2,s-sper-q3,s-sper-q4,s-civp-q1,s-civp-q2,s-civp-q3,s-civp-q4,s-grmi-q1,#N/A,#N/A,s-grmi-q4,s-appa-q1,s-appa-q2,#N/A,s-peff-q1,s-peff-q2,s-peff-q3,s-peff-q4,s-peff-q5,s-peff-q6,s-sbel-q1,s-sbel-q2,s-sbel-q3,s-sbel-q4,s-sbel-q5,s-phys-q1,s-phys-q2,s-phys-q3,s-phys-q4,s-vale-q1,#N/A,#N/A,s-vale-q4,#N/A,s-acst-q2,s-acst-q3,#N/A,#N/A,s-grit-q1,s-grit-q2,s-grit-q3,s-grit-q4,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A -2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,,,,,,,anonymous,EN,1,8,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,some non-integer response,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,, +2020-09-29 18:28:41,2020-09-29 18:48:28,0,73.249.89.226,6,1186,0,2020-09-30T18:48:50,student_survey_response_1,,,,,,,anonymous,EN,1,8,,,,dddd,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,some non-integer response,6,,,,5,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,EN,,,, 2021-02-23 15:12:58,2021-02-23 15:13:17,0,50.207.254.114,0,19,0,2021-02-24T15:13:19,student_survey_response_2,,,,,,,anonymous,EN,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,NA,,,,,,,,,,,,,,,,,,,,,EN,,,, 2021-03-31 9:50:19,2021-03-31 9:59:01,0,108.7.17.250,100,522,1,2021-03-31T09:59:02,student_survey_response_3,,,,,42.53340149,-70.96530151,anonymous,EN,3,2,12,4,108,3300,7,1,,,,,,,,,,,,,,2,4,2,1,4,3,3,,,,,3,3,3,3,,,,,NA,,,,,,,,,3,2,3,3,2,1,3,4,1,3,3,4,4,2,4,3,3,4,3,3,3,4,3,3,3,3,3,,,,,,,,,,3,4,4,2,3,3,1,,3,,EN,Math teacher,,, 2021-03-31 9:50:09,2021-03-31 10:00:16,0,67.186.188.168,100,607,1,2021-03-31T10:00:17,student_survey_response_4,,,,,42.63510132,-71.30139923,anonymous,EN,3,2,12,18,108,2064,7,1,,2,2,1,,,,,,,,,,,,,,,,,,,,,,,,,3,5,3,3,,,,,,,,,,4,4,3,4,5,1,1,5,1,3,2,4,4,1,2,1,3,2,3,3,3,4,2,5,3,4,5,5,3,3,4,3,,,,,4,4,4,4,3,5,2,,2,,EN,,,,English teacher diff --git a/spec/services/survey_responses_data_loader_spec.rb b/spec/services/survey_responses_data_loader_spec.rb index 026b7db1..195a07af 100644 --- a/spec/services/survey_responses_data_loader_spec.rb +++ b/spec/services/survey_responses_data_loader_spec.rb @@ -81,7 +81,7 @@ describe SurveyResponsesDataLoader do end it 'loads all the survey item responses for a given survey response' do - expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').count).to eq 0 + expect(SurveyItemResponse.where(response_id: 'student_survey_response_1').count).to eq 2 expect(SurveyItemResponse.where(response_id: 'student_survey_response_2').count).to eq 0 expect(SurveyItemResponse.where(response_id: 'student_survey_response_3').count).to eq 25 expect(SurveyItemResponse.where(response_id: 'student_survey_response_4').count).to eq 22