fix: make survey data uploads threaded

app/services/survey_responses_data_loader.rb

@@ -28,7 +28,6 @@ class SurveyResponsesDataLoader
     headers_array = CSV.parse(headers).first
     all_survey_items = survey_items(headers:)
 
-    survey_item_responses = []
     batch_size = 1000
 
     lines = []
@@ -38,9 +37,22 @@ class SurveyResponsesDataLoader
 
       lines << line
     end
+    slices = []
 
     lines.each_slice(batch_size) do |slice|
+      slices << slice
+    end
+
+    pool_size = 4
+    jobs = Queue.new
+    slices.each { |slice| jobs << slice }
+
+    workers = pool_size.times.map do
+      Thread.new do
+        while slice = jobs.pop(true)
+
           slice.each do |line|
+            survey_item_responses = []
             CSV.parse(line, headers:).map do |row|
               row = process_row(row: SurveyItemValues.new(row:, headers: headers_array,
                                                           survey_items: all_survey_items, schools:, academic_years:))
@@ -55,6 +67,11 @@ class SurveyResponsesDataLoader
             survey_item_responses = []
           end
         end
+      rescue ThreadError
+      end
+    end
+
+    workers.each(&:join)
 
     GC.start
   end