From fb963f63e0ad361eac53fed7abbb82b050a784de Mon Sep 17 00:00:00 2001
From: rebuilt <memoryman51@hotmail.com>
Date: Wed, 26 Apr 2023 15:56:17 -0700
Subject: [PATCH] Clean up rake tasks to get data:load_survey_responses working

---
 lib/tasks/data.rake | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake
index 872dbd89..6f43a016 100644
--- a/lib/tasks/data.rake
+++ b/lib/tasks/data.rake
@@ -1,19 +1,22 @@
 namespace :data do
   desc 'load survey responses'
   task load_survey_responses: :environment do
-    Dir.glob(Rails.root.join('data', 'survey_responses', '*.csv')).each do |filepath|
-      puts "=====================> Loading data from csv at path: #{filepath}"
-      SurveyResponsesDataLoader.load_data filepath:
+    survey_item_response_count = SurveyItemResponse.count
+    student_count = Student.count
+    path = '/data/survey_responses/clean/'
+    Sftp::Directory.open(path:) do |file|
+      SurveyResponsesDataLoader.from_file(file:)
     end
-    puts "=====================> Completed loading #{SurveyItemResponse.count} survey responses"
+    puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
 
-    puts 'Resetting response rates'
-    ResponseRateLoader.reset
-    puts "=====================> Completed loading #{ResponseRate.count} survey responses"
+    Sftp::Directory.open(path:) do |file|
+      StudentLoader.from_file(file:, rules: [])
+    end
+    puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
 
     puts 'Resetting race scores'
-    RaceScoreLoader.reset(fast_processing: false)
-    puts "=====================> Completed loading #{RaceScore.count} survey responses"
+    RaceScoreLoader.reset(fast_processing: true)
+    puts "=====================> Completed loading #{RaceScore.count} race scores"
 
     Rails.cache.clear
   end
@@ -46,10 +49,6 @@ namespace :data do
     end
     puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
 
-    puts 'Resetting response rates'
-    ResponseRateLoader.reset
-    puts "=====================> Completed loading #{ResponseRate.count} response rates"
-
     puts 'Resetting race scores'
     RaceScoreLoader.reset(fast_processing: false)
     puts "=====================> Completed loading #{RaceScore.count} race scores"