From 2d90516f9f7f2f7bb29e49f29b4b3b18a8363518 Mon Sep 17 00:00:00 2001
From: rebuilt <memoryman51@hotmail.com>
Date: Mon, 30 Oct 2023 19:44:52 -0700
Subject: [PATCH] feat: add command for loading survey responses from arbitrary
 sftp directory

---
 README.md           | 25 +++++++++++++++++++------
 lib/tasks/data.rake | 18 ++++++++++++++++++
 2 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index eb915a70..555426f3 100644
--- a/README.md
+++ b/README.md
@@ -130,30 +130,43 @@ How to run the data loading task:
 
 ```bash
 # locally
-$ bundle exec rake data:load_survey_responses
+bundle exec rake data:load_survey_responses
 
 # on heroku staging environment
-$ heroku run:detached -a mciea-beta bundle exec rake data:load_survey_responses
+heroku run:detached -a mciea-beta bundle exec rake data:load_survey_responses
 
 # on heroku production environment
-$ heroku run:detached -a mciea-dashboard bundle exec rake data:load_survey_responses
+heroku run:detached -a mciea-dashboard bundle exec rake data:load_survey_responses
+```
+
+Or if you want to load data from a specific directory
+
+```bash
+# locally
+SFTP_PATH=/data/survey_responses/2022_23 bundle exec rake data:load_survey_responses_from_path
+
+# on heroku staging environment
+heroku run:detached -a mciea-beta SFTP_PATH=/data/survey_responses/2022_23 bundle exec rake data:load_survey_responses_from_path
+
+# on heroku production environment
+heroku run:detached -a mciea-dashboard SFTP_PATH=/data/survey_responses/2022_23 bundle exec rake data:load_survey_responses_from_path
 ```
 
 Or if you only want to load data for Lowell
 
 ```bash
 # locally
-$ bundle exec rake data:load_survey_responses_for_lowell
+bundle exec rake data:load_survey_responses_for_lowell
 ```
 
 For convenience, you can use the following script for loading data on Heroku:
 
 ```bash
 # on heroku staging environment
-$ ./scripts/load_survey_responses_on_heroku beta
+./scripts/load_survey_responses_on_heroku beta
 
 # on heroku production environment
-$ ./scripts/load_survey_responses_on_heroku dashboard
+./scripts/load_survey_responses_on_heroku dashboard
 ```
 
 There is also an example one-off task to load a single csv at a time.
diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake
index 3bead793..73223ff9 100644
--- a/lib/tasks/data.rake
+++ b/lib/tasks/data.rake
@@ -17,6 +17,24 @@ namespace :data do
     Rails.cache.clear
   end
 
+  desc "load survey responses from a specific directory"
+  task load_survey_responses_from_path: :environment do
+    survey_item_response_count = SurveyItemResponse.count
+    student_count = Student.count
+    path = "#{ENV['SFTP_PATH']}"
+    Sftp::Directory.open(path:) do |file|
+      SurveyResponsesDataLoader.new.from_file(file:)
+    end
+    puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
+
+    Sftp::Directory.open(path:) do |file|
+      StudentLoader.from_file(file:, rules: [])
+    end
+    puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
+
+    Rails.cache.clear
+  end
+
   desc "load admin_data"
   task load_admin_data: :environment do
     original_count = AdminDataValue.count