From 6d770938ef5d0f8ec388681421dbdc5b99240e5b Mon Sep 17 00:00:00 2001
From: rebuilt <memoryman51@hotmail.com>
Date: Wed, 15 Feb 2023 17:12:55 -0800
Subject: [PATCH] Swap survey data loading from local files to SFTP

---
 lib/tasks/data.rake    | 62 ++++++++++++++++++++++--------------------
 lib/tasks/one_off.rake | 31 +++++++++++++++++++--
 2 files changed, 61 insertions(+), 32 deletions(-)

diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake
index c8447718..9b4abdb6 100644
--- a/lib/tasks/data.rake
+++ b/lib/tasks/data.rake
@@ -3,27 +3,28 @@ require 'csv'
 namespace :data do
   desc 'load survey responses'
   task load_survey_responses: :environment do
-    survey_item_response_count = surveyitemresponse.count
-    student_count = student.count
-    sftp.directory.open(path: '/data/survey_responses/clean/') do |file|
-      surveyresponsesdataloader.from_file(file:)
+    survey_item_response_count = SurveyItemResponse.count
+    student_count = Student.count
+    path = '/data/survey_responses/clean/'
+    Sftp::Directory.open(path:) do |file|
+      SurveyResponsesDataLoader.from_file(file:)
     end
-    puts "=====================> completed loading #{surveyitemresponse.count - survey_item_response_count} survey responses. #{surveyitemresponse.count} total responses in the database"
+    puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
 
-    sftp.directory.open(path: '/data/survey_responses/clean/') do |file|
-      studentloader.from_file(file:, rules: [rule.skipnonlowellschools])
+    Sftp::Directory.open(path:) do |file|
+      StudentLoader.from_file(file:, rules: [])
     end
-    puts "=====================> completed loading #{student.count - student_count} students. #{student.count} total students"
+    puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
 
-    puts 'resetting response rates'
-    responserateloader.reset
-    puts "=====================> completed loading #{responserate.count} response rates"
+    puts 'Resetting response rates'
+    ResponseRateLoader.reset
+    puts "=====================> Completed loading #{ResponseRate.count} response rates"
 
-    puts 'resetting race scores'
-    racescoreloader.reset(fast_processing: false)
-    puts "=====================> completed loading #{racescore.count} race scores"
+    puts 'Resetting race scores'
+    RaceScoreLoader.reset(fast_processing: false)
+    puts "=====================> Completed loading #{RaceScore.count} race scores"
 
-    rails.cache.clear
+    Rails.cache.clear
   end
 
   desc 'seed only lowell'
@@ -40,27 +41,28 @@ namespace :data do
 
   desc 'load survey responses for lowell schools'
   task load_survey_responses_for_lowell: :environment do
-    survey_item_response_count = surveyitemresponse.count
-    student_count = student.count
-    sftp.directory.open(path: '/data/survey_responses/clean/') do |file|
-      surveyresponsesdataloader.from_file(file:)
+    survey_item_response_count = SurveyItemResponse.count
+    student_count = Student.count
+    path = '/data/survey_responses/clean/'
+    Sftp::Directory.open(path:) do |file|
+      SurveyResponsesDataLoader.from_file(file:)
     end
-    puts "=====================> completed loading #{surveyitemresponse.count - survey_item_response_count} survey responses. #{surveyitemresponse.count} total responses in the database"
+    puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
 
-    sftp.directory.open(path: '/data/survey_responses/clean/') do |file|
-      studentloader.from_file(file:, rules: [rule.skipnonlowellschools])
+    Sftp::Directory.open(path:) do |file|
+      StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools])
     end
-    puts "=====================> completed loading #{student.count - student_count} students. #{student.count} total students"
+    puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
 
-    puts 'resetting response rates'
-    responserateloader.reset
-    puts "=====================> completed loading #{responserate.count} response rates"
+    puts 'Resetting response rates'
+    ResponseRateLoader.reset
+    puts "=====================> Completed loading #{ResponseRate.count} response rates"
 
-    puts 'resetting race scores'
-    racescoreloader.reset(fast_processing: false)
-    puts "=====================> completed loading #{racescore.count} race scores"
+    puts 'Resetting race scores'
+    RaceScoreLoader.reset(fast_processing: false)
+    puts "=====================> Completed loading #{RaceScore.count} race scores"
 
-    rails.cache.clear
+    Rails.cache.clear
   end
 
   desc 'load students for lowell'
diff --git a/lib/tasks/one_off.rake b/lib/tasks/one_off.rake
index c875afc4..f874e762 100644
--- a/lib/tasks/one_off.rake
+++ b/lib/tasks/one_off.rake
@@ -138,11 +138,38 @@ namespace :one_off do
     puts values
   end
 
-  desc 'delete survey item responses for 2016-2017'
-  task delete_survey_item_responses_2016_17: :environment do
+  desc 'delete survey item responses for 2016-2018'
+  task delete_survey_item_responses_2016_18: :environment do
     academic_years = AcademicYear.where(range: %w[2016-17 2017-18])
     response_count = SurveyItemResponse.where(academic_year: academic_years).count
     SurveyItemResponse.where(academic_year: academic_years).delete_all
     puts "Deleted #{response_count} survey item responses"
   end
+
+  desc 'load survey responses for 2016-18'
+  task load_survey_responses_2016_18: :environment do
+    survey_item_response_count = SurveyItemResponse.count
+    academic_years = AcademicYear.where(range: %w[2016-17 2017-18])
+    student_count = Student.count
+    path = '/data/survey_responses/clean/2016-18'
+    Sftp::Directory.open(path:) do |file|
+      SurveyResponsesDataLoader.from_file(file:)
+    end
+    puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
+
+    Sftp::Directory.open(path:) do |file|
+      StudentLoader.from_file(file:, rules: [])
+    end
+    puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
+
+    puts 'Resetting response rates'
+    ResponseRateLoader.reset(academic_years:)
+    puts "=====================> Completed loading #{ResponseRate.count} response rates"
+
+    # puts 'Resetting race scores'
+    # RaceScoreLoader.reset(fast_processing: false)
+    # puts "=====================> Completed loading #{RaceScore.count} race scores"
+
+    Rails.cache.clear
+  end
 end