From 7dec0be5afecd574cfff061293ef157418cdfb27 Mon Sep 17 00:00:00 2001
From: Gabe Farrell <gabe@mnrva.dev>
Date: Fri, 23 Jun 2023 18:33:19 +0000
Subject: [PATCH] Add seeding & scripts for Daley 2023-24

---
 data/enrollment/enrollment.csv |  1 +
 db/seeds.rb                    |  2 +-
 lib/tasks/data.rake            | 12 ++++++++
 lib/tasks/one_off.rake         | 53 ++++++++++++++++++++++++++++++++++
 4 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/data/enrollment/enrollment.csv b/data/enrollment/enrollment.csv
index ea153198..89710d9b 100644
--- a/data/enrollment/enrollment.csv
+++ b/data/enrollment/enrollment.csv
@@ -1,4 +1,5 @@
 Raw likert calculation,Likert Score,Admin Data Item,Academic Year,School Name,DESE ID,PK,K,1,2,3,4,5,6,7,8,9,10,11,12,SP,Total
+NA,NA,"",2023-24,Lowell - James S Daley Middle School,01600315,       0,       0,       0,       0,       0,       0,     179,     155,     166,     174,       0,       0,       0,       0,       0,     674
 NA,NA,"",2022-23,Abby Kelley Foster Charter Public (District) - Abby Kelley Foster Charter Public School,04450105,       0,     120,     120,     118,     121,     124,     121,     122,     114,     110,      89,      83,      90,      90,       0,"   1,422"
 NA,NA,"",2022-23,Abington - Abington Early Education Program,00010001,      87,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,      87
 NA,NA,"",2022-23,Abington - Abington High,00010505,       0,       0,       0,       0,       0,       0,       0,       0,       0,       0,     137,     127,     146,     131,       7,     548
diff --git a/db/seeds.rb b/db/seeds.rb
index c6f72ac0..03946580 100644
--- a/db/seeds.rb
+++ b/db/seeds.rb
@@ -2,7 +2,7 @@ require "#{Rails.root}/app/lib/seeder"
 
 seeder = Seeder.new
 
-seeder.seed_academic_years '2016-17', '2017-18', '2018-19', '2019-20', '2020-21', '2021-22', '2022-23'
+seeder.seed_academic_years '2016-17', '2017-18', '2018-19', '2019-20', '2020-21', '2021-22', '2022-23', '2023-24'
 seeder.seed_districts_and_schools Rails.root.join('data', 'master_list_of_schools_and_districts.csv')
 seeder.seed_sqm_framework Rails.root.join('data', 'sqm_framework.csv')
 seeder.seed_demographics Rails.root.join('data', 'demographics.csv')
diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake
index b8c2b11c..1d88bc56 100644
--- a/lib/tasks/data.rake
+++ b/lib/tasks/data.rake
@@ -45,6 +45,18 @@ namespace :data do
     Rails.cache.clear
   end
 
+  desc 'delete non-lowell schools and districts'
+  task delete_non_lowell: :environment do
+    schools = School.all.reject { |s| s.district.name == 'Lowell' }
+    ResponseRate.where(school: schools).delete_all
+    RaceScore.where(school: schools).delete_all
+    Respondent.where(school: schools).delete_all
+    schools.each { |school| school.delete }
+    districts = District.all.reject { |district| district.name == 'Lowell' }
+    districts.each { |district| district.delete }
+  end
+
+
   desc 'load students for lowell'
   task load_students_for_lowell: :environment do
     student_count = Student.count
diff --git a/lib/tasks/one_off.rake b/lib/tasks/one_off.rake
index 722b77c3..6d166864 100644
--- a/lib/tasks/one_off.rake
+++ b/lib/tasks/one_off.rake
@@ -112,6 +112,14 @@ namespace :one_off do
     # should be somewhere near 295738
   end
 
+  desc 'delete 2023-24 survey responses'
+  task delete_survey_responses_2023_24: :environment do
+    response_count = SurveyItemResponse.all.count
+    SurveyItemResponse.where(academic_year: AcademicYear.find_by_range('2023-24')).delete_all
+
+    puts "=====================> Deleted #{response_count - SurveyItemResponse.all.count} survey responses"
+  end
+
   desc 'load survey responses for lowell schools 2022-23'
   task load_survey_responses_for_lowell_2022_23: :environment do
     survey_item_response_count = SurveyItemResponse.count
@@ -141,4 +149,49 @@ namespace :one_off do
     SurveyItemResponse.where(response_id: 'R_27fKhVfyeKGMF5q').delete_all
     SurveyItemResponse.where(response_id: 'R_2cjPX1Ngxr2Hc4c').delete_all
   end
+
+  desc 'upload spring survey results to 23-24'
+  task upload_spring23: :environment do
+      new_files = Array.new
+      input_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'clean')
+      Dir.foreach(input_filepath) do |filename|
+          next if filename.start_with?('.') # skip hidden files and ./.. directories
+          # this can probably be replaced with Dir.join or similar
+          input_filename = Rails.root.join('tmp', 'data', 'rpp_data', 'clean', filename).to_s
+          sftptogo_url = ENV['SFTPTOGO_URL']
+          uri = URI.parse(sftptogo_url)
+          Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
+              puts "Uploading #{filename}..."
+              sftp.upload!(input_filename, "/data/survey_responses/2023-24/#{filename}")
+          end
+          new_files.append(filename)
+      end
+      # print remote directory contents with new files marked
+      path = '/data/survey_responses/2023-24/'
+      Sftp::Directory.open(path:) do |file|
+          # the open method already prints all the contents...
+      end
+  end
+
+  desc 'load spring survey responses'
+  task load_spring_survey_responses: :environment do
+    survey_item_response_count = SurveyItemResponse.count
+    student_count = Student.count
+    path = '/data/survey_responses/2023-24/'
+    Sftp::Directory.open(path:) do |file|
+      SurveyResponsesDataLoader.from_file(file:)
+    end
+    puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses"
+
+    Sftp::Directory.open(path:) do |file|
+      StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools])
+    end
+    puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
+
+    puts 'Resetting race scores'
+    RaceScoreLoader.reset(fast_processing: false)
+    puts "=====================> Completed loading #{RaceScore.count} race scores"
+
+    Rails.cache.clear
+  end
 end