Add seeding & scripts for Daley 2023-24

lib/tasks/data.rake

@@ -45,6 +45,18 @@ namespace :data do
     Rails.cache.clear
   end
 
+  desc 'delete non-lowell schools and districts'
+  task delete_non_lowell: :environment do
+    schools = School.all.reject { |s| s.district.name == 'Lowell' }
+    ResponseRate.where(school: schools).delete_all
+    RaceScore.where(school: schools).delete_all
+    Respondent.where(school: schools).delete_all
+    schools.each { |school| school.delete }
+    districts = District.all.reject { |district| district.name == 'Lowell' }
+    districts.each { |district| district.delete }
+  end
+
+
   desc 'load students for lowell'
   task load_students_for_lowell: :environment do
     student_count = Student.count

lib/tasks/one_off.rake

@@ -112,6 +112,14 @@ namespace :one_off do
     # should be somewhere near 295738
   end
 
+  desc 'delete 2023-24 survey responses'
+  task delete_survey_responses_2023_24: :environment do
+    response_count = SurveyItemResponse.all.count
+    SurveyItemResponse.where(academic_year: AcademicYear.find_by_range('2023-24')).delete_all
+
+    puts "=====================> Deleted #{response_count - SurveyItemResponse.all.count} survey responses"
+  end
+
   desc 'load survey responses for lowell schools 2022-23'
   task load_survey_responses_for_lowell_2022_23: :environment do
     survey_item_response_count = SurveyItemResponse.count
@@ -141,4 +149,49 @@ namespace :one_off do
     SurveyItemResponse.where(response_id: 'R_27fKhVfyeKGMF5q').delete_all
     SurveyItemResponse.where(response_id: 'R_2cjPX1Ngxr2Hc4c').delete_all
   end
+
+  desc 'upload spring survey results to 23-24'
+  task upload_spring23: :environment do
+      new_files = Array.new
+      input_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'clean')
+      Dir.foreach(input_filepath) do |filename|
+          next if filename.start_with?('.') # skip hidden files and ./.. directories
+          # this can probably be replaced with Dir.join or similar
+          input_filename = Rails.root.join('tmp', 'data', 'rpp_data', 'clean', filename).to_s
+          sftptogo_url = ENV['SFTPTOGO_URL']
+          uri = URI.parse(sftptogo_url)
+          Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
+              puts "Uploading #{filename}..."
+              sftp.upload!(input_filename, "/data/survey_responses/2023-24/#{filename}")
+          end
+          new_files.append(filename)
+      end
+      # print remote directory contents with new files marked
+      path = '/data/survey_responses/2023-24/'
+      Sftp::Directory.open(path:) do |file|
+          # the open method already prints all the contents...
+      end
+  end
+
+  desc 'load spring survey responses'
+  task load_spring_survey_responses: :environment do
+    survey_item_response_count = SurveyItemResponse.count
+    student_count = Student.count
+    path = '/data/survey_responses/2023-24/'
+    Sftp::Directory.open(path:) do |file|
+      SurveyResponsesDataLoader.from_file(file:)
+    end
+    puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses"
+
+    Sftp::Directory.open(path:) do |file|
+      StudentLoader.from_file(file:, rules: [Rule::SkipNonLowellSchools])
+    end
+    puts "=====================> Completed loading #{Student.count - student_count} students. #{Student.count} total students"
+
+    puts 'Resetting race scores'
+    RaceScoreLoader.reset(fast_processing: false)
+    puts "=====================> Completed loading #{RaceScore.count} race scores"
+
+    Rails.cache.clear
+  end
 end