From 1c4cec3fedbd27561170f1ad293dd3d9baa0a5ab Mon Sep 17 00:00:00 2001 From: nelson Date: Fri, 7 Feb 2025 12:51:33 -0800 Subject: [PATCH] chore: rename sftp environment variable --- app/services/sftp/directory.rb | 4 +-- lib/tasks/upload.rake | 52 ++++++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/app/services/sftp/directory.rb b/app/services/sftp/directory.rb index bca3e63c..15dbc34f 100644 --- a/app/services/sftp/directory.rb +++ b/app/services/sftp/directory.rb @@ -4,8 +4,8 @@ require 'uri' module Sftp class Directory def self.open(path: '/data/survey_responses/clean', &block) - sftptogo_url = ENV['SFTPTOGO_URL'] - uri = URI.parse(sftptogo_url) + sftp_url = ENV['SFTP_URL'] + uri = URI.parse(sftp_url) Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| sftp.dir.foreach(path) do |entry| next unless entry.file? diff --git a/lib/tasks/upload.rake b/lib/tasks/upload.rake index 54cdd6b7..a1a2a10d 100644 --- a/lib/tasks/upload.rake +++ b/lib/tasks/upload.rake @@ -7,8 +7,8 @@ namespace :upload do next if filename.start_with?('.') # skip hidden files and ./.. directories # this can probably be replaced with Dir.join or similar input_filename = Rails.root.join('tmp', 'data', 'rpp_data', 'clean', filename).to_s - sftptogo_url = ENV['SFTPTOGO_URL'] - uri = URI.parse(sftptogo_url) + sftp_url = ENV['SFTP_URL'] + uri = URI.parse(sftp_url) Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| puts "Uploading #{filename}..." sftp.upload!(input_filename, "/data/survey_responses/clean/#{filename}") @@ -21,4 +21,50 @@ namespace :upload do # the open method already prints all the contents... end end -end \ No newline at end of file + + desc 'upload cleaned ECP CSVs to the SFTP server' + task ecp: :environment do + new_files = Array.new + input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'clean') + Dir.foreach(input_filepath) do |filename| + next if filename.start_with?('.') # skip hidden files and ./.. directories + # this can probably be replaced with Dir.join or similar + input_filename = Rails.root.join('tmp', 'data', 'ecp_data', 'clean', filename).to_s + sftp_url = ENV['SFTP_URL'] + uri = URI.parse(sftp_url) + Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| + puts "Uploading #{filename}..." + sftp.upload!(input_filename, "/data/survey_responses/clean/#{filename}") + end + new_files.append(filename) + end + # print remote directory contents with new files marked + path = '/data/survey_responses/clean/' + Sftp::Directory.open(path:) do |file| + # the open method already prints all the contents... + end + end + + desc 'upload cleaned MCIEA CSVs to the SFTP server' + task mciea: :environment do + new_files = Array.new + input_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'clean') + Dir.foreach(input_filepath) do |filename| + next if filename.start_with?('.') # skip hidden files and ./.. directories + # this can probably be replaced with Dir.join or similar + input_filename = Rails.root.join('tmp', 'data', 'mciea_data', 'clean', filename).to_s + sftp_url = ENV['SFTP_URL'] + uri = URI.parse(sftp_url) + Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| + puts "Uploading #{filename}..." + sftp.upload!(input_filename, "/data/survey_responses/clean/#{filename}") + end + new_files.append(filename) + end + # print remote directory contents with new files marked + path = '/data/survey_responses/clean/' + Sftp::Directory.open(path:) do |file| + # the open method already prints all the contents... + end + end +end