chore: rename sftp environment variable

main-eol
nelson 10 months ago
parent 934a306b8a
commit 3d2b32cebf

@ -4,8 +4,8 @@ require 'uri'
module Sftp module Sftp
class Directory class Directory
def self.open(path: '/data/survey_responses/clean', &block) def self.open(path: '/data/survey_responses/clean', &block)
sftptogo_url = ENV['ECP_SFTPTOGO_URL'] sftp_url = ENV['SFTP_URL']
uri = URI.parse(sftptogo_url) uri = URI.parse(sftp_url)
Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
sftp.dir.foreach(path) do |entry| sftp.dir.foreach(path) do |entry|
next unless entry.file? next unless entry.file?

@ -1,32 +0,0 @@
require 'net/sftp'
require 'uri'
module Sftp
class RaceLoader
def self.load_data(path: '/data/survey_responses/')
SurveyItemResponse.update_all(student_id: nil)
StudentRace.delete_all
Student.delete_all
sftptogo_url = ENV['SFTPTOGO_URL']
uri = URI.parse(sftptogo_url)
Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
sftp.dir.foreach(path) do |entry|
filename = entry.name
puts filename
sftp.file.open(filepath(path:, filename:), 'r') do |f|
StudentLoader.from_file(file: f, rules: [])
end
end
end
end
def self.filepath(path:, filename:)
path += '/' unless path.end_with?('/')
"#{path}#{filename}"
end
private_class_method :filepath
end
end

@ -7,8 +7,8 @@ namespace :upload do
next if filename.start_with?('.') # skip hidden files and ./.. directories next if filename.start_with?('.') # skip hidden files and ./.. directories
# this can probably be replaced with Dir.join or similar # this can probably be replaced with Dir.join or similar
input_filename = Rails.root.join('tmp', 'data', 'rpp_data', 'clean', filename).to_s input_filename = Rails.root.join('tmp', 'data', 'rpp_data', 'clean', filename).to_s
sftptogo_url = ENV['SFTPTOGO_URL'] sftp_url = ENV['SFTP_URL']
uri = URI.parse(sftptogo_url) uri = URI.parse(sftp_url)
Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
puts "Uploading #{filename}..." puts "Uploading #{filename}..."
sftp.upload!(input_filename, "/data/survey_responses/clean/#{filename}") sftp.upload!(input_filename, "/data/survey_responses/clean/#{filename}")
@ -21,7 +21,7 @@ namespace :upload do
# the open method already prints all the contents... # the open method already prints all the contents...
end end
end end
desc 'upload cleaned ECP CSVs to the SFTP server' desc 'upload cleaned ECP CSVs to the SFTP server'
task ecp: :environment do task ecp: :environment do
new_files = Array.new new_files = Array.new
@ -30,8 +30,8 @@ namespace :upload do
next if filename.start_with?('.') # skip hidden files and ./.. directories next if filename.start_with?('.') # skip hidden files and ./.. directories
# this can probably be replaced with Dir.join or similar # this can probably be replaced with Dir.join or similar
input_filename = Rails.root.join('tmp', 'data', 'ecp_data', 'clean', filename).to_s input_filename = Rails.root.join('tmp', 'data', 'ecp_data', 'clean', filename).to_s
sftptogo_url = ENV['ECP_SFTPTOGO_URL'] sftp_url = ENV['SFTP_URL']
uri = URI.parse(sftptogo_url) uri = URI.parse(sftp_url)
Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
puts "Uploading #{filename}..." puts "Uploading #{filename}..."
sftp.upload!(input_filename, "/data/survey_responses/clean/#{filename}") sftp.upload!(input_filename, "/data/survey_responses/clean/#{filename}")
@ -44,7 +44,7 @@ namespace :upload do
# the open method already prints all the contents... # the open method already prints all the contents...
end end
end end
desc 'upload cleaned MCIEA CSVs to the SFTP server' desc 'upload cleaned MCIEA CSVs to the SFTP server'
task mciea: :environment do task mciea: :environment do
new_files = Array.new new_files = Array.new
@ -53,8 +53,8 @@ namespace :upload do
next if filename.start_with?('.') # skip hidden files and ./.. directories next if filename.start_with?('.') # skip hidden files and ./.. directories
# this can probably be replaced with Dir.join or similar # this can probably be replaced with Dir.join or similar
input_filename = Rails.root.join('tmp', 'data', 'mciea_data', 'clean', filename).to_s input_filename = Rails.root.join('tmp', 'data', 'mciea_data', 'clean', filename).to_s
sftptogo_url = ENV['MCIEA_SFTPTOGO_URL'] sftp_url = ENV['SFTP_URL']
uri = URI.parse(sftptogo_url) uri = URI.parse(sftp_url)
Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
puts "Uploading #{filename}..." puts "Uploading #{filename}..."
sftp.upload!(input_filename, "/data/survey_responses/clean/#{filename}") sftp.upload!(input_filename, "/data/survey_responses/clean/#{filename}")
@ -67,4 +67,4 @@ namespace :upload do
# the open method already prints all the contents... # the open method already prints all the contents...
end end
end end
end end

Loading…
Cancel
Save