gabehf/Dashboard
1
0
Fork 0
mirror of https://github.com/edcommonwealth/Dashboard.git synced 2026-03-07 13:38:12 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: get cleaner running

Browse source
This commit is contained in:
Nelson Jovel 2024-01-31 16:16:24 -08:00
parent 4874c2c88c
commit da99e220c1
5 changed files with 105 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

131
Gemfile.lock
View file

@ -1,7 +1,7 @@
PATH PATH
remote: . remote: .
specs: specs:
dashboard (0.1.10) dashboard (0.1.11)
bcrypt_pbkdf bcrypt_pbkdf
cssbundling-rails cssbundling-rails
ed25519 ed25519
@ -10,7 +10,6 @@ PATH
net-sftp net-sftp
rails (>= 7.1.2) rails (>= 7.1.2)
rspec-rails rspec-rails
standard_deviation
stimulus-rails stimulus-rails
turbo-rails turbo-rails
watir watir
@ -18,35 +17,35 @@ PATH
GEM GEM
remote: https://rubygems.org/ remote: https://rubygems.org/
specs: specs:
actioncable (7.1.2) actioncable (7.1.3)
actionpack (= 7.1.2) actionpack (= 7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
nio4r (~> 2.0) nio4r (~> 2.0)
websocket-driver (>= 0.6.1) websocket-driver (>= 0.6.1)
zeitwerk (~> 2.6) zeitwerk (~> 2.6)
actionmailbox (7.1.2) actionmailbox (7.1.3)
actionpack (= 7.1.2) actionpack (= 7.1.3)
activejob (= 7.1.2) activejob (= 7.1.3)
activerecord (= 7.1.2) activerecord (= 7.1.3)
activestorage (= 7.1.2) activestorage (= 7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
mail (>= 2.7.1) mail (>= 2.7.1)
net-imap net-imap
net-pop net-pop
net-smtp net-smtp
actionmailer (7.1.2) actionmailer (7.1.3)
actionpack (= 7.1.2) actionpack (= 7.1.3)
actionview (= 7.1.2) actionview (= 7.1.3)
activejob (= 7.1.2) activejob (= 7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
mail (~> 2.5, >= 2.5.4) mail (~> 2.5, >= 2.5.4)
net-imap net-imap
net-pop net-pop
net-smtp net-smtp
rails-dom-testing (~> 2.2) rails-dom-testing (~> 2.2)
actionpack (7.1.2) actionpack (7.1.3)
actionview (= 7.1.2) actionview (= 7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
nokogiri (>= 1.8.5) nokogiri (>= 1.8.5)
racc racc
rack (>= 2.2.4) rack (>= 2.2.4)
@ -54,35 +53,35 @@ GEM
rack-test (>= 0.6.3) rack-test (>= 0.6.3)
rails-dom-testing (~> 2.2) rails-dom-testing (~> 2.2)
rails-html-sanitizer (~> 1.6) rails-html-sanitizer (~> 1.6)
actiontext (7.1.2) actiontext (7.1.3)
actionpack (= 7.1.2) actionpack (= 7.1.3)
activerecord (= 7.1.2) activerecord (= 7.1.3)
activestorage (= 7.1.2) activestorage (= 7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
globalid (>= 0.6.0) globalid (>= 0.6.0)
nokogiri (>= 1.8.5) nokogiri (>= 1.8.5)
actionview (7.1.2) actionview (7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
builder (~> 3.1) builder (~> 3.1)
erubi (~> 1.11) erubi (~> 1.11)
rails-dom-testing (~> 2.2) rails-dom-testing (~> 2.2)
rails-html-sanitizer (~> 1.6) rails-html-sanitizer (~> 1.6)
activejob (7.1.2) activejob (7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
globalid (>= 0.3.6) globalid (>= 0.3.6)
activemodel (7.1.2) activemodel (7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
activerecord (7.1.2) activerecord (7.1.3)
activemodel (= 7.1.2) activemodel (= 7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
timeout (>= 0.4.0) timeout (>= 0.4.0)
activestorage (7.1.2) activestorage (7.1.3)
actionpack (= 7.1.2) actionpack (= 7.1.3)
activejob (= 7.1.2) activejob (= 7.1.3)
activerecord (= 7.1.2) activerecord (= 7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
marcel (~> 1.0) marcel (~> 1.0)
activesupport (7.1.2) activesupport (7.1.3)
base64 base64
bigdecimal bigdecimal
concurrent-ruby (~> 1.0, >= 1.0.2) concurrent-ruby (~> 1.0, >= 1.0.2)
@ -97,19 +96,19 @@ GEM
rake (>= 10.4, < 14.0) rake (>= 10.4, < 14.0)
base64 (0.2.0) base64 (0.2.0)
bcrypt_pbkdf (1.1.0) bcrypt_pbkdf (1.1.0)
bigdecimal (3.1.5) bigdecimal (3.1.6)
bindex (0.8.1) bindex (0.8.1)
builder (3.2.4) builder (3.2.4)
concurrent-ruby (1.2.2) concurrent-ruby (1.2.3)
connection_pool (2.4.1) connection_pool (2.4.1)
crass (1.0.6) crass (1.0.6)
cssbundling-rails (1.3.3) cssbundling-rails (1.4.0)
railties (>= 6.0.0) railties (>= 6.0.0)
date (3.3.4) date (3.3.4)
debug (1.9.1) debug (1.9.1)
irb (~> 1.10) irb (~> 1.10)
reline (>= 0.3.8) reline (>= 0.3.8)
diff-lcs (1.5.0) diff-lcs (1.5.1)
drb (2.2.0) drb (2.2.0)
ruby2_keywords ruby2_keywords
ed25519 (1.3.0) ed25519 (1.3.0)
@ -125,11 +124,11 @@ GEM
activesupport (>= 6.1) activesupport (>= 6.1)
i18n (1.14.1) i18n (1.14.1)
concurrent-ruby (~> 1.0) concurrent-ruby (~> 1.0)
io-console (0.7.1) io-console (0.7.2)
irb (1.11.1) irb (1.11.1)
rdoc rdoc
reline (>= 0.4.2) reline (>= 0.4.2)
jsbundling-rails (1.2.2) jsbundling-rails (1.3.0)
railties (>= 6.0.0) railties (>= 6.0.0)
loofah (2.22.0) loofah (2.22.0)
crass (~> 1.0.2) crass (~> 1.0.2)
@ -141,7 +140,7 @@ GEM
net-smtp net-smtp
marcel (1.0.2) marcel (1.0.2)
mini_mime (1.1.5) mini_mime (1.1.5)
minitest (5.20.0) minitest (5.21.2)
mutex_m (0.2.0) mutex_m (0.2.0)
net-imap (0.4.9.1) net-imap (0.4.9.1)
date date
@ -164,7 +163,7 @@ GEM
puma (6.4.2) puma (6.4.2)
nio4r (~> 2.0) nio4r (~> 2.0)
racc (1.7.3) racc (1.7.3)
rack (3.0.8) rack (3.0.9)
rack-session (2.0.0) rack-session (2.0.0)
rack (>= 3.0.0) rack (>= 3.0.0)
rack-test (2.1.0) rack-test (2.1.0)
@ -172,20 +171,20 @@ GEM
rackup (2.1.0) rackup (2.1.0)
rack (>= 3) rack (>= 3)
webrick (~> 1.8) webrick (~> 1.8)
rails (7.1.2) rails (7.1.3)
actioncable (= 7.1.2) actioncable (= 7.1.3)
actionmailbox (= 7.1.2) actionmailbox (= 7.1.3)
actionmailer (= 7.1.2) actionmailer (= 7.1.3)
actionpack (= 7.1.2) actionpack (= 7.1.3)
actiontext (= 7.1.2) actiontext (= 7.1.3)
actionview (= 7.1.2) actionview (= 7.1.3)
activejob (= 7.1.2) activejob (= 7.1.3)
activemodel (= 7.1.2) activemodel (= 7.1.3)
activerecord (= 7.1.2) activerecord (= 7.1.3)
activestorage (= 7.1.2) activestorage (= 7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
bundler (>= 1.15.0) bundler (>= 1.15.0)
railties (= 7.1.2) railties (= 7.1.3)
rails-dom-testing (2.2.0) rails-dom-testing (2.2.0)
activesupport (>= 5.0.0) activesupport (>= 5.0.0)
minitest minitest
@ -193,9 +192,9 @@ GEM
rails-html-sanitizer (1.6.0) rails-html-sanitizer (1.6.0)
loofah (~> 2.21) loofah (~> 2.21)
nokogiri (~> 1.14) nokogiri (~> 1.14)
railties (7.1.2) railties (7.1.3)
actionpack (= 7.1.2) actionpack (= 7.1.3)
activesupport (= 7.1.2) activesupport (= 7.1.3)
irb irb
rackup (>= 1.0.0) rackup (>= 1.0.0)
rake (>= 12.2) rake (>= 12.2)
@ -216,7 +215,7 @@ GEM
rspec-mocks (3.12.6) rspec-mocks (3.12.6)
diff-lcs (>= 1.2.0, < 2.0) diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.12.0) rspec-support (~> 3.12.0)
rspec-rails (6.1.0) rspec-rails (6.1.1)
actionpack (>= 6.1) actionpack (>= 6.1)
activesupport (>= 6.1) activesupport (>= 6.1)
railties (>= 6.1) railties (>= 6.1)
@ -227,7 +226,8 @@ GEM
rspec-support (3.12.1) rspec-support (3.12.1)
ruby2_keywords (0.0.5) ruby2_keywords (0.0.5)
rubyzip (2.3.2) rubyzip (2.3.2)
selenium-webdriver (4.16.0) selenium-webdriver (4.17.0)
base64 (~> 0.2)
rexml (~> 3.2, >= 3.2.5) rexml (~> 3.2, >= 3.2.5)
rubyzip (>= 1.2.2, < 3.0) rubyzip (>= 1.2.2, < 3.0)
websocket (~> 1.0) websocket (~> 1.0)
@ -276,6 +276,7 @@ DEPENDENCIES
pg pg
puma puma
sprockets-rails sprockets-rails
standard_deviation
web-console web-console
BUNDLED WITH BUNDLED WITH

1
app/services/dashboard/cleaner.rb
View file

@ -14,6 +14,7 @@ module Dashboard
Dir.glob(Rails.root.join(input_filepath, "*.csv")).each do |filepath| Dir.glob(Rails.root.join(input_filepath, "*.csv")).each do |filepath|
puts filepath puts filepath
File.open(filepath) do |file| File.open(filepath) do |file|
puts "opening file"
processed_data = process_raw_file(file:) processed_data = process_raw_file(file:)
processed_data in [headers, clean_csv, log_csv, data] processed_data in [headers, clean_csv, log_csv, data]
return if data.empty? return if data.empty?

2
dashboard.gemspec
View file

@ -30,10 +30,10 @@ Gem::Specification.new do |spec|
spec.add_dependency "net-sftp" spec.add_dependency "net-sftp"
spec.add_dependency "rails", ">= 7.1.2" spec.add_dependency "rails", ">= 7.1.2"
spec.add_dependency "rspec-rails" spec.add_dependency "rspec-rails"
spec.add_dependency "standard_deviation"
spec.add_dependency "stimulus-rails" spec.add_dependency "stimulus-rails"
spec.add_dependency "turbo-rails" spec.add_dependency "turbo-rails"
spec.add_dependency "watir" spec.add_dependency "watir"
spec.add_development_dependency "factory_bot_rails" spec.add_development_dependency "factory_bot_rails"
spec.add_development_dependency "standard_deviation"
end end

2
lib/dashboard/version.rb
View file

@ -1,3 +1,3 @@
module Dashboard module Dashboard
VERSION = "0.1.10" VERSION = "0.1.11"
end end

36
lib/tasks/clean.rake Normal file
View file

@ -0,0 +1,36 @@
namespace :dashboard do
namespace :clean do
# These tasks must be run in their respective project so the correct schools are in the database
desc "clean ecp data"
task ecp: :environment do
input_filepath = Rails.root.join("tmp", "data", "ecp_data", "raw")
output_filepath = Rails.root.join("tmp", "data", "ecp_data", "clean")
log_filepath = Rails.root.join("tmp", "data", "ecp_data", "removed")
puts "running cleaner"
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
desc "clean prepped data"
task prepped: :environment do
input_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped")
output_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "clean")
log_filepath = Rails.root.join("tmp", "data", "ecp_data", "prepped", "removed")
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
desc "clean mciea data"
task mciea: :environment do
input_filepath = Rails.root.join("tmp", "data", "mciea_data", "raw")
output_filepath = Rails.root.join("tmp", "data", "mciea_data", "clean")
log_filepath = Rails.root.join("tmp", "data", "mciea_data", "removed")
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
desc "clean rpp data"
task rpp: :environment do
input_filepath = Rails.root.join("tmp", "data", "rpp_data", "raw")
output_filepath = Rails.root.join("tmp", "data", "rpp_data", "clean")
log_filepath = Rails.root.join("tmp", "data", "rpp_data", "removed")
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
end
end
end