gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 13:38:18 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: add ability to merge disaggregation data with raw survey data to

Browse source 
produce a cleaned csv with merged income disaggregation columns
This commit is contained in:
Nelson Jovel 2023-06-13 17:41:32 -07:00 • committed by rebuilt
parent fae530d21f
commit 0a2c5e02c5
13 changed files with 1203 additions and 229 deletions
Download patch file Download diff file Expand all files Collapse all files

12
lib/tasks/clean.rake
View file

@ -5,7 +5,8 @@ namespace :clean do
input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'raw')
output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'removed')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
end
desc 'clean prepped data'
@ -13,14 +14,16 @@ namespace :clean do
input_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped')
output_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'prepped', 'removed')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
end
desc 'clean mciea data'
task mciea: :environment do
input_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'raw')
output_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'mciea_data', 'removed')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
end
desc 'clean rpp data'
@ -28,6 +31,7 @@ namespace :clean do
input_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'raw')
output_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'clean')
log_filepath = Rails.root.join('tmp', 'data', 'rpp_data', 'removed')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:).clean
disaggregation_filepath = Rails.root.join('tmp', 'data', 'ecp_data', 'disaggregation')
Cleaner.new(input_filepath:, output_filepath:, log_filepath:, disaggregation_filepath:).clean
end
end