From 6f265302ea976c9d62398639498a09e8f363aa9c Mon Sep 17 00:00:00 2001 From: Nelson Jovel Date: Mon, 11 Dec 2023 15:39:20 -0800 Subject: [PATCH] feat: if the filename includes the words 'form' or 'part' add that to the resulting cleaned filename --- app/services/cleaner.rb | 36 ++- ...epped_2022_23_student_survey_responses.csv | 294 ------------------ ...epped_2022_23_teacher_survey_responses.csv | 38 --- .../sample_file_with_duplicate_headers.csv | 2 + .../raw/sample_maynard_raw_student_survey.csv | 72 ++--- spec/services/cleaner_spec.rb | 84 ++++- spec/spec_helper.rb | 11 + 7 files changed, 149 insertions(+), 388 deletions(-) delete mode 100644 spec/fixtures/raw/prepped_2022_23_student_survey_responses.csv delete mode 100644 spec/fixtures/raw/prepped_2022_23_teacher_survey_responses.csv create mode 100644 spec/fixtures/raw/sample_file_with_duplicate_headers.csv diff --git a/app/services/cleaner.rb b/app/services/cleaner.rb index 65e7414b..dbf1829b 100644 --- a/app/services/cleaner.rb +++ b/app/services/cleaner.rb @@ -17,14 +17,15 @@ class Cleaner processed_data in [headers, clean_csv, log_csv, data] return if data.empty? - filename = filename(headers:, data:) + filename = filename(headers:, data:, filepath:) write_csv(data: clean_csv, output_filepath:, filename:) write_csv(data: log_csv, output_filepath: log_filepath, prefix: "removed.", filename:) end end end - def filename(headers:, data:) + def filename(headers:, data:, filepath:) + output = [] survey_item_ids = headers.filter(&:present?).filter do |header| header.start_with?("s-", "t-") end.reject { |item| item.end_with? "-1" } @@ -35,15 +36,35 @@ class Cleaner row.district.short_name end.to_set.to_a - districts.join(".").to_s + "." + survey_type.to_s + "." + range + ".csv" + schools = data.map do |row| + row.school.name + end.to_set + + part = filepath&.match(/[\b\s_.]+(part|form)[\W*_](?