gabehf/sqm-dashboards
1
0
Fork 0
mirror of https://github.com/edcommonwealth/sqm-dashboards.git synced 2026-03-07 21:48:16 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: if the filename includes the words 'form' or 'part' add that to the resulting cleaned filename

Browse source
This commit is contained in:
Nelson Jovel 2023-12-11 15:39:20 -08:00
parent d90a83e510
commit f028e6c884
2 changed files with 68 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

20
app/services/cleaner.rb
View file

@ -17,14 +17,15 @@ class Cleaner
processed_data in [headers, clean_csv, log_csv, data]
return if data.empty?
filename = filename(headers:, data:)
filename = filename(headers:, data:, filepath:)
write_csv(data: clean_csv, output_filepath:, filename:)
write_csv(data: log_csv, output_filepath: log_filepath, prefix: "removed.", filename:)
end
end
end
def filename(headers:, data:)
def filename(headers:, data:, filepath:)
output = []
survey_item_ids = headers.filter(&:present?).filter do |header|
header.start_with?("s-", "t-")
end.reject { |item| item.end_with? "-1" }
@ -39,11 +40,18 @@ class Cleaner
row.school.name
end.to_set
# Only add school to filename when there's a single school
school_name = ""
school_name = schools.first.parameterize + "." if schools.length == 1
part = filepath&.match(/[\b\s_.]+(part|form)[\W*_](?<label>[\w\d])/i)&.named_captures&.[]("label")&.upcase
# byebug unless filepath.nil?
districts.join(".").to_s + "." + school_name + survey_type.to_s + "." + range + ".csv"
school_name = schools.first.parameterize
output << districts.join(".")
output << school_name if schools.length == 1
output << survey_type.to_s
output << "Part-" + part unless part.nil?
output << range
output << "csv"
output.join(".")
end
def process_raw_file(file:)

59
spec/services/cleaner_spec.rb
View file

@ -154,7 +154,7 @@ RSpec.describe Cleaner do
data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: standard_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: standard_survey_items, data:
headers: standard_survey_items, data:, filepath: nil
)
expect(filename).to eq "maynard.maynard-high-school.standard.2022-23.csv"
end
@ -166,7 +166,7 @@ RSpec.describe Cleaner do
data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: short_form_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: short_form_survey_items, data:
headers: short_form_survey_items, data:, filepath: nil
)
expect(filename).to eq "maynard.maynard-high-school.short_form.2022-23.csv"
end
@ -179,7 +179,7 @@ RSpec.describe Cleaner do
data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: early_education_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: early_education_survey_items, data:
headers: early_education_survey_items, data:, filepath: nil
)
expect(filename).to eq "maynard.maynard-high-school.early_education.2022-23.csv"
end
@ -191,7 +191,7 @@ RSpec.describe Cleaner do
data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: teacher_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: teacher_survey_items, data:
headers: teacher_survey_items, data:, filepath: nil
)
expect(filename).to eq "maynard.maynard-high-school.teacher.2022-23.csv"
end
@ -205,11 +205,60 @@ RSpec.describe Cleaner do
SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "222_222" },
headers: teacher_survey_items, genders: nil, survey_items:, schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: teacher_survey_items, data:
headers: teacher_survey_items, data:, filepath: nil
)
expect(filename).to eq "maynard.district2.teacher.2022-23.csv"
end
end
context "when the file name includes the words 'part'" do
it "adds the part to the filename" do
survey_items = SurveyItem.where(survey_item_id: early_education_survey_items)
data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: early_education_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: early_education_survey_items, data:, filepath: "/data/survey_responses/maynard early ed_ form a.2022-23.csv"
)
expect(filename).to eq "maynard.maynard-high-school.early_education.Part-A.2022-23.csv"
end
end
context "when the file name includes the words 'form'" do
it "adds the part to the filename" do
survey_items = SurveyItem.where(survey_item_id: early_education_survey_items)
data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: early_education_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: early_education_survey_items, data:, filepath: "/data/survey_responses/maynard early ed_ form f.2022-23.csv"
)
expect(filename).to eq "maynard.maynard-high-school.early_education.Part-F.2022-23.csv"
end
end
context "when the file name includes the words 'form'" do
it "adds the part to the filename" do
survey_items = SurveyItem.where(survey_item_id: early_education_survey_items)
data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: early_education_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: early_education_survey_items, data:, filepath: "/data/survey_responses/maynard early ed_ form_f.2022-23.csv"
)
expect(filename).to eq "maynard.maynard-high-school.early_education.Part-F.2022-23.csv"
end
end
context "when the file name includes the words 'form'" do
it "adds the part to the filename" do
survey_items = SurveyItem.where(survey_item_id: early_education_survey_items)
data = [SurveyItemValues.new(row: { "Recorded Date" => recorded_date, "Dese ID" => "1_740_505" }, headers: early_education_survey_items, genders: nil, survey_items:,
schools: School.school_hash)]
filename = Cleaner.new(input_filepath:, output_filepath:, log_filepath:).filename(
headers: early_education_survey_items, data:, filepath: "/data/survey_responses/maynard early ed_ form&f.2022-23.csv"
)
expect(filename).to eq "maynard.maynard-high-school.early_education.Part-F.2022-23.csv"
end
end
end
end
end