gabehf/Dashboard
1
0
Fork 0
mirror of https://github.com/edcommonwealth/Dashboard.git synced 2026-03-07 21:38:14 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: make sure CSV and File objects are properly namespaced

Browse source
This commit is contained in:
Nelson Jovel 2024-02-07 15:39:27 -08:00
parent a99ed183a1
commit ed1310f93c
34 changed files with 327 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

4
app/lib/dashboard/seeder.rb
View file

@ -12,7 +12,7 @@ module Dashboard
def seed_districts_and_schools(csv_file)
dese_ids = []
schools = []
CSV.parse(File.read(csv_file), headers: true) do |row|
::CSV.parse(::File.read(csv_file), headers: true) do |row|
district_name = row["District"].strip
district_code = row["District Code"].try(:strip)
@ -39,7 +39,7 @@ module Dashboard
def seed_sqm_framework(csv_file)
admin_data_item_ids = []
CSV.parse(File.read(csv_file), headers: true) do |row|
::CSV.parse(::File.read(csv_file), headers: true) do |row|
category_id = row["Category ID"].strip
category_name = row["Category"].strip
category = Category.find_or_create_by!(category_id:)

14
app/services/dashboard/cleaner.rb
View file

@ -13,7 +13,7 @@ module Dashboard
def clean
Dir.glob(Rails.root.join(input_filepath, "*.csv")).each do |filepath|
puts filepath
File.open(filepath) do |file|
::File.open(filepath) do |file|
puts "opening file"
processed_data = process_raw_file(file:)
processed_data in [headers, clean_csv, log_csv, data]
@ -67,7 +67,7 @@ module Dashboard
clean_csv = []
log_csv = []
data = []
headers = CSV.parse(file.first).first
headers = ::CSV.parse(file.first).first
duplicate_header = headers.detect { |header| headers.count(header) > 1 }
unless duplicate_header.nil?
puts "\n>>>>>>>>>>>>>>>>>> Duplicate header found. This will misalign column headings. Please delete or rename the duplicate column: #{duplicate_header} \n>>>>>>>>>>>>>> \n"
@ -85,7 +85,7 @@ module Dashboard
all_survey_items = survey_items(headers:)
file.lazy.each_slice(1000) do |lines|
CSV.parse(lines.join, headers:).map do |row|
::CSV.parse(lines.join, headers:).map do |row|
values = SurveyItemValues.new(row:, headers:,
survey_items: all_survey_items, schools:)
next unless values.valid_school?
@ -121,12 +121,12 @@ module Dashboard
end
def write_csv(data:, output_filepath:, filename:, prefix: "")
csv = CSV.generate do |csv|
csv = ::CSV.generate do |csv|
data.each do |row|
csv << row
end
end
File.write(output_filepath.join(prefix + filename), csv)
::File.write(output_filepath.join(prefix + filename), csv)
end
def schools
@ -145,11 +145,11 @@ module Dashboard
end
def create_ouput_directory
FileUtils.mkdir_p output_filepath
::FileUtils.mkdir_p output_filepath
end
def create_log_directory
FileUtils.mkdir_p log_filepath
::FileUtils.mkdir_p log_filepath
end
end
end

2
app/services/dashboard/demographic_loader.rb
View file

@ -5,7 +5,7 @@ require "csv"
module Dashboard
class DemographicLoader
def self.load_data(filepath:)
CSV.parse(File.read(filepath), headers: true) do |row|
::CSV.parse(::File.read(filepath), headers: true) do |row|
process_race(row:)
process_gender(row:)
create_from_column(column: "Income", row:, model: Income)

2
app/services/dashboard/dese/enrollments.rb
View file

@ -26,7 +26,7 @@ module Dashboard
def student_count(filepath:, dese_id:, year:)
@students ||= {}
if @students.count == 0
CSV.parse(File.read(filepath), headers: true).map do |row|
::CSV.parse(::File.read(filepath), headers: true).map do |row|
academic_year = row["Academic Year"]
school_id = row["DESE ID"].to_i
total = row["Total"].gsub(",", "").to_i

2
app/services/dashboard/dese/loader.rb
View file

@ -5,7 +5,7 @@ module Dashboard
class Loader
def self.load_data(filepath:)
admin_data_values = []
CSV.parse(File.read(filepath), headers: true) do |row|
::CSV.parse(::File.read(filepath), headers: true) do |row|
score = likert_score(row:)
next unless valid_likert_score(likert_score: score)

2
app/services/dashboard/dese/three_b_two.rb
View file

@ -76,7 +76,7 @@ module Dashboard
@teachers ||= {}
@years_with_data ||= Set.new
if @teachers.count == 0
CSV.parse(File.read(filepath), headers: true).map do |row|
::CSV.parse(::File.read(filepath), headers: true).map do |row|
academic_year = row["Academic Year"]
@years_with_data << academic_year
school_id = row["DESE ID"].to_i

2
app/services/dashboard/enrollment_loader.rb
View file

@ -6,7 +6,7 @@ module Dashboard
class EnrollmentLoader
def load_data(filepath:)
enrollments = []
CSV.parse(File.read(filepath), headers: true) do |row|
::CSV.parse(::File.read(filepath), headers: true) do |row|
row = EnrollmentRowValues.new(row:, schools: school_hash, academic_years: academic_year_hash)
next unless row.school.present? && row.academic_year.present?

2
app/services/dashboard/staffing_loader.rb
View file

@ -7,7 +7,7 @@ module Dashboard
def self.load_data(filepath:)
schools = []
respondents = []
CSV.parse(File.read(filepath), headers: true) do |row|
::CSV.parse(::File.read(filepath), headers: true) do |row|
row = StaffingRowValues.new(row:)
next unless row.school.present? && row.academic_year.present?

16
app/services/dashboard/survey_responses_data_loader.rb
View file

@ -3,13 +3,13 @@
module Dashboard
class SurveyResponsesDataLoader
def load_data(filepath:)
File.open(filepath) do |file|
::File.open(filepath) do |file|
headers = file.first
headers_array = CSV.parse(headers).first
headers_array = ::CSV.parse(headers).first
all_survey_items = survey_items(headers:)
file.lazy.each_slice(500) do |lines|
survey_item_responses = CSV.parse(lines.join, headers:).map do |row|
survey_item_responses = ::CSV.parse(lines.join, headers:).map do |row|
process_row(row: SurveyItemValues.new(row:, headers: headers_array, survey_items: all_survey_items,
schools:))
end
@ -22,7 +22,7 @@ module Dashboard
def from_file(file:)
headers = file.gets
headers_array = CSV.parse(headers).first
headers_array = ::CSV.parse(headers).first
all_survey_items = survey_items(headers:)
survey_item_responses = []
@ -31,7 +31,7 @@ module Dashboard
line = file.gets
next unless line.present?
CSV.parse(line, headers:).map do |row|
::CSV.parse(line, headers:).map do |row|
values = process_row(row: SurveyItemValues.new(row:, headers: headers_array,
survey_items: all_survey_items, schools:))
survey_item_responses << values if values.present?
@ -132,9 +132,9 @@ module Dashboard
end
def get_survey_item_ids_from_headers(headers:)
CSV.parse(headers).first
.filter(&:present?)
.filter { |header| header.start_with? "t-", "s-" }
::CSV.parse(headers).first
.filter(&:present?)
.filter { |header| header.start_with? "t-", "s-" }
end
end
end