diff --git a/app/services/dashboard/sftp/directory.rb b/app/services/dashboard/sftp/directory.rb index b8c14b8..65eb505 100644 --- a/app/services/dashboard/sftp/directory.rb +++ b/app/services/dashboard/sftp/directory.rb @@ -1,29 +1,31 @@ -require 'net/sftp' -require 'uri' -require 'csv' +require "net/sftp" +require "uri" +require "csv" -module Sftp - class Directory - def self.open(path: '/data/survey_responses/clean', &block) - sftptogo_url = ENV['MCIEA_SFTPTOGO_URL'] - uri = URI.parse(sftptogo_url) - Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| - sftp.dir.foreach(path) do |entry| - next unless entry.file? +module Dashboard + module Sftp + class Directory + def self.open(path: "/data/survey_responses/clean", &block) + sftptogo_url = ENV["ECP_SFTPTOGO_URL"] + uri = URI.parse(sftptogo_url) + Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| + sftp.dir.foreach(path) do |entry| + next unless entry.file? - filename = entry.name - puts filename + filename = entry.name + puts filename - sftp.file.open(filepath(path:, filename:), 'r', &block) + sftp.file.open(filepath(path:, filename:), "r", &block) + end end end - end - def self.filepath(path:, filename:) - path += '/' unless path.end_with?('/') - "#{path}#{filename}" - end + def self.filepath(path:, filename:) + path += "/" unless path.end_with?("/") + "#{path}#{filename}" + end - private_class_method :filepath + private_class_method :filepath + end end end diff --git a/app/services/dashboard/sftp/race_loader.rb b/app/services/dashboard/sftp/race_loader.rb index 3331938..97e92de 100644 --- a/app/services/dashboard/sftp/race_loader.rb +++ b/app/services/dashboard/sftp/race_loader.rb @@ -1,33 +1,35 @@ -require 'net/sftp' -require 'uri' -require 'csv' +require "net/sftp" +require "uri" +require "csv" -module Sftp - class RaceLoader - def self.load_data(path: '/data/survey_responses/') - SurveyItemResponse.update_all(student_id: nil) - StudentRace.delete_all - Student.delete_all +module Dashboard + module Sftp + class RaceLoader + def self.load_data(path: "/data/survey_responses/") + SurveyItemResponse.update_all(student_id: nil) + StudentRace.delete_all + Student.delete_all - sftptogo_url = ENV['SFTPTOGO_URL'] - uri = URI.parse(sftptogo_url) - Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| - sftp.dir.foreach(path) do |entry| - filename = entry.name - puts filename + sftptogo_url = ENV["SFTPTOGO_URL"] + uri = URI.parse(sftptogo_url) + Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp| + sftp.dir.foreach(path) do |entry| + filename = entry.name + puts filename - sftp.file.open(filepath(path:, filename:), 'r') do |f| - StudentLoader.from_file(file: f, rules: [Rule::SkipNonLowellSchools]) + sftp.file.open(filepath(path:, filename:), "r") do |f| + StudentLoader.from_file(file: f, rules: [Rule::SkipNonLowellSchools]) + end end end end - end - def self.filepath(path:, filename:) - path += '/' unless path.end_with?('/') - "#{path}#{filename}" - end + def self.filepath(path:, filename:) + path += "/" unless path.end_with?("/") + "#{path}#{filename}" + end - private_class_method :filepath + private_class_method :filepath + end end end diff --git a/app/services/dashboard/survey_responses_data_loader.rb b/app/services/dashboard/survey_responses_data_loader.rb index e813953..6671f28 100644 --- a/app/services/dashboard/survey_responses_data_loader.rb +++ b/app/services/dashboard/survey_responses_data_loader.rb @@ -3,6 +3,7 @@ module Dashboard class SurveyResponsesDataLoader def load_data(filepath:) + byebug File.open(filepath) do |file| headers = file.first headers_array = CSV.parse(headers).first @@ -13,8 +14,7 @@ module Dashboard process_row(row: SurveyItemValues.new(row:, headers: headers_array, survey_items: all_survey_items, schools:)) end - SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, - on_duplicate_key_update: :all + SurveyItemResponse.upsert_all(survey_item_responses, unique_by: :response_id) end end end @@ -25,25 +25,29 @@ module Dashboard all_survey_items = survey_items(headers:) survey_item_responses = [] - row_count = 0 + # row_count = 0 until file.eof? line = file.gets next unless line.present? CSV.parse(line, headers:).map do |row| - survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array, - survey_items: all_survey_items, schools:)) + values = process_row(row: SurveyItemValues.new(row:, headers: headers_array, + survey_items: all_survey_items, schools:)) + survey_item_responses << values if values.present? end - row_count += 1 - next unless row_count == 500 + # row_count += 1 + # next unless row_count == 500 - SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all - survey_item_responses = [] - row_count = 0 + # SurveyItemResponse.upsert_all(survey_item_responses, unique_by: :response_id) + # survey_item_responses = [] + # row_count = 0 end - - SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all + survey_item_responses = survey_item_responses.flatten.compact + SurveyItemResponse.upsert_all(survey_item_responses, + unique_by: %i[response_id dashboard_academic_year_id dashboard_school_id + dashboard_survey_item_id]) + # SurveyItemResponse.upsert_all(survey_item_responses, update_only: [:likert_score]) end private @@ -81,9 +85,9 @@ module Dashboard def process_survey_items(row:) student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid) - student.races.delete_all - tmp_races = row.races.map { |race| races[race] } - student.races += tmp_races + # student.races.delete_all + # tmp_races = row.races.map { |race| races[race] } + # student.races += tmp_races row.survey_items.map do |survey_item| likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next @@ -93,31 +97,29 @@ module Dashboard next end response = row.survey_item_response(survey_item:) - create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, student:) + build_response(survey_item_response: response, likert_score:, row:, survey_item:, student:) end.compact end - def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, student:) + def build_response(survey_item_response:, likert_score:, row:, survey_item:, student:) gender = genders[row.gender] grade = row.grade income = incomes[row.income.parameterize] ell = ells[row.ell] sped = speds[row.sped] + recorded_date = row.recorded_date - if survey_item_response.present? - survey_item_response.likert_score = likert_score - survey_item_response.grade = grade - survey_item_response.gender = gender - survey_item_response.recorded_date = row.recorded_date - survey_item_response.income = income - survey_item_response.ell = ell - survey_item_response.sped = sped - survey_item_response.student = student - survey_item_response - else - SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:, - likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:, student:) - end + { response_id: row.response_id, + dashboard_academic_year_id: row.academic_year.id, + dashboard_school_id: row.school.id, + dashboard_survey_item_id: survey_item.id, + likert_score: likert_score.to_i, + grade:, + dashboard_gender_id: gender.id, + recorded_date:, + dashboard_income_id: income.id, + dashboard_ell_id: ell.id, + dashboard_sped_id: sped.id } end def survey_items(headers:) @@ -130,12 +132,4 @@ module Dashboard .filter { |header| header.start_with? "t-", "s-" } end end - - module StringMonkeyPatches - def valid_likert_score? - to_i.between? 1, 5 - end - end - - String.include StringMonkeyPatches end diff --git a/app/views/dashboard/overview/_quality_framework_indicators.erb b/app/views/dashboard/overview/_quality_framework_indicators.erb index f438ae9..8133d02 100644 --- a/app/views/dashboard/overview/_quality_framework_indicators.erb +++ b/app/views/dashboard/overview/_quality_framework_indicators.erb @@ -6,7 +6,7 @@

- <%= link_to [@district, @school, category_presenter, { year: @academic_year.range }] do %> + <%= link_to district_school_category_path(@district, @school, category_presenter, {year: @academic_year.range}) do %> <%= category_presenter.name %> <% end %>

diff --git a/db/migrate/20240104192128_create_dashboard_survey_item_responses.rb b/db/migrate/20240104192128_create_dashboard_survey_item_responses.rb index b2d46d7..abbe1af 100644 --- a/db/migrate/20240104192128_create_dashboard_survey_item_responses.rb +++ b/db/migrate/20240104192128_create_dashboard_survey_item_responses.rb @@ -16,5 +16,8 @@ class CreateDashboardSurveyItemResponses < ActiveRecord::Migration[7.1] t.timestamps end + add_index :dashboard_survey_item_responses, %i[dashboard_school_id dashboard_academic_year_id] + add_index :dashboard_survey_item_responses, + %i[response_id dashboard_school_id dashboard_academic_year_id dashboard_survey_item_id], unique: true end end diff --git a/lib/tasks/dashboard_tasks.rake b/lib/tasks/dashboard_tasks.rake index b1a40a5..e054df7 100644 --- a/lib/tasks/dashboard_tasks.rake +++ b/lib/tasks/dashboard_tasks.rake @@ -1,14 +1,16 @@ +include Dashboard + namespace :dashboard do namespace :data do desc "load survey responses" task load_survey_responses: :environment do - survey_item_response_count = SurveyItemResponse.count + survey_item_response_count = Dashboard::SurveyItemResponse.count student_count = Student.count path = "/data/survey_responses/clean/" - Sftp::Directory.open(path:) do |file| - SurveyResponsesDataLoader.new.from_file(file:) + ::Sftp::Directory.open(path:) do |file| + Dashboard::SurveyResponsesDataLoader.new.from_file(file:) end - puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" + puts "=====================> Completed loading #{Dashboard::SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database" Rails.cache.clear end diff --git a/spec/dummy/config/credentials.yml.enc b/spec/dummy/config/credentials.yml.enc new file mode 100644 index 0000000..944193a --- /dev/null +++ b/spec/dummy/config/credentials.yml.enc @@ -0,0 +1 @@ +w0UoY2qMIqo1VCeX8KqsreT2AVVYCfzD4Eh0NVjDnrC+MTHRCto6QnYzzI7R43hle5WeChXFC3MNisW1LYQTmqnfGV6YPk1bYkyzrRo1DA0ObbizYiGnHFPIuTl0PUBxM4dM/dbrHIVa6dJPlupfhsMFKAhITtLZd/cKg6vQsQk32Hdpis4sOSkUqTBC0DcIS4oCjqBW7KOTPft1DyDPFobz9laKoVgWW9sLXwJ3ETQ5VpFjIovTp4/p11GB1CZCbcyw0JnKIzr45RUd//ZIuxAenOcodhABNqmfPMR9Uqyi+keYFc99oA+i7WjOiBd9ZpEtfYdO/voso7A9oxO/BTwOnt+KMCVlPNF6e7G1oWJQhsR8ZWWOYrYGeuIbSaOuNesAgGk6hupSjTyo2h0hGLEF9BznOf5EP86HrnB3L5041wbtyNbB2ers6F/z6xozKi9c2ULdQl93QUoJ/7BB/Lz0G1p+oedp5IIdLmHjS+oGquEf/OdkZdKIqiGFOEAGwTPc98ZJWncI3jW5UYxLJDcoOz9HIiaJX6b+yHKrjv0p79t1f1v3er6J6Hk8TafdO7+sjMcEMZOY5NnAmQumB1s3DXDxYU0588wrk2C2f+G0COTrevznFO8emQHpPuct6nHMuALfNegDCwAPYi4=--bpaVQ6fuv5gN3jZU--8DcQfciEdVpnAQjmtCvQ3g== \ No newline at end of file diff --git a/spec/dummy/config/master.key b/spec/dummy/config/master.key new file mode 100644 index 0000000..9c32c38 --- /dev/null +++ b/spec/dummy/config/master.key @@ -0,0 +1 @@ +509ee7d28bf17b106698d4efc62fa139 \ No newline at end of file diff --git a/spec/dummy/db/schema.rb b/spec/dummy/db/schema.rb index d1f45bb..9991353 100644 --- a/spec/dummy/db/schema.rb +++ b/spec/dummy/db/schema.rb @@ -225,10 +225,12 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_04_192128) do t.index ["dashboard_ell_id"], name: "index_dashboard_survey_item_responses_on_dashboard_ell_id" t.index ["dashboard_gender_id"], name: "index_dashboard_survey_item_responses_on_dashboard_gender_id" t.index ["dashboard_income_id"], name: "index_dashboard_survey_item_responses_on_dashboard_income_id" + t.index ["dashboard_school_id", "dashboard_academic_year_id"], name: "idx_on_dashboard_school_id_dashboard_academic_year__44af844634" t.index ["dashboard_school_id"], name: "index_dashboard_survey_item_responses_on_dashboard_school_id" t.index ["dashboard_sped_id"], name: "index_dashboard_survey_item_responses_on_dashboard_sped_id" t.index ["dashboard_student_id"], name: "index_dashboard_survey_item_responses_on_dashboard_student_id" t.index ["dashboard_survey_item_id"], name: "idx_on_dashboard_survey_item_id_3f6652fbc6" + t.index ["response_id", "dashboard_school_id", "dashboard_academic_year_id", "dashboard_survey_item_id"], name: "idx_on_response_id_dashboard_school_id_dashboard_ac_5b0b3359c0", unique: true end create_table "dashboard_survey_items", force: :cascade do |t|