gabehf/Dashboard
1
0
Fork 0
mirror of https://github.com/edcommonwealth/Dashboard.git synced 2026-03-07 21:38:14 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: load survey item responses. Show overview page without server errors

Browse source
This commit is contained in:
Nelson Jovel 2024-01-26 07:43:29 -08:00
parent 589c0f7e11
commit a538eb72f2
9 changed files with 94 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

18
app/services/dashboard/sftp/directory.rb
View file

@ -1,11 +1,12 @@
require 'net/sftp'
require 'uri'
require 'csv'
require "net/sftp"
require "uri"
require "csv"
module Sftp
module Dashboard
module Sftp
class Directory
def self.open(path: '/data/survey_responses/clean', &block)
sftptogo_url = ENV['MCIEA_SFTPTOGO_URL']
def self.open(path: "/data/survey_responses/clean", &block)
sftptogo_url = ENV["ECP_SFTPTOGO_URL"]
uri = URI.parse(sftptogo_url)
Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
sftp.dir.foreach(path) do |entry|
@ -14,16 +15,17 @@ module Sftp
filename = entry.name
puts filename
sftp.file.open(filepath(path:, filename:), 'r', &block)
sftp.file.open(filepath(path:, filename:), "r", &block)
end
end
end
def self.filepath(path:, filename:)
path += '/' unless path.end_with?('/')
path += "/" unless path.end_with?("/")
"#{path}#{filename}"
end
private_class_method :filepath
end
end
end

18
app/services/dashboard/sftp/race_loader.rb
View file

@ -1,22 +1,23 @@
require 'net/sftp'
require 'uri'
require 'csv'
require "net/sftp"
require "uri"
require "csv"
module Sftp
module Dashboard
module Sftp
class RaceLoader
def self.load_data(path: '/data/survey_responses/')
def self.load_data(path: "/data/survey_responses/")
SurveyItemResponse.update_all(student_id: nil)
StudentRace.delete_all
Student.delete_all
sftptogo_url = ENV['SFTPTOGO_URL']
sftptogo_url = ENV["SFTPTOGO_URL"]
uri = URI.parse(sftptogo_url)
Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
sftp.dir.foreach(path) do |entry|
filename = entry.name
puts filename
sftp.file.open(filepath(path:, filename:), 'r') do |f|
sftp.file.open(filepath(path:, filename:), "r") do |f|
StudentLoader.from_file(file: f, rules: [Rule::SkipNonLowellSchools])
end
end
@ -24,10 +25,11 @@ module Sftp
end
def self.filepath(path:, filename:)
path += '/' unless path.end_with?('/')
path += "/" unless path.end_with?("/")
"#{path}#{filename}"
end
private_class_method :filepath
end
end
end

70
app/services/dashboard/survey_responses_data_loader.rb
View file

@ -3,6 +3,7 @@
module Dashboard
class SurveyResponsesDataLoader
def load_data(filepath:)
byebug
File.open(filepath) do |file|
headers = file.first
headers_array = CSV.parse(headers).first
@ -13,8 +14,7 @@ module Dashboard
process_row(row: SurveyItemValues.new(row:, headers: headers_array, survey_items: all_survey_items,
schools:))
end
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500,
on_duplicate_key_update: :all
SurveyItemResponse.upsert_all(survey_item_responses, unique_by: :response_id)
end
end
end
@ -25,25 +25,29 @@ module Dashboard
all_survey_items = survey_items(headers:)
survey_item_responses = []
row_count = 0
# row_count = 0
until file.eof?
line = file.gets
next unless line.present?
CSV.parse(line, headers:).map do |row|
survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array,
values = process_row(row: SurveyItemValues.new(row:, headers: headers_array,
survey_items: all_survey_items, schools:))
survey_item_responses << values if values.present?
end
row_count += 1
next unless row_count == 500
# row_count += 1
# next unless row_count == 500
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all
survey_item_responses = []
row_count = 0
# SurveyItemResponse.upsert_all(survey_item_responses, unique_by: :response_id)
# survey_item_responses = []
# row_count = 0
end
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all
survey_item_responses = survey_item_responses.flatten.compact
SurveyItemResponse.upsert_all(survey_item_responses,
unique_by: %i[response_id dashboard_academic_year_id dashboard_school_id
dashboard_survey_item_id])
# SurveyItemResponse.upsert_all(survey_item_responses, update_only: [:likert_score])
end
private
@ -81,9 +85,9 @@ module Dashboard
def process_survey_items(row:)
student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid)
student.races.delete_all
tmp_races = row.races.map { |race| races[race] }
student.races += tmp_races
# student.races.delete_all
# tmp_races = row.races.map { |race| races[race] }
# student.races += tmp_races
row.survey_items.map do |survey_item|
likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next
@ -93,31 +97,29 @@ module Dashboard
next
end
response = row.survey_item_response(survey_item:)
create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, student:)
build_response(survey_item_response: response, likert_score:, row:, survey_item:, student:)
end.compact
end
def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, student:)
def build_response(survey_item_response:, likert_score:, row:, survey_item:, student:)
gender = genders[row.gender]
grade = row.grade
income = incomes[row.income.parameterize]
ell = ells[row.ell]
sped = speds[row.sped]
recorded_date = row.recorded_date
if survey_item_response.present?
survey_item_response.likert_score = likert_score
survey_item_response.grade = grade
survey_item_response.gender = gender
survey_item_response.recorded_date = row.recorded_date
survey_item_response.income = income
survey_item_response.ell = ell
survey_item_response.sped = sped
survey_item_response.student = student
survey_item_response
else
SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:,
likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:, student:)
end
{ response_id: row.response_id,
dashboard_academic_year_id: row.academic_year.id,
dashboard_school_id: row.school.id,
dashboard_survey_item_id: survey_item.id,
likert_score: likert_score.to_i,
grade:,
dashboard_gender_id: gender.id,
recorded_date:,
dashboard_income_id: income.id,
dashboard_ell_id: ell.id,
dashboard_sped_id: sped.id }
end
def survey_items(headers:)
@ -130,12 +132,4 @@ module Dashboard
.filter { |header| header.start_with? "t-", "s-" }
end
end
module StringMonkeyPatches
def valid_likert_score?
to_i.between? 1, 5
end
end
String.include StringMonkeyPatches
end

2
app/views/dashboard/overview/_quality_framework_indicators.erb
View file

@ -6,7 +6,7 @@
<div class="text-center">
<h3 class="sub-header-3">
<%= link_to [@district, @school, category_presenter, { year: @academic_year.range }] do %>
<%= link_to district_school_category_path(@district, @school, category_presenter, {year: @academic_year.range}) do %>
<%= category_presenter.name %>
<% end %>
</h3>

3
db/migrate/20240104192128_create_dashboard_survey_item_responses.rb
View file

@ -16,5 +16,8 @@ class CreateDashboardSurveyItemResponses < ActiveRecord::Migration[7.1]
t.timestamps
end
add_index :dashboard_survey_item_responses, %i[dashboard_school_id dashboard_academic_year_id]
add_index :dashboard_survey_item_responses,
%i[response_id dashboard_school_id dashboard_academic_year_id dashboard_survey_item_id], unique: true
end
end

10
lib/tasks/dashboard_tasks.rake
View file

@ -1,14 +1,16 @@
include Dashboard
namespace :dashboard do
namespace :data do
desc "load survey responses"
task load_survey_responses: :environment do
survey_item_response_count = SurveyItemResponse.count
survey_item_response_count = Dashboard::SurveyItemResponse.count
student_count = Student.count
path = "/data/survey_responses/clean/"
Sftp::Directory.open(path:) do |file|
SurveyResponsesDataLoader.new.from_file(file:)
::Sftp::Directory.open(path:) do |file|
Dashboard::SurveyResponsesDataLoader.new.from_file(file:)
end
puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
puts "=====================> Completed loading #{Dashboard::SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
Rails.cache.clear
end

1
spec/dummy/config/credentials.yml.enc Normal file
View file

@ -0,0 +1 @@
w0UoY2qMIqo1VCeX8KqsreT2AVVYCfzD4Eh0NVjDnrC+MTHRCto6QnYzzI7R43hle5WeChXFC3MNisW1LYQTmqnfGV6YPk1bYkyzrRo1DA0ObbizYiGnHFPIuTl0PUBxM4dM/dbrHIVa6dJPlupfhsMFKAhITtLZd/cKg6vQsQk32Hdpis4sOSkUqTBC0DcIS4oCjqBW7KOTPft1DyDPFobz9laKoVgWW9sLXwJ3ETQ5VpFjIovTp4/p11GB1CZCbcyw0JnKIzr45RUd//ZIuxAenOcodhABNqmfPMR9Uqyi+keYFc99oA+i7WjOiBd9ZpEtfYdO/voso7A9oxO/BTwOnt+KMCVlPNF6e7G1oWJQhsR8ZWWOYrYGeuIbSaOuNesAgGk6hupSjTyo2h0hGLEF9BznOf5EP86HrnB3L5041wbtyNbB2ers6F/z6xozKi9c2ULdQl93QUoJ/7BB/Lz0G1p+oedp5IIdLmHjS+oGquEf/OdkZdKIqiGFOEAGwTPc98ZJWncI3jW5UYxLJDcoOz9HIiaJX6b+yHKrjv0p79t1f1v3er6J6Hk8TafdO7+sjMcEMZOY5NnAmQumB1s3DXDxYU0588wrk2C2f+G0COTrevznFO8emQHpPuct6nHMuALfNegDCwAPYi4=--bpaVQ6fuv5gN3jZU--8DcQfciEdVpnAQjmtCvQ3g==

1
spec/dummy/config/master.key Normal file
View file

@ -0,0 +1 @@
509ee7d28bf17b106698d4efc62fa139

2
spec/dummy/db/schema.rb
View file

@ -225,10 +225,12 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_04_192128) do
t.index ["dashboard_ell_id"], name: "index_dashboard_survey_item_responses_on_dashboard_ell_id"
t.index ["dashboard_gender_id"], name: "index_dashboard_survey_item_responses_on_dashboard_gender_id"
t.index ["dashboard_income_id"], name: "index_dashboard_survey_item_responses_on_dashboard_income_id"
t.index ["dashboard_school_id", "dashboard_academic_year_id"], name: "idx_on_dashboard_school_id_dashboard_academic_year__44af844634"
t.index ["dashboard_school_id"], name: "index_dashboard_survey_item_responses_on_dashboard_school_id"
t.index ["dashboard_sped_id"], name: "index_dashboard_survey_item_responses_on_dashboard_sped_id"
t.index ["dashboard_student_id"], name: "index_dashboard_survey_item_responses_on_dashboard_student_id"
t.index ["dashboard_survey_item_id"], name: "idx_on_dashboard_survey_item_id_3f6652fbc6"
t.index ["response_id", "dashboard_school_id", "dashboard_academic_year_id", "dashboard_survey_item_id"], name: "idx_on_response_id_dashboard_school_id_dashboard_ac_5b0b3359c0", unique: true
end
create_table "dashboard_survey_items", force: :cascade do |t|