gabehf/Dashboard
1
0
Fork 0
mirror of https://github.com/edcommonwealth/Dashboard.git synced 2026-03-07 13:38:12 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: load survey item responses. Show overview page without server errors

Browse source
This commit is contained in:
Nelson Jovel 2024-01-26 07:43:29 -08:00
parent 589c0f7e11
commit a538eb72f2
9 changed files with 94 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

42
app/services/dashboard/sftp/directory.rb
View file

@ -1,29 +1,31 @@
require 'net/sftp'
require 'uri'
require 'csv'
require "net/sftp"
require "uri"
require "csv"
module Sftp
class Directory
def self.open(path: '/data/survey_responses/clean', &block)
sftptogo_url = ENV['MCIEA_SFTPTOGO_URL']
uri = URI.parse(sftptogo_url)
Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
sftp.dir.foreach(path) do |entry|
next unless entry.file?
module Dashboard
module Sftp
class Directory
def self.open(path: "/data/survey_responses/clean", &block)
sftptogo_url = ENV["ECP_SFTPTOGO_URL"]
uri = URI.parse(sftptogo_url)
Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
sftp.dir.foreach(path) do |entry|
next unless entry.file?
filename = entry.name
puts filename
filename = entry.name
puts filename
sftp.file.open(filepath(path:, filename:), 'r', &block)
sftp.file.open(filepath(path:, filename:), "r", &block)
end
end
end
end
def self.filepath(path:, filename:)
path += '/' unless path.end_with?('/')
"#{path}#{filename}"
end
def self.filepath(path:, filename:)
path += "/" unless path.end_with?("/")
"#{path}#{filename}"
end
private_class_method :filepath
private_class_method :filepath
end
end
end

48
app/services/dashboard/sftp/race_loader.rb
View file

@ -1,33 +1,35 @@
require 'net/sftp'
require 'uri'
require 'csv'
require "net/sftp"
require "uri"
require "csv"
module Sftp
class RaceLoader
def self.load_data(path: '/data/survey_responses/')
SurveyItemResponse.update_all(student_id: nil)
StudentRace.delete_all
Student.delete_all
module Dashboard
module Sftp
class RaceLoader
def self.load_data(path: "/data/survey_responses/")
SurveyItemResponse.update_all(student_id: nil)
StudentRace.delete_all
Student.delete_all
sftptogo_url = ENV['SFTPTOGO_URL']
uri = URI.parse(sftptogo_url)
Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
sftp.dir.foreach(path) do |entry|
filename = entry.name
puts filename
sftptogo_url = ENV["SFTPTOGO_URL"]
uri = URI.parse(sftptogo_url)
Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
sftp.dir.foreach(path) do |entry|
filename = entry.name
puts filename
sftp.file.open(filepath(path:, filename:), 'r') do |f|
StudentLoader.from_file(file: f, rules: [Rule::SkipNonLowellSchools])
sftp.file.open(filepath(path:, filename:), "r") do |f|
StudentLoader.from_file(file: f, rules: [Rule::SkipNonLowellSchools])
end
end
end
end
end
def self.filepath(path:, filename:)
path += '/' unless path.end_with?('/')
"#{path}#{filename}"
end
def self.filepath(path:, filename:)
path += "/" unless path.end_with?("/")
"#{path}#{filename}"
end
private_class_method :filepath
private_class_method :filepath
end
end
end

72
app/services/dashboard/survey_responses_data_loader.rb
View file

@ -3,6 +3,7 @@
module Dashboard
class SurveyResponsesDataLoader
def load_data(filepath:)
byebug
File.open(filepath) do |file|
headers = file.first
headers_array = CSV.parse(headers).first
@ -13,8 +14,7 @@ module Dashboard
process_row(row: SurveyItemValues.new(row:, headers: headers_array, survey_items: all_survey_items,
schools:))
end
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500,
on_duplicate_key_update: :all
SurveyItemResponse.upsert_all(survey_item_responses, unique_by: :response_id)
end
end
end
@ -25,25 +25,29 @@ module Dashboard
all_survey_items = survey_items(headers:)
survey_item_responses = []
row_count = 0
# row_count = 0
until file.eof?
line = file.gets
next unless line.present?
CSV.parse(line, headers:).map do |row|
survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array,
survey_items: all_survey_items, schools:))
values = process_row(row: SurveyItemValues.new(row:, headers: headers_array,
survey_items: all_survey_items, schools:))
survey_item_responses << values if values.present?
end
row_count += 1
next unless row_count == 500
# row_count += 1
# next unless row_count == 500
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all
survey_item_responses = []
row_count = 0
# SurveyItemResponse.upsert_all(survey_item_responses, unique_by: :response_id)
# survey_item_responses = []
# row_count = 0
end
SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all
survey_item_responses = survey_item_responses.flatten.compact
SurveyItemResponse.upsert_all(survey_item_responses,
unique_by: %i[response_id dashboard_academic_year_id dashboard_school_id
dashboard_survey_item_id])
# SurveyItemResponse.upsert_all(survey_item_responses, update_only: [:likert_score])
end
private
@ -81,9 +85,9 @@ module Dashboard
def process_survey_items(row:)
student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid)
student.races.delete_all
tmp_races = row.races.map { |race| races[race] }
student.races += tmp_races
# student.races.delete_all
# tmp_races = row.races.map { |race| races[race] }
# student.races += tmp_races
row.survey_items.map do |survey_item|
likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next
@ -93,31 +97,29 @@ module Dashboard
next
end
response = row.survey_item_response(survey_item:)
create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, student:)
build_response(survey_item_response: response, likert_score:, row:, survey_item:, student:)
end.compact
end
def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, student:)
def build_response(survey_item_response:, likert_score:, row:, survey_item:, student:)
gender = genders[row.gender]
grade = row.grade
income = incomes[row.income.parameterize]
ell = ells[row.ell]
sped = speds[row.sped]
recorded_date = row.recorded_date
if survey_item_response.present?
survey_item_response.likert_score = likert_score
survey_item_response.grade = grade
survey_item_response.gender = gender
survey_item_response.recorded_date = row.recorded_date
survey_item_response.income = income
survey_item_response.ell = ell
survey_item_response.sped = sped
survey_item_response.student = student
survey_item_response
else
SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:,
likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:, student:)
end
{ response_id: row.response_id,
dashboard_academic_year_id: row.academic_year.id,
dashboard_school_id: row.school.id,
dashboard_survey_item_id: survey_item.id,
likert_score: likert_score.to_i,
grade:,
dashboard_gender_id: gender.id,
recorded_date:,
dashboard_income_id: income.id,
dashboard_ell_id: ell.id,
dashboard_sped_id: sped.id }
end
def survey_items(headers:)
@ -130,12 +132,4 @@ module Dashboard
.filter { |header| header.start_with? "t-", "s-" }
end
end
module StringMonkeyPatches
def valid_likert_score?
to_i.between? 1, 5
end
end
String.include StringMonkeyPatches
end

2
app/views/dashboard/overview/_quality_framework_indicators.erb
View file

@ -6,7 +6,7 @@
<div class="text-center">
<h3 class="sub-header-3">
<%= link_to [@district, @school, category_presenter, { year: @academic_year.range }] do %>
<%= link_to district_school_category_path(@district, @school, category_presenter, {year: @academic_year.range}) do %>
<%= category_presenter.name %>
<% end %>
</h3>