chore: load survey item responses. Show overview page without server errors

app/services/dashboard/sftp/directory.rb

@@ -1,29 +1,31 @@
-require 'net/sftp'
+require "net/sftp"
-require 'uri'
+require "uri"
-require 'csv'
+require "csv"
 
-module Sftp
+module Dashboard
-  class Directory
+  module Sftp
-    def self.open(path: '/data/survey_responses/clean', &block)
+    class Directory
-      sftptogo_url = ENV['MCIEA_SFTPTOGO_URL']
+      def self.open(path: "/data/survey_responses/clean", &block)
-      uri = URI.parse(sftptogo_url)
+        sftptogo_url = ENV["ECP_SFTPTOGO_URL"]
-      Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
+        uri = URI.parse(sftptogo_url)
-        sftp.dir.foreach(path) do |entry|
+        Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
-          next unless entry.file?
+          sftp.dir.foreach(path) do |entry|
+            next unless entry.file?
 
-          filename = entry.name
+            filename = entry.name
-          puts filename
+            puts filename
 
-          sftp.file.open(filepath(path:, filename:), 'r', &block)
+            sftp.file.open(filepath(path:, filename:), "r", &block)
+          end
         end
       end
-    end
 
-    def self.filepath(path:, filename:)
+      def self.filepath(path:, filename:)
-      path += '/' unless path.end_with?('/')
+        path += "/" unless path.end_with?("/")
-      "#{path}#{filename}"
+        "#{path}#{filename}"
-    end
+      end
 
-    private_class_method :filepath
+      private_class_method :filepath
+    end
   end
 end

app/services/dashboard/sftp/race_loader.rb

@@ -1,33 +1,35 @@
-require 'net/sftp'
+require "net/sftp"
-require 'uri'
+require "uri"
-require 'csv'
+require "csv"
 
-module Sftp
+module Dashboard
-  class RaceLoader
+  module Sftp
-    def self.load_data(path: '/data/survey_responses/')
+    class RaceLoader
-      SurveyItemResponse.update_all(student_id: nil)
+      def self.load_data(path: "/data/survey_responses/")
-      StudentRace.delete_all
+        SurveyItemResponse.update_all(student_id: nil)
-      Student.delete_all
+        StudentRace.delete_all
+        Student.delete_all
 
-      sftptogo_url = ENV['SFTPTOGO_URL']
+        sftptogo_url = ENV["SFTPTOGO_URL"]
-      uri = URI.parse(sftptogo_url)
+        uri = URI.parse(sftptogo_url)
-      Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
+        Net::SFTP.start(uri.host, uri.user, password: uri.password) do |sftp|
-        sftp.dir.foreach(path) do |entry|
+          sftp.dir.foreach(path) do |entry|
-          filename = entry.name
+            filename = entry.name
-          puts filename
+            puts filename
 
-          sftp.file.open(filepath(path:, filename:), 'r') do |f|
+            sftp.file.open(filepath(path:, filename:), "r") do |f|
-            StudentLoader.from_file(file: f, rules: [Rule::SkipNonLowellSchools])
+              StudentLoader.from_file(file: f, rules: [Rule::SkipNonLowellSchools])
+            end
           end
         end
       end
-    end
 
-    def self.filepath(path:, filename:)
+      def self.filepath(path:, filename:)
-      path += '/' unless path.end_with?('/')
+        path += "/" unless path.end_with?("/")
-      "#{path}#{filename}"
+        "#{path}#{filename}"
-    end
+      end
 
-    private_class_method :filepath
+      private_class_method :filepath
+    end
   end
 end

app/services/dashboard/survey_responses_data_loader.rb

@@ -3,6 +3,7 @@
 module Dashboard
   class SurveyResponsesDataLoader
     def load_data(filepath:)
+      byebug
       File.open(filepath) do |file|
         headers = file.first
         headers_array = CSV.parse(headers).first
@@ -13,8 +14,7 @@ module Dashboard
             process_row(row: SurveyItemValues.new(row:, headers: headers_array, survey_items: all_survey_items,
                                                   schools:))
           end
-          SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500,
+          SurveyItemResponse.upsert_all(survey_item_responses, unique_by: :response_id)
-                                                                           on_duplicate_key_update: :all
         end
       end
     end
@@ -25,25 +25,29 @@ module Dashboard
       all_survey_items = survey_items(headers:)
 
       survey_item_responses = []
-      row_count = 0
+      # row_count = 0
       until file.eof?
         line = file.gets
         next unless line.present?
 
         CSV.parse(line, headers:).map do |row|
-          survey_item_responses << process_row(row: SurveyItemValues.new(row:, headers: headers_array,
+          values = process_row(row: SurveyItemValues.new(row:, headers: headers_array,
-                                                                         survey_items: all_survey_items, schools:))
+                                                         survey_items: all_survey_items, schools:))
+          survey_item_responses << values if values.present?
         end
 
-        row_count += 1
+        # row_count += 1
-        next unless row_count == 500
+        # next unless row_count == 500
 
-        SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all
+        # SurveyItemResponse.upsert_all(survey_item_responses, unique_by: :response_id)
-        survey_item_responses = []
+        # survey_item_responses = []
-        row_count = 0
+        # row_count = 0
       end
-
+      survey_item_responses = survey_item_responses.flatten.compact
-      SurveyItemResponse.import survey_item_responses.compact.flatten, batch_size: 500, on_duplicate_key_update: :all
+      SurveyItemResponse.upsert_all(survey_item_responses,
+                                    unique_by: %i[response_id dashboard_academic_year_id dashboard_school_id
+                                                  dashboard_survey_item_id])
+      # SurveyItemResponse.upsert_all(survey_item_responses, update_only: [:likert_score])
     end
 
     private
@@ -81,9 +85,9 @@ module Dashboard
 
     def process_survey_items(row:)
       student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid)
-      student.races.delete_all
+      # student.races.delete_all
-      tmp_races = row.races.map { |race| races[race] }
+      # tmp_races = row.races.map { |race| races[race] }
-      student.races += tmp_races
+      # student.races += tmp_races
 
       row.survey_items.map do |survey_item|
         likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next
@@ -93,31 +97,29 @@ module Dashboard
           next
         end
         response = row.survey_item_response(survey_item:)
-        create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, student:)
+        build_response(survey_item_response: response, likert_score:, row:, survey_item:, student:)
       end.compact
     end
 
-    def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, student:)
+    def build_response(survey_item_response:, likert_score:, row:, survey_item:, student:)
       gender = genders[row.gender]
       grade = row.grade
       income = incomes[row.income.parameterize]
       ell = ells[row.ell]
       sped = speds[row.sped]
+      recorded_date = row.recorded_date
 
-      if survey_item_response.present?
+      { response_id: row.response_id,
-        survey_item_response.likert_score = likert_score
+        dashboard_academic_year_id: row.academic_year.id,
-        survey_item_response.grade = grade
+        dashboard_school_id: row.school.id,
-        survey_item_response.gender = gender
+        dashboard_survey_item_id: survey_item.id,
-        survey_item_response.recorded_date = row.recorded_date
+        likert_score: likert_score.to_i,
-        survey_item_response.income = income
+        grade:,
-        survey_item_response.ell = ell
+        dashboard_gender_id: gender.id,
-        survey_item_response.sped = sped
+        recorded_date:,
-        survey_item_response.student = student
+        dashboard_income_id: income.id,
-        survey_item_response
+        dashboard_ell_id: ell.id,
-      else
+        dashboard_sped_id: sped.id }
-        SurveyItemResponse.new(response_id: row.response_id, academic_year: row.academic_year, school: row.school, survey_item:,
-                               likert_score:, grade:, gender:, recorded_date: row.recorded_date, income:, ell:, sped:, student:)
-      end
     end
 
     def survey_items(headers:)
@@ -130,12 +132,4 @@ module Dashboard
          .filter { |header| header.start_with? "t-", "s-" }
     end
   end
-
-  module StringMonkeyPatches
-    def valid_likert_score?
-      to_i.between? 1, 5
-    end
-  end
-
-  String.include StringMonkeyPatches
 end

app/views/dashboard/overview/_quality_framework_indicators.erb

@@ -6,7 +6,7 @@
     <div class="text-center">
       <h3 class="sub-header-3">
 
-        <%= link_to [@district, @school, category_presenter, { year: @academic_year.range }] do %>
+          <%= link_to district_school_category_path(@district, @school, category_presenter, {year: @academic_year.range}) do %>
           <%= category_presenter.name %>
         <% end %>
       </h3>

db/migrate/20240104192128_create_dashboard_survey_item_responses.rb

@@ -16,5 +16,8 @@ class CreateDashboardSurveyItemResponses < ActiveRecord::Migration[7.1]
 
       t.timestamps
     end
+    add_index :dashboard_survey_item_responses, %i[dashboard_school_id dashboard_academic_year_id]
+    add_index :dashboard_survey_item_responses,
+              %i[response_id dashboard_school_id dashboard_academic_year_id dashboard_survey_item_id], unique: true
   end
 end

lib/tasks/dashboard_tasks.rake

@@ -1,14 +1,16 @@
+include Dashboard
+
 namespace :dashboard do
   namespace :data do
     desc "load survey responses"
     task load_survey_responses: :environment do
-      survey_item_response_count = SurveyItemResponse.count
+      survey_item_response_count = Dashboard::SurveyItemResponse.count
       student_count = Student.count
       path = "/data/survey_responses/clean/"
-      Sftp::Directory.open(path:) do |file|
+      ::Sftp::Directory.open(path:) do |file|
-        SurveyResponsesDataLoader.new.from_file(file:)
+        Dashboard::SurveyResponsesDataLoader.new.from_file(file:)
       end
-      puts "=====================> Completed loading #{SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
+      puts "=====================> Completed loading #{Dashboard::SurveyItemResponse.count - survey_item_response_count} survey responses. #{SurveyItemResponse.count} total responses in the database"
 
       Rails.cache.clear
     end

spec/dummy/config/credentials.yml.enc

@@ -0,0 +1 @@
+w0UoY2qMIqo1VCeX8KqsreT2AVVYCfzD4Eh0NVjDnrC+MTHRCto6QnYzzI7R43hle5WeChXFC3MNisW1LYQTmqnfGV6YPk1bYkyzrRo1DA0ObbizYiGnHFPIuTl0PUBxM4dM/dbrHIVa6dJPlupfhsMFKAhITtLZd/cKg6vQsQk32Hdpis4sOSkUqTBC0DcIS4oCjqBW7KOTPft1DyDPFobz9laKoVgWW9sLXwJ3ETQ5VpFjIovTp4/p11GB1CZCbcyw0JnKIzr45RUd//ZIuxAenOcodhABNqmfPMR9Uqyi+keYFc99oA+i7WjOiBd9ZpEtfYdO/voso7A9oxO/BTwOnt+KMCVlPNF6e7G1oWJQhsR8ZWWOYrYGeuIbSaOuNesAgGk6hupSjTyo2h0hGLEF9BznOf5EP86HrnB3L5041wbtyNbB2ers6F/z6xozKi9c2ULdQl93QUoJ/7BB/Lz0G1p+oedp5IIdLmHjS+oGquEf/OdkZdKIqiGFOEAGwTPc98ZJWncI3jW5UYxLJDcoOz9HIiaJX6b+yHKrjv0p79t1f1v3er6J6Hk8TafdO7+sjMcEMZOY5NnAmQumB1s3DXDxYU0588wrk2C2f+G0COTrevznFO8emQHpPuct6nHMuALfNegDCwAPYi4=--bpaVQ6fuv5gN3jZU--8DcQfciEdVpnAQjmtCvQ3g==

spec/dummy/config/master.key

@@ -0,0 +1 @@
+509ee7d28bf17b106698d4efc62fa139

spec/dummy/db/schema.rb

@@ -225,10 +225,12 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_04_192128) do
     t.index ["dashboard_ell_id"], name: "index_dashboard_survey_item_responses_on_dashboard_ell_id"
     t.index ["dashboard_gender_id"], name: "index_dashboard_survey_item_responses_on_dashboard_gender_id"
     t.index ["dashboard_income_id"], name: "index_dashboard_survey_item_responses_on_dashboard_income_id"
+    t.index ["dashboard_school_id", "dashboard_academic_year_id"], name: "idx_on_dashboard_school_id_dashboard_academic_year__44af844634"
     t.index ["dashboard_school_id"], name: "index_dashboard_survey_item_responses_on_dashboard_school_id"
     t.index ["dashboard_sped_id"], name: "index_dashboard_survey_item_responses_on_dashboard_sped_id"
     t.index ["dashboard_student_id"], name: "index_dashboard_survey_item_responses_on_dashboard_student_id"
     t.index ["dashboard_survey_item_id"], name: "idx_on_dashboard_survey_item_id_3f6652fbc6"
+    t.index ["response_id", "dashboard_school_id", "dashboard_academic_year_id", "dashboard_survey_item_id"], name: "idx_on_response_id_dashboard_school_id_dashboard_ac_5b0b3359c0", unique: true
   end
 
   create_table "dashboard_survey_items", force: :cascade do |t|