From 18b479b8b283682ce78599c065814b30790808e7 Mon Sep 17 00:00:00 2001 From: rebuilt Date: Fri, 8 Jul 2022 19:07:23 -0700 Subject: [PATCH] Refactor based on rubocop and reek suggestions --- app/controllers/analyze_controller.rb | 13 + app/models/measure.rb | 109 +- app/models/score.rb | 1 + app/models/subcategory.rb | 9 +- app/presenters/analyze_bar_presenter.rb | 8 +- app/services/survey_responses_data_loader.rb | 65 +- lib/tasks/data.rake | 1204 +++++++++--------- 7 files changed, 732 insertions(+), 677 deletions(-) diff --git a/app/controllers/analyze_controller.rb b/app/controllers/analyze_controller.rb index 8bc4be52..2893a566 100644 --- a/app/controllers/analyze_controller.rb +++ b/app/controllers/analyze_controller.rb @@ -1,15 +1,28 @@ class AnalyzeController < SqmApplicationController def index + assign_categories + assign_subcategories + assign_measures + assign_academic_years + end + + def assign_categories @category ||= Category.find_by_category_id(params[:category]) @category ||= Category.order(:category_id).first @categories = Category.all.order(:category_id) + end + def assign_subcategories @subcategories = @category.subcategories.order(:subcategory_id) @subcategory ||= Subcategory.find_by_subcategory_id(params[:subcategory]) @subcategory ||= @subcategories.first + end + def assign_measures @measures = @subcategory.measures.order(:measure_id).includes(%i[scales admin_data_items subcategory]) + end + def assign_academic_years @available_academic_years = AcademicYear.order(:range).all @academic_year_params = params[:academic_years].split(',') if params[:academic_years] @selected_academic_years = [] diff --git a/app/models/measure.rb b/app/models/measure.rb index 60d79470..fb3f9e09 100644 --- a/app/models/measure.rb +++ b/app/models/measure.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class Measure < ActiveRecord::Base belongs_to :subcategory, counter_cache: true has_one :category, through: :subcategory @@ -24,9 +26,9 @@ class Measure < ActiveRecord::Base def student_survey_items_by_survey_type(school:, academic_year:) survey = Survey.where(school:, academic_year:).first - return survey_items.student_survey_items.short_form_items if survey.form == 'short' + return student_survey_items.short_form_items if survey.form == 'short' - survey_items.student_survey_items + student_survey_items end def teacher_scales @@ -61,31 +63,14 @@ class Measure < ActiveRecord::Base def score(school:, academic_year:) @score ||= Hash.new do |memo, (school, academic_year)| - meets_student_threshold = sufficient_student_data?(school:, academic_year:) - meets_teacher_threshold = sufficient_teacher_data?(school:, academic_year:) - meets_admin_data_threshold = any_admin_data_collected?(school:, academic_year:) - lacks_sufficient_survey_data = !meets_student_threshold && !meets_teacher_threshold - incalculable_score = lacks_sufficient_survey_data && !includes_admin_data_items? - - next Score.new(nil, false, false, false) if incalculable_score - - scores = [] - if meets_teacher_threshold - scores << collect_survey_item_average(survey_items: teacher_survey_items, school:, - academic_year:) - end - if meets_student_threshold - scores << collect_survey_item_average(survey_items: student_survey_items_by_survey_type(school:, academic_year:), school:, - academic_year:) - end - scores << collect_admin_scale_average(admin_data_items:, school:, academic_year:) if includes_admin_data_items? + next Score::NIL_SCORE if incalculable_score(school:, academic_year:) + scores = collect_averages_for_teacher_student_and_admin_data(school:, academic_year:) average = scores.flatten.compact.remove_blanks.average - next Score.new(nil, false, false, false) if average.nan? + next Score::NIL_SCORE if average.nan? - memo[[school, academic_year]] = - Score.new(average, meets_teacher_threshold, meets_student_threshold, meets_admin_data_threshold) + memo[[school, academic_year]] = scorify(average:, school:, academic_year:) end @score[[school, academic_year]] @@ -94,10 +79,7 @@ class Measure < ActiveRecord::Base def student_score(school:, academic_year:) @student_score ||= Hash.new do |memo, (school, academic_year)| meets_student_threshold = sufficient_student_data?(school:, academic_year:) - if meets_student_threshold - average = collect_survey_item_average(survey_items: student_survey_items_by_survey_type(school:, academic_year:), school:, - academic_year:) - end + average = student_average(school:, academic_year:) if meets_student_threshold memo[[school, academic_year]] = scorify(average:, school:, academic_year:) end @@ -107,10 +89,7 @@ class Measure < ActiveRecord::Base def teacher_score(school:, academic_year:) @teacher_score ||= Hash.new do |memo, (school, academic_year)| meets_teacher_threshold = sufficient_teacher_data?(school:, academic_year:) - if meets_teacher_threshold - average = collect_survey_item_average(survey_items: teacher_survey_items, school:, - academic_year:) - end + average = teacher_average(school:, academic_year:) if meets_teacher_threshold memo[[school, academic_year]] = scorify(average:, school:, academic_year:) end @@ -141,12 +120,11 @@ class Measure < ActiveRecord::Base def any_admin_data_collected?(school:, academic_year:) @any_admin_data_collected ||= Hash.new do |memo, (school, academic_year)| - total_collected_admin_data_items = scales.map do |scale| - scale.admin_data_items.map do |admin_data_item| + total_collected_admin_data_items = + admin_data_items.map do |admin_data_item| admin_data_item.admin_data_values.where(school:, academic_year:).count - end - end.flatten.sum - memo[[school, academic_year]] = total_collected_admin_data_items > 0 + end.flatten.sum + memo[[school, academic_year]] = total_collected_admin_data_items.positive? end @any_admin_data_collected[[school, academic_year]] end @@ -178,9 +156,11 @@ class Measure < ActiveRecord::Base def collect_admin_scale_average(admin_data_items:, school:, academic_year:) @collect_admin_scale_average ||= Hash.new do |memo, (admin_data_items, school, academic_year)| - memo[[admin_data_items, school, academic_year]] = admin_data_items.map do |admin_data_item| - admin_value = admin_data_item.admin_data_values.where(school:, academic_year:).first - admin_value.likert_score if admin_value.present? + memo[[admin_data_items, school, academic_year]] = begin + admin_values = AdminDataValue.where(school:, academic_year:) + admin_values.map do |admin_value| + admin_value.likert_score if admin_value.present? + end end end @collect_admin_scale_average[[admin_data_items, school, academic_year]] @@ -204,19 +184,58 @@ class Measure < ActiveRecord::Base def sufficient_student_data?(school:, academic_year:) return @sufficient_student_data ||= false unless includes_student_survey_items? - return @sufficient_student_data ||= false if student_survey_items_by_survey_type(school:, academic_year:).all? do |survey_item| - survey_item.survey_item_responses.where(school:, academic_year:).none? - end + return @sufficient_student_data ||= false if no_student_responses_exist?(school:, academic_year:) @sufficient_student_data ||= subcategory.response_rate(school:, academic_year:).meets_student_threshold? end def sufficient_teacher_data?(school:, academic_year:) return @sufficient_teacher_data ||= false unless includes_teacher_survey_items? - return @sufficient_teacher_data ||= false if teacher_survey_items.all? do |survey_item| - survey_item.survey_item_responses.where(school:, academic_year:).none? - end + return @sufficient_teacher_data ||= false if no_teacher_responses_exist?(school:, academic_year:) @sufficient_teacher_data ||= subcategory.response_rate(school:, academic_year:).meets_teacher_threshold? end + + def no_student_responses_exist?(school:, academic_year:) + student_survey_items_by_survey_type(school:, + academic_year:).all? do |survey_item| + survey_item.survey_item_responses.where(school:, + academic_year:).none? + end + end + + def no_teacher_responses_exist?(school:, academic_year:) + teacher_survey_items.all? do |survey_item| + survey_item.survey_item_responses.where(school:, + academic_year:).none? + end + end + + def incalculable_score(school:, academic_year:) + meets_student_threshold = sufficient_student_data?(school:, academic_year:) + meets_teacher_threshold = sufficient_teacher_data?(school:, academic_year:) + lacks_sufficient_survey_data = !meets_student_threshold && !meets_teacher_threshold + lacks_sufficient_survey_data && !includes_admin_data_items? + end + + def collect_averages_for_teacher_student_and_admin_data(school:, academic_year:) + scores = [] + scores << teacher_average(school:, academic_year:) if sufficient_teacher_data?(school:, academic_year:) + scores << student_average(school:, academic_year:) if sufficient_student_data?(school:, academic_year:) + scores << admin_data_averages(school:, academic_year:) if includes_admin_data_items? + scores + end + + def teacher_average(school:, academic_year:) + collect_survey_item_average(survey_items: teacher_survey_items, school:, academic_year:) + end + + def student_average(school:, academic_year:) + collect_survey_item_average(survey_items: student_survey_items_by_survey_type(school:, academic_year:), school:, + academic_year:) + end + + def admin_data_averages(school:, academic_year:) + collect_admin_scale_average(admin_data_items:, school:, academic_year:) + end end diff --git a/app/models/score.rb b/app/models/score.rb index dd6830a9..c53012e4 100644 --- a/app/models/score.rb +++ b/app/models/score.rb @@ -1,2 +1,3 @@ class Score < Struct.new(:average, :meets_teacher_threshold?, :meets_student_threshold?, :meets_admin_data_threshold?) + NIL_SCORE = Score.new(nil, false, false, false) end diff --git a/app/models/subcategory.rb b/app/models/subcategory.rb index 2a670def..17250b96 100644 --- a/app/models/subcategory.rb +++ b/app/models/subcategory.rb @@ -17,7 +17,7 @@ class Subcategory < ActiveRecord::Base end if @response_rate[[school, academic_year]].nil? - @response_rate[[school, academic_year]] = create_response_rate(subcategory: self, school:, academic_year:) + @response_rate[[school, academic_year]] = create_response_rate(school:, academic_year:) end @response_rate[[school, academic_year]] @@ -25,10 +25,11 @@ class Subcategory < ActiveRecord::Base private - def create_response_rate(subcategory:, school:, academic_year:) + def create_response_rate(school:, academic_year:) student = StudentResponseRateCalculator.new(subcategory: self, school:, academic_year:) teacher = TeacherResponseRateCalculator.new(subcategory: self, school:, academic_year:) - ResponseRate.create(school:, academic_year:, subcategory: self, student_response_rate: student.rate, teacher_response_rate: teacher.rate, - meets_student_threshold: student.meets_student_threshold?, meets_teacher_threshold: teacher.meets_teacher_threshold?) + ResponseRate.create(school:, academic_year:, subcategory: self, student_response_rate: student.rate, + teacher_response_rate: teacher.rate, meets_student_threshold: student.meets_student_threshold?, + meets_teacher_threshold: teacher.meets_teacher_threshold?) end end diff --git a/app/presenters/analyze_bar_presenter.rb b/app/presenters/analyze_bar_presenter.rb index 99d05314..0cb03204 100644 --- a/app/presenters/analyze_bar_presenter.rb +++ b/app/presenters/analyze_bar_presenter.rb @@ -41,7 +41,7 @@ class AnalyzeBarPresenter else 0.0 end - bar_height < MINIMUM_BAR_HEIGHT ? MINIMUM_BAR_HEIGHT : bar_height + enforce_minimum_height(bar_height:) end def percentage @@ -63,4 +63,10 @@ class AnalyzeBarPresenter score.average.round(6) end + + private + + def enforce_minimum_height(bar_height:) + bar_height < MINIMUM_BAR_HEIGHT ? MINIMUM_BAR_HEIGHT : bar_height + end end diff --git a/app/services/survey_responses_data_loader.rb b/app/services/survey_responses_data_loader.rb index 8f39c125..4639bc47 100644 --- a/app/services/survey_responses_data_loader.rb +++ b/app/services/survey_responses_data_loader.rb @@ -4,11 +4,7 @@ class SurveyResponsesDataLoader def self.load_data(filepath:) File.open(filepath) do |file| headers = file.first - - survey_item_ids = CSV.parse(headers, headers: true).headers - .filter { |header| header.present? } - .filter { |header| header.start_with? 't-' or header.start_with? 's-' } - survey_items = SurveyItem.where(survey_item_id: survey_item_ids) + survey_items = SurveyItem.where(survey_item_id: get_survey_item_ids_from_headers(file:, headers:)) file.lazy.each_slice(1000) do |lines| survey_item_responses = CSV.parse(lines.join, headers:).map do |row| @@ -23,17 +19,16 @@ class SurveyResponsesDataLoader private def self.process_row(row:, survey_items:) - response_date = Date.parse(row['Recorded Date'] || row['RecordedDate']) - academic_year = AcademicYear.find_by_date response_date - - response_id = row['Response ID'] || row['ResponseId'] || row['ResponseID'] + return unless dese_id?(row['DESE ID']) - dese_id = row['DESE ID'] - return if dese_id.nil? + school = School.find_by_dese_id(row['DESE ID']) + return unless school.present? - school = School.find_by_dese_id(dese_id) - return if school.nil? + process_survey_items(row:, survey_items:, school:) + end + def self.process_survey_items(row:, survey_items:, school:) + response_id = row['Response ID'] || row['ResponseId'] || row['ResponseID'] survey_items.map do |survey_item| likert_score = row[survey_item.survey_item_id] next if likert_score.nil? @@ -44,22 +39,42 @@ class SurveyResponsesDataLoader end survey_item_response = SurveyItemResponse.where(response_id:, survey_item:).first - if survey_item_response.present? - survey_item_response.update!(likert_score:) if survey_item_response.likert_score != likert_score - next - else - SurveyItemResponse.new( - response_id:, - academic_year:, - school:, - survey_item:, - likert_score: - ) - end + create_or_update_survey_item_response(survey_item_response:, likert_score:, school:, response_id:, row:, + survey_item:) end.compact end + def self.create_or_update_survey_item_response(survey_item_response:, likert_score:, school:, row:, survey_item:, response_id:) + if survey_item_response.present? + survey_item_response.update!(likert_score:) if survey_item_response.likert_score != likert_score + [] + else + SurveyItemResponse.new(response_id:, academic_year: academic_year(row), school:, survey_item:, likert_score:) + end + end + + def self.get_survey_item_ids_from_headers(file:, headers:) + CSV.parse(headers, headers: true).headers + .filter { |header| header.present? } + .filter { |header| header.start_with? 't-' or header.start_with? 's-' } + end + + def self.dese_id?(dese_id) + dese_id.present? + end + + def self.response_date(row) + Date.parse(row['Recorded Date'] || row['RecordedDate']) + end + + def self.academic_year(row) + AcademicYear.find_by_date response_date(row) + end + private_class_method :process_row + private_class_method :process_survey_items + private_class_method :get_survey_item_ids_from_headers + private_class_method :dese_id? end module StringMonkeyPatches diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake index 9fb2d0cb..d56e4d7b 100644 --- a/lib/tasks/data.rake +++ b/lib/tasks/data.rake @@ -33,7 +33,7 @@ require 'csv' namespace :data do - @year = 2019 + # @year = 2019 desc 'load survey responses' task load_survey_responses: :environment do @@ -90,606 +90,606 @@ namespace :data do end end - desc 'Load in all data' - task load: :environment do - # return if School.count > 0 - Rake::Task['data:load_categories'].invoke - Rake::Task['data:load_questions'].invoke - Rake::Task['db:seed'].invoke - Rake::Task['data:load_responses'].invoke - Rake::Task['data:load_nonlikert_values'].invoke - end - - desc 'Check question / category data against existing data' - task check_questions: :environment do - csv_string = File.read(File.expand_path("../../../data/MeasureKey#{@year}.csv", __FILE__)) - csv = CSV.parse(csv_string, headers: true) - - t = Time.new - csv.each_with_index do |question, _index| - existing_question = Question.created_in(@year).find_by_external_id(question['qid']) - if existing_question.blank? - puts "NOT FOUND: #{question['qid']} -> #{question['Question Text']}" - else - puts "MULTIPLE FOUND: #{question['qid']}" if Question.where(external_id: question['qid']).count > 1 - - question_text = question['Question Text'].gsub(/[[:space:]]/, ' ').strip - if existing_question.text != question_text - puts "CHANGED TEXT: #{question['qid']} -> #{question_text} != #{existing_question.text}" - end - - 5.times do |j| - i = j + 1 - if existing_question.send("option#{i}") != question["R#{i}"] - if question["R#{i}"].blank? - puts "MISSING #{i}: #{question.inspect}" - else - puts "CHANGED OPTION #{i}: #{question['qid']} -> #{question["R#{i}"]} != #{existing_question.send("option#{i}")}" - end - end - end - end - end - end - - desc 'Sync questions / category data against existing data' - task sync_questions: :environment do - csv_string = File.read(File.expand_path("../../../data/MeasureKey#{@year}.csv", __FILE__)) - csv = CSV.parse(csv_string, headers: true) - - t = Time.new - csv.each_with_index do |question, _index| - existing_question = Question.created_in(@year).find_by_external_id(question['qid']) - if existing_question.blank? - categories = Category.where(name: question['Category Name'].titleize) - if categories.blank? - puts "Category not found for new question: #{question['Category Name'].titleize} - #{question['qid']}" - elsif categories.count > 1 - puts "Multiple categories found for new question: #{question['Category Name']} - #{question['qid']}" - else - puts "CREATING NEW QUESTION: #{categories.first.name} - #{question['qid']} - #{question['Question Text']}" - categories.first.questions.create( - external_id: question['qid'], - text: question['Question Text'], - option1: question['R1'], - option2: question['R2'], - option3: question['R3'], - option4: question['R4'], - option5: question['R5'], - target_group: question['qid'].starts_with?('s') ? 'for_students' : 'for_teachers', - reverse: question['Reverse'] == '1' - ) - end - else - if Question.created_in(@year).where(external_id: question['qid']).count > 1 - puts "MULTIPLE FOUND: #{question['qid']}" - end - - question_text = question['Question Text'].gsub(/[[:space:]]/, ' ').strip - if existing_question.text != question_text - existing_question.update(text: question_text) - puts "UPDATING TEXT: #{question['qid']} -> #{question_text} != #{existing_question.text}" - end - - 5.times do |j| - i = j + 1 - if existing_question.send("option#{i}") != question["R#{i}"] - if question["R#{i}"].blank? - puts "MISSING #{i}: #{question.inspect}" - else - existing_question.update("option#{i}": question["R#{i}"]) - puts "UPDATING OPTION #{i}: #{question['qid']} -> #{question["R#{i}"]} != #{existing_question.send("option#{i}")}" - end - end - end - end - end - end - - desc 'Load in category data' - task load_categories: :environment do - measures = JSON.parse(File.read(File.expand_path('../../data/measures.json', __dir__))) - measures.each_with_index do |measure, index| - category = Category.create_with( - blurb: measure['blurb'], - description: measure['text'], - external_id: measure['id'] || index + 1 - ).find_or_create_by(name: measure['title']) - - measure['sub'].keys.sort.each do |key| - subinfo = measure['sub'][key] - subcategory = category.child_categories.create_with( - blurb: subinfo['blurb'], - description: subinfo['text'], - external_id: key - ).find_or_create_by(name: subinfo['title']) - - subinfo['measures'].keys.sort.each do |subinfo_key| - subsubinfo = subinfo['measures'][subinfo_key] - subsubcategory = subcategory.child_categories.create_with( - blurb: subsubinfo['blurb'], - description: subsubinfo['text'], - external_id: subinfo_key - ).find_or_create_by(name: subsubinfo['title']) - - next unless subsubinfo['nonlikert'].present? - - subsubinfo['nonlikert'].each do |nonlikert_info| - puts("NONLIKERT FOUND: #{nonlikert_info['title']}") - nonlikert = subsubcategory.child_categories.create_with( - benchmark_description: nonlikert_info['benchmark_explanation'], - benchmark: nonlikert_info['benchmark'] - ).find_or_create_by(name: nonlikert_info['title']) - end - end - end - end - end - - desc 'Load in question data from json' - task load_questions: :environment do - variations = [ - '[Field-MathTeacher][Field-ScienceTeacher][Field-EnglishTeacher][Field-SocialTeacher]', - 'teacher' - ] - - questions = JSON.parse(File.read(File.expand_path('../../data/questions.json', __dir__))) - questions.each do |question| - category = nil - question['category'].split('-').each do |external_id| - categories = category.present? ? category.child_categories : Category - category = categories.where(external_id:).first - next unless category.nil? - - puts 'NOTHING' - puts external_id - puts categories.inspect - category = categories.create(name: question['Category Name'], external_id:) - end - question_text = question['text'].gsub(/[[:space:]]/, ' ').strip - if question_text.index('.* teacher').nil? - category.questions.create( - text: question_text, - option1: question['answers'][0], - option2: question['answers'][1], - option3: question['answers'][2], - option4: question['answers'][3], - option5: question['answers'][4], - for_recipient_students: question['child'].present?, - reverse: question['Reverse'] == '1' - ) - else - variations.each do |variation| - category.questions.create( - text: question_text.gsub('.* teacher', variation), - option1: question['answers'][0], - option2: question['answers'][1], - option3: question['answers'][2], - option4: question['answers'][3], - option5: question['answers'][4], - for_recipient_students: question['child'].present?, - reverse: question['Reverse'] == '1' - ) - end - end - end - end - - desc 'Load in question data from csv' - task load_questions_csv: :environment do - variations = [ - '[Field-MathTeacher][Field-ScienceTeacher][Field-EnglishTeacher][Field-SocialTeacher]', - 'teacher' - ] - - csv_string = File.read(File.expand_path("../../../data/MeasureKey#{@year}.csv", __FILE__)) - csv = CSV.parse(csv_string, headers: true) - - t = Time.new - csv.each_with_index do |question, _index| - category = nil - question['Category19'].split('-').each do |external_id_raw| - external_id = external_id_raw.gsub(/[[:space:]]/, ' ').strip - categories = category.present? ? category.child_categories : Category - category = categories.where(external_id:).first - next unless category.nil? - - puts 'NOTHING' - puts "#{question['Category']} -- #{external_id}" - puts categories.map { |c| - "#{c.name} - |#{c.external_id}| == |#{external_id}|: - #{external_id == c.external_id}" - }.join(' ---- ') - category = categories.create(name: question['Category Name'], external_id:) - end - question_text = question['Question Text'].gsub(/[[:space:]]/, ' ').strip - if question_text.index('.* teacher').nil? - category.questions.create( - text: question_text, - option1: question['R1'], - option2: question['R2'], - option3: question['R3'], - option4: question['R4'], - option5: question['R5'], - for_recipient_students: question['Level'] == 'Students', - external_id: question['qid'], - reverse: question['Reverse'] == '1' - ) - else - variations.each do |variation| - category.questions.create( - text: question_text.gsub('.* teacher', variation), - option1: question['R1'], - option2: question['R2'], - option3: question['R3'], - option4: question['R4'], - option5: question['R5'], - for_recipient_students: question['Level'] == 'Students', - external_id: question['qid'], - reverse: question['Reverse'] == '1' - ) - end - end - end - end - - desc 'Load in student and teacher responses' - task load_responses: :environment do - ENV['BULK_PROCESS'] = 'true' - answer_dictionary = { - 'Slightly': 'Somewhat', - 'an incredible': 'a tremendous', - 'a little': 'a little bit', - 'slightly': 'somewhat', - 'a little well': 'slightly well', - 'quite': 'very', - 'a tremendous': 'a very great', - 'somewhat clearly': 'somewhat', - 'almost never': 'once in a while', - 'always': 'all the time', - 'not at all strong': 'not strong at all', - 'each': 'every' - } - respondent_map = {} - - unknown_schools = {} - missing_questions = {} - bad_answers = {} - - timeToRun = 120_000 * 60 - startIndex = 0 - stopIndex = 1_000_000 - startTime = Time.new - - # ['teacher_responses'].each do |file| - %w[student_responses teacher_responses].each do |file| - recipients = file.split('_')[0] - target_group = Question.target_groups["for_#{recipients}s"] - csv_string = File.read(File.expand_path("../../../data/#{file}_#{@year}.csv", __FILE__)) - csv = CSV.parse(csv_string, headers: true) - puts("LOADING CSV: #{csv.length} ROWS") - - t = Time.new - csv.each_with_index do |row, index| - next if index < startIndex - - if Time.new - startTime >= timeToRun || index > stopIndex - puts("ENDING #{timeToRun} SECONDS: #{Time.new - startTime} = #{startIndex} -> #{index} = #{index - startIndex} or #{(Time.new - t) / (index - startIndex)} per second") - break - end - - if index % 10 == 0 - puts("DATAMSG: PROCESSING ROW: #{index} OUT OF #{csv.length} ROWS: #{Time.new - t} - Total: #{Time.new - startTime} - #{timeToRun - (Time.new - startTime)} TO GO / #{stopIndex - startIndex} ROWS TO GO") - t = Time.new - end - - district_name = row['Q111'].strip - if district_name.blank? || district_name == 'NA' - puts "DISTRICT NOT FOUND: #{district_name}" - next - end - # district_name = row['To begin, please select your district.'] if district_name.nil? - district = District.find_or_create_by(name: district_name, state_id: 1) - - school_name = row['SchoolName'].strip - - if school_name.blank? || school_name == 'NA' - puts "BLANK SCHOOL NAME: #{district.name} - #{index}" - next - end - - school = district.schools.find_or_create_by(name: school_name) - - if school.nil? - next if unknown_schools[school_name] - - puts "DATAERROR: Unable to find school: #{school_name} - #{index}" - unknown_schools[school_name] = true - next - end - - respondent_id = "#{recipients}-#{index}-#{row['ResponseId'].strip}" - recipient_id = respondent_map["#{school.id}-#{@year}-#{respondent_id}"] - recipient = school.recipients.where(id: recipient_id).first if recipient_id.present? - - if recipient.nil? - begin - recipient = school.recipients.create( - name: "Survey Respondent Id: #{respondent_id}" - ) - rescue StandardError - puts "DATAERROR: INDEX: #{index} ERROR AT #{index} - #{district.name} - #{school_name} #{school}: #{respondent_id}" - end - respondent_map["#{school.id}-#{respondent_id}"] = recipient.id - end - - recipient_list = school.recipient_lists.find_by_name("#{recipients.titleize} List") - recipient_list = school.recipient_lists.create(name: "#{recipients.titleize} List") if recipient_list.nil? - recipient_list.recipient_id_array << recipient.id - recipient_list.save! - - row.each do |key, value| - t1 = Time.new - next if value.nil? or key.nil? or value.to_s == '-99' - - key = key.gsub(/[[:space:]]/, ' ').gsub(/\./, '-').strip.gsub(/\s+/, ' ') - key = key.gsub(/-4-5/, '').gsub(/-6-12/, '') - value = value.gsub(/[[:space:]]/, ' ').strip.downcase - - begin - question = Question.created_in(@year).find_by_external_id(key) - rescue Exception => e - puts "DATAERROR: INDEX: #{index} Failed finding question: #{key} -> #{e}" - end - - if question.nil? - next if missing_questions[key] - - puts "DATAERROR: Unable to find question: #{key}" - missing_questions[key] = true - next - elsif question.unknown? - question.update_attributes(target_group:) - end - - if value.to_i.blank? - answer_index = question.option_index(value) - answer_dictionary.each do |k, v| - break if answer_index.present? - - answer_index = question.option_index(value.gsub(k.to_s, v.to_s)) - answer_index = question.option_index(value.gsub(v.to_s, k.to_s)) if answer_index.nil? - end - - if answer_index.nil? - next if bad_answers[key] - - puts "DATAERROR: Unable to find answer: #{key} = #{value.downcase.strip} - #{question.options.inspect}" - bad_answers[key] = true - next - end - else - answer_index = value.to_i - end - - next if answer_index == 0 - - # answer_index = 6 - answer_index if question.reverse? - - responded_at = begin - Date.strptime(row['recordedDate'], '%Y-%m-%d %H:%M:%S') - rescue StandardError - Date.today - end - begin - recipient.attempts.create(question:, answer_index:, responded_at:) - rescue Exception => e - puts "DATAERROR: INDEX: #{index} Attempt failed for #{recipient.inspect} -> QUESTION: #{question.inspect}, ANSWER_INDEX: #{answer_index}, RESPONDED_AT: #{responded_at}, ERROR: #{e}" - next - end - end - end - end - ENV.delete('BULK_PROCESS') - - sync_school_category_aggregates - - # Recipient.created_in(@year).each { |r| r.update_counts } - end - - desc 'Load in nonlikert values for each school' - task load_nonlikert_values: :environment do - csv_string = File.read(File.expand_path('../../data/MCIEA_18-19AdminData_Final.csv', __dir__)) - # csv_string = File.read(File.expand_path("../../../data/MCIEA_16-17_SGP.csv", __FILE__)) - csv = CSV.parse(csv_string, headers: true) - puts("LOADING NONLIKERT CSV: #{csv.length} ROWS") - - errors = [] - csv.each_with_index do |row, _index| - next if row['Likert_Value'].blank? - - base = Category - category_ids = row['Category'].split('-') - category_ids.each do |category_id| - category_id = category_id.downcase if category_id.downcase =~ /i/ - base = base.find_by_external_id(category_id) - if base.nil? - row['reason'] = "Unable to find category_id #{category_id} for category #{row['Category']}" - errors << row - next - end - base = base.child_categories - end - - nonlikert_category = base.where(name: row['NonLikert Title']).first - - if nonlikert_category.nil? - row['reason'] = "Unable to find nonlikert category: #{row['NonLikert Title']} in " - errors << row - next - elsif (benchmark = row['Benchmark']).present? - nonlikert_category.update(benchmark:) - end - - district = District.where(name: row['District'], state_id: 1).first - if district.blank? - row['reason'] = "DISTRICT NOT FOUND: #{row['District']}" - errors << row - next - end - - school = district.schools.where(name: row['School']).first - if school.blank? - row['reason'] = "SCHOOL NOT FOUND: #{row['School']} (#{district.name})" - errors << row - next - end - - school_category = school.school_categories.find_or_create_by(category: nonlikert_category, year: @year.to_s) - if school_category.blank? - row['reason'] = "SCHOOL CATEGORY NOT FOUND: #{school.name} (#{district.name}) #{nonlikert_category.name}" - errors << row - next - end - - zscore = ([-2, [row['Likert_Value'].to_f - 3, 2].min].max * 10).to_i.to_f / 10.0 - school_category.update( - nonlikert: row['NL_Value'], - zscore: zscore.to_f, - year: @year.to_s, - valid_child_count: 1 - ) - - school_category.reload.sync_aggregated_responses - end - - errors.each do |error| - puts "#{error['reason']}: #{error['NonLikert Title']} -> #{error['Likert_Value']}" - end - - puts "COUNT: #{SchoolCategory.where(attempt_count: 0, - answer_index_total: 0).where('nonlikert is not null and zscore is null').count}" - end - - desc 'Load in custom zones for each category' - task load_custom_zones: :environment do - ENV['BULK_PROCESS'] = 'true' - - csv_string = File.read(File.expand_path('../../data/Benchmarks2016-2017.csv', __dir__)) - csv = CSV.parse(csv_string, headers: true) - - csv.each_with_index do |row, _index| - next if row['Warning High'].blank? - - category = Category.find_by_name(row['Subcategory']) - - if category.nil? - puts "Unable to find category #{row['Subcategory']}" - next - end - - custom_zones = [ - row['Warning High'], - row['Watch High'], - row['Growth High'], - row['Approval High'], - 5 - ] - - puts "#{category.name} -> #{custom_zones.join(',')}" - - category.update(zones: custom_zones.join(',')) - end - - ENV.delete('BULK_PROCESS') - - Category.all.each { |category| category.sync_child_zones } - end - - desc 'Move all likert survey results to a new submeasure of current measure' - task move_likert_to_submeasures: :environment do - Question.all.each do |q| - category = q.category - next unless category.name.index('Scale').nil? - - new_category_name = "#{category.name} Scale" - new_category = category.child_categories.where(name: new_category_name).first - if new_category.nil? - new_category = category.child_categories.create( - name: new_category_name, - blurb: "This measure contains all survey responses for #{category.name}.", - description: "The following survey questions concern perceptions of #{category.name}.", - zones: category.zones - ) - end - q.update(category: new_category) - end - - # sync_school_category_aggregates - end - - desc 'Sync all school category aggregates' - task sync: :environment do - sync_school_category_aggregates - - # Recipient.created_in(@year).each { |r| r.update_counts } - end - - desc 'Create School Questions' - task create_school_questions: :environment do - Category.joins(:questions).uniq.all.each do |category| - category.school_categories.in(@year).joins(school: :district).where("districts.name = 'Boston'").find_in_batches(batch_size: 100) do |group| - group.each do |school_category| - school_questions = [] - new_school_questions = [] - category.questions.created_in(@year).each do |question| - school = school_category.school - next if school.district.name != 'Boston' - - school_question = school_category.school_questions.for(school, question).first - if school_question.present? - school_questions << school_question - else - attempt_data = Attempt - .joins(:question) - .created_in(school_category.year) - .for_question(question) - .for_school(school) - .select('count(attempts.answer_index) as response_count') - .select('sum(case when questions.reverse then 6 - attempts.answer_index else attempts.answer_index end) as answer_index_total')[0] - - available_responders = school.available_responders_for(question) - - school_question = school_category.school_questions.new( - school:, - question:, - school_category:, - year: school_category.year, - attempt_count: available_responders, - response_count: attempt_data.response_count, - response_rate: attempt_data.response_count.to_f / available_responders.to_f, - response_total: attempt_data.answer_index_total - ) - new_school_questions << school_question - school_questions << school_question - end - end - - SchoolQuestion.import new_school_questions - end - end - end - end - - def sync_school_category_aggregates - School.all.each do |school| - Category.all.each do |category| - school_category = SchoolCategory.for(school, category).in(@year).first - school_category = school.school_categories.create(category:, year: @year) if school_category.nil? - school_category.sync_aggregated_responses - end - end - end -end + # desc 'Load in all data' + # task load: :environment do + # # return if School.count > 0 + # Rake::Task['data:load_categories'].invoke + # Rake::Task['data:load_questions'].invoke + # Rake::Task['db:seed'].invoke + # Rake::Task['data:load_responses'].invoke + # Rake::Task['data:load_nonlikert_values'].invoke + # end + + # desc 'Check question / category data against existing data' + # task check_questions: :environment do + # csv_string = File.read(File.expand_path("../../../data/MeasureKey#{@year}.csv", __FILE__)) + # csv = CSV.parse(csv_string, headers: true) + + # t = Time.new + # csv.each_with_index do |question, _index| + # existing_question = Question.created_in(@year).find_by_external_id(question['qid']) + # if existing_question.blank? + # puts "NOT FOUND: #{question['qid']} -> #{question['Question Text']}" + # else + # puts "MULTIPLE FOUND: #{question['qid']}" if Question.where(external_id: question['qid']).count > 1 + + # question_text = question['Question Text'].gsub(/[[:space:]]/, ' ').strip + # if existing_question.text != question_text + # puts "CHANGED TEXT: #{question['qid']} -> #{question_text} != #{existing_question.text}" + # end + + # 5.times do |j| + # i = j + 1 + # if existing_question.send("option#{i}") != question["R#{i}"] + # if question["R#{i}"].blank? + # puts "MISSING #{i}: #{question.inspect}" + # else + # puts "CHANGED OPTION #{i}: #{question['qid']} -> #{question["R#{i}"]} != #{existing_question.send("option#{i}")}" + # end + # end + # end + # end + # end + # end + + # desc 'Sync questions / category data against existing data' + # task sync_questions: :environment do + # csv_string = File.read(File.expand_path("../../../data/MeasureKey#{@year}.csv", __FILE__)) + # csv = CSV.parse(csv_string, headers: true) + + # t = Time.new + # csv.each_with_index do |question, _index| + # existing_question = Question.created_in(@year).find_by_external_id(question['qid']) + # if existing_question.blank? + # categories = Category.where(name: question['Category Name'].titleize) + # if categories.blank? + # puts "Category not found for new question: #{question['Category Name'].titleize} - #{question['qid']}" + # elsif categories.count > 1 + # puts "Multiple categories found for new question: #{question['Category Name']} - #{question['qid']}" + # else + # puts "CREATING NEW QUESTION: #{categories.first.name} - #{question['qid']} - #{question['Question Text']}" + # categories.first.questions.create( + # external_id: question['qid'], + # text: question['Question Text'], + # option1: question['R1'], + # option2: question['R2'], + # option3: question['R3'], + # option4: question['R4'], + # option5: question['R5'], + # target_group: question['qid'].starts_with?('s') ? 'for_students' : 'for_teachers', + # reverse: question['Reverse'] == '1' + # ) + # end + # else + # if Question.created_in(@year).where(external_id: question['qid']).count > 1 + # puts "MULTIPLE FOUND: #{question['qid']}" + # end + + # question_text = question['Question Text'].gsub(/[[:space:]]/, ' ').strip + # if existing_question.text != question_text + # existing_question.update(text: question_text) + # puts "UPDATING TEXT: #{question['qid']} -> #{question_text} != #{existing_question.text}" + # end + + # 5.times do |j| + # i = j + 1 + # if existing_question.send("option#{i}") != question["R#{i}"] + # if question["R#{i}"].blank? + # puts "MISSING #{i}: #{question.inspect}" + # else + # existing_question.update("option#{i}": question["R#{i}"]) + # puts "UPDATING OPTION #{i}: #{question['qid']} -> #{question["R#{i}"]} != #{existing_question.send("option#{i}")}" + # end + # end + # end + # end + # end + # end + + # desc 'Load in category data' + # task load_categories: :environment do + # measures = JSON.parse(File.read(File.expand_path('../../data/measures.json', __dir__))) + # measures.each_with_index do |measure, index| + # category = Category.create_with( + # blurb: measure['blurb'], + # description: measure['text'], + # external_id: measure['id'] || index + 1 + # ).find_or_create_by(name: measure['title']) + + # measure['sub'].keys.sort.each do |key| + # subinfo = measure['sub'][key] + # subcategory = category.child_categories.create_with( + # blurb: subinfo['blurb'], + # description: subinfo['text'], + # external_id: key + # ).find_or_create_by(name: subinfo['title']) + + # subinfo['measures'].keys.sort.each do |subinfo_key| + # subsubinfo = subinfo['measures'][subinfo_key] + # subsubcategory = subcategory.child_categories.create_with( + # blurb: subsubinfo['blurb'], + # description: subsubinfo['text'], + # external_id: subinfo_key + # ).find_or_create_by(name: subsubinfo['title']) + + # next unless subsubinfo['nonlikert'].present? + + # subsubinfo['nonlikert'].each do |nonlikert_info| + # puts("NONLIKERT FOUND: #{nonlikert_info['title']}") + # nonlikert = subsubcategory.child_categories.create_with( + # benchmark_description: nonlikert_info['benchmark_explanation'], + # benchmark: nonlikert_info['benchmark'] + # ).find_or_create_by(name: nonlikert_info['title']) + # end + # end + # end + # end + # end + + # desc 'Load in question data from json' + # task load_questions: :environment do + # variations = [ + # '[Field-MathTeacher][Field-ScienceTeacher][Field-EnglishTeacher][Field-SocialTeacher]', + # 'teacher' + # ] + + # questions = JSON.parse(File.read(File.expand_path('../../data/questions.json', __dir__))) + # questions.each do |question| + # category = nil + # question['category'].split('-').each do |external_id| + # categories = category.present? ? category.child_categories : Category + # category = categories.where(external_id:).first + # next unless category.nil? + + # puts 'NOTHING' + # puts external_id + # puts categories.inspect + # category = categories.create(name: question['Category Name'], external_id:) + # end + # question_text = question['text'].gsub(/[[:space:]]/, ' ').strip + # if question_text.index('.* teacher').nil? + # category.questions.create( + # text: question_text, + # option1: question['answers'][0], + # option2: question['answers'][1], + # option3: question['answers'][2], + # option4: question['answers'][3], + # option5: question['answers'][4], + # for_recipient_students: question['child'].present?, + # reverse: question['Reverse'] == '1' + # ) + # else + # variations.each do |variation| + # category.questions.create( + # text: question_text.gsub('.* teacher', variation), + # option1: question['answers'][0], + # option2: question['answers'][1], + # option3: question['answers'][2], + # option4: question['answers'][3], + # option5: question['answers'][4], + # for_recipient_students: question['child'].present?, + # reverse: question['Reverse'] == '1' + # ) + # end + # end + # end + # end + + # desc 'Load in question data from csv' + # task load_questions_csv: :environment do + # variations = [ + # '[Field-MathTeacher][Field-ScienceTeacher][Field-EnglishTeacher][Field-SocialTeacher]', + # 'teacher' + # ] + + # csv_string = File.read(File.expand_path("../../../data/MeasureKey#{@year}.csv", __FILE__)) + # csv = CSV.parse(csv_string, headers: true) + + # t = Time.new + # csv.each_with_index do |question, _index| + # category = nil + # question['Category19'].split('-').each do |external_id_raw| + # external_id = external_id_raw.gsub(/[[:space:]]/, ' ').strip + # categories = category.present? ? category.child_categories : Category + # category = categories.where(external_id:).first + # next unless category.nil? + + # puts 'NOTHING' + # puts "#{question['Category']} -- #{external_id}" + # puts categories.map { |c| + # "#{c.name} - |#{c.external_id}| == |#{external_id}|: - #{external_id == c.external_id}" + # }.join(' ---- ') + # category = categories.create(name: question['Category Name'], external_id:) + # end + # question_text = question['Question Text'].gsub(/[[:space:]]/, ' ').strip + # if question_text.index('.* teacher').nil? + # category.questions.create( + # text: question_text, + # option1: question['R1'], + # option2: question['R2'], + # option3: question['R3'], + # option4: question['R4'], + # option5: question['R5'], + # for_recipient_students: question['Level'] == 'Students', + # external_id: question['qid'], + # reverse: question['Reverse'] == '1' + # ) + # else + # variations.each do |variation| + # category.questions.create( + # text: question_text.gsub('.* teacher', variation), + # option1: question['R1'], + # option2: question['R2'], + # option3: question['R3'], + # option4: question['R4'], + # option5: question['R5'], + # for_recipient_students: question['Level'] == 'Students', + # external_id: question['qid'], + # reverse: question['Reverse'] == '1' + # ) + # end + # end + # end + # end + + # desc 'Load in student and teacher responses' + # task load_responses: :environment do + # ENV['BULK_PROCESS'] = 'true' + # answer_dictionary = { + # 'Slightly': 'Somewhat', + # 'an incredible': 'a tremendous', + # 'a little': 'a little bit', + # 'slightly': 'somewhat', + # 'a little well': 'slightly well', + # 'quite': 'very', + # 'a tremendous': 'a very great', + # 'somewhat clearly': 'somewhat', + # 'almost never': 'once in a while', + # 'always': 'all the time', + # 'not at all strong': 'not strong at all', + # 'each': 'every' + # } + # respondent_map = {} + + # unknown_schools = {} + # missing_questions = {} + # bad_answers = {} + + # timeToRun = 120_000 * 60 + # startIndex = 0 + # stopIndex = 1_000_000 + # startTime = Time.new + + # # ['teacher_responses'].each do |file| + # %w[student_responses teacher_responses].each do |file| + # recipients = file.split('_')[0] + # target_group = Question.target_groups["for_#{recipients}s"] + # csv_string = File.read(File.expand_path("../../../data/#{file}_#{@year}.csv", __FILE__)) + # csv = CSV.parse(csv_string, headers: true) + # puts("LOADING CSV: #{csv.length} ROWS") + + # t = Time.new + # csv.each_with_index do |row, index| + # next if index < startIndex + + # if Time.new - startTime >= timeToRun || index > stopIndex + # puts("ENDING #{timeToRun} SECONDS: #{Time.new - startTime} = #{startIndex} -> #{index} = #{index - startIndex} or #{(Time.new - t) / (index - startIndex)} per second") + # break + # end + + # if index % 10 == 0 + # puts("DATAMSG: PROCESSING ROW: #{index} OUT OF #{csv.length} ROWS: #{Time.new - t} - Total: #{Time.new - startTime} - #{timeToRun - (Time.new - startTime)} TO GO / #{stopIndex - startIndex} ROWS TO GO") + # t = Time.new + # end + + # district_name = row['Q111'].strip + # if district_name.blank? || district_name == 'NA' + # puts "DISTRICT NOT FOUND: #{district_name}" + # next + # end + # # district_name = row['To begin, please select your district.'] if district_name.nil? + # district = District.find_or_create_by(name: district_name, state_id: 1) + + # school_name = row['SchoolName'].strip + + # if school_name.blank? || school_name == 'NA' + # puts "BLANK SCHOOL NAME: #{district.name} - #{index}" + # next + # end + + # school = district.schools.find_or_create_by(name: school_name) + + # if school.nil? + # next if unknown_schools[school_name] + + # puts "DATAERROR: Unable to find school: #{school_name} - #{index}" + # unknown_schools[school_name] = true + # next + # end + + # respondent_id = "#{recipients}-#{index}-#{row['ResponseId'].strip}" + # recipient_id = respondent_map["#{school.id}-#{@year}-#{respondent_id}"] + # recipient = school.recipients.where(id: recipient_id).first if recipient_id.present? + + # if recipient.nil? + # begin + # recipient = school.recipients.create( + # name: "Survey Respondent Id: #{respondent_id}" + # ) + # rescue StandardError + # puts "DATAERROR: INDEX: #{index} ERROR AT #{index} - #{district.name} - #{school_name} #{school}: #{respondent_id}" + # end + # respondent_map["#{school.id}-#{respondent_id}"] = recipient.id + # end + + # recipient_list = school.recipient_lists.find_by_name("#{recipients.titleize} List") + # recipient_list = school.recipient_lists.create(name: "#{recipients.titleize} List") if recipient_list.nil? + # recipient_list.recipient_id_array << recipient.id + # recipient_list.save! + + # row.each do |key, value| + # t1 = Time.new + # next if value.nil? or key.nil? or value.to_s == '-99' + + # key = key.gsub(/[[:space:]]/, ' ').gsub(/\./, '-').strip.gsub(/\s+/, ' ') + # key = key.gsub(/-4-5/, '').gsub(/-6-12/, '') + # value = value.gsub(/[[:space:]]/, ' ').strip.downcase + + # begin + # question = Question.created_in(@year).find_by_external_id(key) + # rescue Exception => e + # puts "DATAERROR: INDEX: #{index} Failed finding question: #{key} -> #{e}" + # end + + # if question.nil? + # next if missing_questions[key] + + # puts "DATAERROR: Unable to find question: #{key}" + # missing_questions[key] = true + # next + # elsif question.unknown? + # question.update_attributes(target_group:) + # end + + # if value.to_i.blank? + # answer_index = question.option_index(value) + # answer_dictionary.each do |k, v| + # break if answer_index.present? + + # answer_index = question.option_index(value.gsub(k.to_s, v.to_s)) + # answer_index = question.option_index(value.gsub(v.to_s, k.to_s)) if answer_index.nil? + # end + + # if answer_index.nil? + # next if bad_answers[key] + + # puts "DATAERROR: Unable to find answer: #{key} = #{value.downcase.strip} - #{question.options.inspect}" + # bad_answers[key] = true + # next + # end + # else + # answer_index = value.to_i + # end + + # next if answer_index == 0 + + # # answer_index = 6 - answer_index if question.reverse? + + # responded_at = begin + # Date.strptime(row['recordedDate'], '%Y-%m-%d %H:%M:%S') + # rescue StandardError + # Date.today + # end + # begin + # recipient.attempts.create(question:, answer_index:, responded_at:) + # rescue Exception => e + # puts "DATAERROR: INDEX: #{index} Attempt failed for #{recipient.inspect} -> QUESTION: #{question.inspect}, ANSWER_INDEX: #{answer_index}, RESPONDED_AT: #{responded_at}, ERROR: #{e}" + # next + # end + # end + # end + # end + # ENV.delete('BULK_PROCESS') + + # sync_school_category_aggregates + + # # Recipient.created_in(@year).each { |r| r.update_counts } + # end + + # desc 'Load in nonlikert values for each school' + # task load_nonlikert_values: :environment do + # csv_string = File.read(File.expand_path('../../data/MCIEA_18-19AdminData_Final.csv', __dir__)) + # # csv_string = File.read(File.expand_path("../../../data/MCIEA_16-17_SGP.csv", __FILE__)) + # csv = CSV.parse(csv_string, headers: true) + # puts("LOADING NONLIKERT CSV: #{csv.length} ROWS") + + # errors = [] + # csv.each_with_index do |row, _index| + # next if row['Likert_Value'].blank? + + # base = Category + # category_ids = row['Category'].split('-') + # category_ids.each do |category_id| + # category_id = category_id.downcase if category_id.downcase =~ /i/ + # base = base.find_by_external_id(category_id) + # if base.nil? + # row['reason'] = "Unable to find category_id #{category_id} for category #{row['Category']}" + # errors << row + # next + # end + # base = base.child_categories + # end + + # nonlikert_category = base.where(name: row['NonLikert Title']).first + + # if nonlikert_category.nil? + # row['reason'] = "Unable to find nonlikert category: #{row['NonLikert Title']} in " + # errors << row + # next + # elsif (benchmark = row['Benchmark']).present? + # nonlikert_category.update(benchmark:) + # end + + # district = District.where(name: row['District'], state_id: 1).first + # if district.blank? + # row['reason'] = "DISTRICT NOT FOUND: #{row['District']}" + # errors << row + # next + # end + + # school = district.schools.where(name: row['School']).first + # if school.blank? + # row['reason'] = "SCHOOL NOT FOUND: #{row['School']} (#{district.name})" + # errors << row + # next + # end + + # school_category = school.school_categories.find_or_create_by(category: nonlikert_category, year: @year.to_s) + # if school_category.blank? + # row['reason'] = "SCHOOL CATEGORY NOT FOUND: #{school.name} (#{district.name}) #{nonlikert_category.name}" + # errors << row + # next + # end + + # zscore = ([-2, [row['Likert_Value'].to_f - 3, 2].min].max * 10).to_i.to_f / 10.0 + # school_category.update( + # nonlikert: row['NL_Value'], + # zscore: zscore.to_f, + # year: @year.to_s, + # valid_child_count: 1 + # ) + + # school_category.reload.sync_aggregated_responses + # end + + # errors.each do |error| + # puts "#{error['reason']}: #{error['NonLikert Title']} -> #{error['Likert_Value']}" + # end + + # puts "COUNT: #{SchoolCategory.where(attempt_count: 0, + # answer_index_total: 0).where('nonlikert is not null and zscore is null').count}" + # end + + # desc 'Load in custom zones for each category' + # task load_custom_zones: :environment do + # ENV['BULK_PROCESS'] = 'true' + + # csv_string = File.read(File.expand_path('../../data/Benchmarks2016-2017.csv', __dir__)) + # csv = CSV.parse(csv_string, headers: true) + + # csv.each_with_index do |row, _index| + # next if row['Warning High'].blank? + + # category = Category.find_by_name(row['Subcategory']) + + # if category.nil? + # puts "Unable to find category #{row['Subcategory']}" + # next + # end + + # custom_zones = [ + # row['Warning High'], + # row['Watch High'], + # row['Growth High'], + # row['Approval High'], + # 5 + # ] + + # puts "#{category.name} -> #{custom_zones.join(',')}" + + # category.update(zones: custom_zones.join(',')) + # end + + # ENV.delete('BULK_PROCESS') + + # Category.all.each { |category| category.sync_child_zones } + # end + + # desc 'Move all likert survey results to a new submeasure of current measure' + # task move_likert_to_submeasures: :environment do + # Question.all.each do |q| + # category = q.category + # next unless category.name.index('Scale').nil? + + # new_category_name = "#{category.name} Scale" + # new_category = category.child_categories.where(name: new_category_name).first + # if new_category.nil? + # new_category = category.child_categories.create( + # name: new_category_name, + # blurb: "This measure contains all survey responses for #{category.name}.", + # description: "The following survey questions concern perceptions of #{category.name}.", + # zones: category.zones + # ) + # end + # q.update(category: new_category) + # end + + # # sync_school_category_aggregates + # end + + # desc 'Sync all school category aggregates' + # task sync: :environment do + # sync_school_category_aggregates + + # # Recipient.created_in(@year).each { |r| r.update_counts } + # end + + # desc 'Create School Questions' + # task create_school_questions: :environment do + # Category.joins(:questions).uniq.all.each do |category| + # category.school_categories.in(@year).joins(school: :district).where("districts.name = 'Boston'").find_in_batches(batch_size: 100) do |group| + # group.each do |school_category| + # school_questions = [] + # new_school_questions = [] + # category.questions.created_in(@year).each do |question| + # school = school_category.school + # next if school.district.name != 'Boston' + + # school_question = school_category.school_questions.for(school, question).first + # if school_question.present? + # school_questions << school_question + # else + # attempt_data = Attempt + # .joins(:question) + # .created_in(school_category.year) + # .for_question(question) + # .for_school(school) + # .select('count(attempts.answer_index) as response_count') + # .select('sum(case when questions.reverse then 6 - attempts.answer_index else attempts.answer_index end) as answer_index_total')[0] + + # available_responders = school.available_responders_for(question) + + # school_question = school_category.school_questions.new( + # school:, + # question:, + # school_category:, + # year: school_category.year, + # attempt_count: available_responders, + # response_count: attempt_data.response_count, + # response_rate: attempt_data.response_count.to_f / available_responders.to_f, + # response_total: attempt_data.answer_index_total + # ) + # new_school_questions << school_question + # school_questions << school_question + # end + # end + + # SchoolQuestion.import new_school_questions + # end + # end + # end + # end + + # def sync_school_category_aggregates + # School.all.each do |school| + # Category.all.each do |category| + # school_category = SchoolCategory.for(school, category).in(@year).first + # school_category = school.school_categories.create(category:, year: @year) if school_category.nil? + # school_category.sync_aggregated_responses + # end + # end + # end +# end # @@ -1012,4 +1012,4 @@ end # end # # ENV.delete('BULK_PROCESS') -# end +end