# PSQL: /Applications/Postgres.app/Contents/Versions/9.6/bin/psql -h localhost # LOAD DATA # RAILS_ENV=development rails db:environment:set db:drop db:create db:migrate; /Applications/Postgres.app/Contents/Versions/9.6/bin/pg_restore --verbose --clean --no-acl --no-owner -h localhost -d mciea_development latest.dump; rake db:migrate; # rails c -> SchoolCategory.update_all(year: '2017') # rake data:load_questions_csv; rake data:load_responses # sudo heroku pg:reset DATABASE -a mciea-beta # sudo heroku pg:backups:restore 'https://s3.amazonaws.com/irrationaldesign/beta-data-080818.dump' DATABASE_URL -a mciea-beta # sudo heroku run rake db:migrate -a mciea-beta # sudo heroku run console -a mciea-beta -> SchoolCategory.update_all(year: '2017') -- # RENAME SCHOOLS = s = SCHOOLS; s.each { |correct, incorrect| District.find_by_name("Boston").schools.find_by_name(incorrect[0]).update(name: correct) } # s.map { |correct, incorrect| District.find_by_name("Boston").schools.find_by_name(incorrect.to_s).merge_into(correct) } # sudo heroku run rake data:load_questions_csv -a mciea-beta # sudo heroku run rake data:load_questions_csv data:sync_questions -a mciea-beta # sudo heroku run:detached rake data:load_responses -a mciea-beta --size performance-l # sudo heroku run rake data:move_likert_to_submeasures -a mciea-beta # sudo heroku run:detached rake data:sync -a mciea-beta --size performance-l # sudo heroku run:detached rake data:create_school_questions -a mciea-dashboard --size performance-l # Add: # # Category: unique_external_id (string) # School Category: year (string) # # Update: # Add year to existing school categories require 'csv' namespace :data do @year = 2019 desc "Load in all data" task load: :environment do # return if School.count > 0 Rake::Task["data:load_categories"].invoke Rake::Task["data:load_questions"].invoke Rake::Task["db:seed"].invoke Rake::Task["data:load_responses"].invoke Rake::Task["data:load_nonlikert_values"].invoke end desc 'Check question / category data against existing data' task check_questions: :environment do csv_string = File.read(File.expand_path("../../../data/MeasureKey#{@year}.csv", __FILE__)) csv = CSV.parse(csv_string, :headers => true) t = Time.new csv.each_with_index do |question, index| existing_question = Question.created_in(@year).find_by_external_id(question['qid']) if existing_question.blank? puts "NOT FOUND: #{question['qid']} -> #{question["Question Text"]}" else if Question.where(external_id: question['qid']).count > 1 puts "MULTIPLE FOUND: #{question['qid']}" end question_text = question['Question Text'].gsub(/[[:space:]]/, ' ').strip if existing_question.text != question_text puts "CHANGED TEXT: #{question['qid']} -> #{question_text} != #{existing_question.text}" end 5.times do |j| i = j + 1 if existing_question.send("option#{i}") != question["R#{i}"] if question["R#{i}"].blank? puts "MISSING #{i}: #{question.inspect}" else puts "CHANGED OPTION #{i}: #{question['qid']} -> #{question["R#{i}"]} != #{existing_question.send("option#{i}")}" end end end end end end desc 'Sync questions / category data against existing data' task sync_questions: :environment do csv_string = File.read(File.expand_path("../../../data/MeasureKey#{@year}.csv", __FILE__)) csv = CSV.parse(csv_string, :headers => true) t = Time.new csv.each_with_index do |question, index| existing_question = Question.created_in(@year).find_by_external_id(question['qid']) if existing_question.blank? categories = Category.where(name: question['Category Name'].titleize) if categories.blank? puts "Category not found for new question: #{question['Category Name'].titleize} - #{question['qid']}" elsif categories.count > 1 puts "Multiple categories found for new question: #{question['Category Name']} - #{question['qid']}" else puts "CREATING NEW QUESTION: #{categories.first.name} - #{question['qid']} - #{question['Question Text']}" categories.first.questions.create( external_id: question['qid'], text: question['Question Text'], option1: question['R1'], option2: question['R2'], option3: question['R3'], option4: question['R4'], option5: question['R5'], target_group: question['qid'].starts_with?('s') ? 'for_students' : 'for_teachers', reverse: question['Reverse'] == "1" ) end else if Question.created_in(@year).where(external_id: question['qid']).count > 1 puts "MULTIPLE FOUND: #{question['qid']}" end question_text = question['Question Text'].gsub(/[[:space:]]/, ' ').strip if existing_question.text != question_text existing_question.update(text: question_text) puts "UPDATING TEXT: #{question['qid']} -> #{question_text} != #{existing_question.text}" end 5.times do |j| i = j + 1 if existing_question.send("option#{i}") != question["R#{i}"] if question["R#{i}"].blank? puts "MISSING #{i}: #{question.inspect}" else existing_question.update("option#{i}": question["R#{i}"]) puts "UPDATING OPTION #{i}: #{question['qid']} -> #{question["R#{i}"]} != #{existing_question.send("option#{i}")}" end end end end end end desc 'Load in category data' task load_categories: :environment do measures = JSON.parse(File.read(File.expand_path('../../../data/measures.json', __FILE__))) measures.each_with_index do |measure, index| category = Category.create_with( blurb: measure['blurb'], description: measure['text'], external_id: measure['id'] || index + 1 ).find_or_create_by(name: measure['title']) measure['sub'].keys.sort.each do |key| subinfo = measure['sub'][key] subcategory = category.child_categories.create_with( blurb: subinfo['blurb'], description: subinfo['text'], external_id: key ).find_or_create_by(name: subinfo['title']) subinfo['measures'].keys.sort.each do |subinfo_key| subsubinfo = subinfo['measures'][subinfo_key] subsubcategory = subcategory.child_categories.create_with( blurb: subsubinfo['blurb'], description: subsubinfo['text'], external_id: subinfo_key ).find_or_create_by(name: subsubinfo['title']) if subsubinfo['nonlikert'].present? subsubinfo['nonlikert'].each do |nonlikert_info| puts("NONLIKERT FOUND: #{nonlikert_info['title']}") nonlikert = subsubcategory.child_categories.create_with( benchmark_description: nonlikert_info['benchmark_explanation'], benchmark: nonlikert_info['benchmark'] ).find_or_create_by(name: nonlikert_info['title']) end end end end end end desc 'Load in question data from json' task load_questions: :environment do variations = [ '[Field-MathTeacher][Field-ScienceTeacher][Field-EnglishTeacher][Field-SocialTeacher]', 'teacher' ] questions = JSON.parse(File.read(File.expand_path('../../../data/questions.json', __FILE__))) questions.each do |question| category = nil question['category'].split('-').each do |external_id| categories = category.present? ? category.child_categories : Category category = categories.where(external_id: external_id).first if category.nil? puts 'NOTHING' puts external_id puts categories.inspect category = categories.create(name: question['Category Name'], external_id: external_id) end end question_text = question['text'].gsub(/[[:space:]]/, ' ').strip if question_text.index('.* teacher').nil? category.questions.create( text: question_text, option1: question['answers'][0], option2: question['answers'][1], option3: question['answers'][2], option4: question['answers'][3], option5: question['answers'][4], for_recipient_students: question['child'].present?, reverse: question['Reverse'] == "1" ) else variations.each do |variation| category.questions.create( text: question_text.gsub('.* teacher', variation), option1: question['answers'][0], option2: question['answers'][1], option3: question['answers'][2], option4: question['answers'][3], option5: question['answers'][4], for_recipient_students: question['child'].present?, reverse: question['Reverse'] == "1" ) end end end end desc 'Load in question data from csv' task load_questions_csv: :environment do variations = [ '[Field-MathTeacher][Field-ScienceTeacher][Field-EnglishTeacher][Field-SocialTeacher]', 'teacher' ] csv_string = File.read(File.expand_path("../../../data/MeasureKey#{@year}.csv", __FILE__)) csv = CSV.parse(csv_string, :headers => true) t = Time.new csv.each_with_index do |question, index| category = nil question['Category19'].split('-').each do |external_id_raw| external_id = external_id_raw.gsub(/[[:space:]]/, ' ').strip categories = category.present? ? category.child_categories : Category category = categories.where(external_id: external_id).first if category.nil? puts 'NOTHING' puts "#{question['Category']} -- #{external_id}" puts categories.map { |c| "#{c.name} - |#{c.external_id}| == |#{external_id}|: - #{external_id == c.external_id}"}.join(" ---- ") category = categories.create(name: question['Category Name'], external_id: external_id) end end question_text = question['Question Text'].gsub(/[[:space:]]/, ' ').strip if question_text.index('.* teacher').nil? category.questions.create( text: question_text, option1: question['R1'], option2: question['R2'], option3: question['R3'], option4: question['R4'], option5: question['R5'], for_recipient_students: question['Level'] == "Students", external_id: question['qid'], reverse: question['Reverse'] == "1" ) else variations.each do |variation| category.questions.create( text: question_text.gsub('.* teacher', variation), option1: question['R1'], option2: question['R2'], option3: question['R3'], option4: question['R4'], option5: question['R5'], for_recipient_students: question['Level'] == "Students", external_id: question['qid'], reverse: question['Reverse'] == "1" ) end end end end desc 'Load in student and teacher responses' task load_responses: :environment do ENV['BULK_PROCESS'] = 'true' answer_dictionary = { 'Slightly': 'Somewhat', 'an incredible': 'a tremendous', 'a little': 'a little bit', 'slightly': 'somewhat', 'a little well': 'slightly well', 'quite': 'very', 'a tremendous': 'a very great', 'somewhat clearly': 'somewhat', 'almost never': 'once in a while', 'always': 'all the time', 'not at all strong': 'not strong at all', 'each': 'every' } respondent_map = {} unknown_schools = {} missing_questions = {} bad_answers = {} timeToRun = 120000 * 60 startIndex = 0 stopIndex = 1000000 startTime = Time.new # ['teacher_responses'].each do |file| ['student_responses', 'teacher_responses'].each do |file| recipients = file.split('_')[0] target_group = Question.target_groups["for_#{recipients}s"] csv_string = File.read(File.expand_path("../../../data/#{file}_#{@year}.csv", __FILE__)) csv = CSV.parse(csv_string, :headers => true) puts("LOADING CSV: #{csv.length} ROWS") t = Time.new csv.each_with_index do |row, index| next if index < startIndex if Time.new - startTime >= timeToRun || index > stopIndex puts("ENDING #{timeToRun} SECONDS: #{Time.new - startTime} = #{startIndex} -> #{index} = #{index - startIndex} or #{(Time.new - t) / (index - startIndex)} per second") break end if index % 10 == 0 puts("DATAMSG: PROCESSING ROW: #{index} OUT OF #{csv.length} ROWS: #{Time.new - t} - Total: #{Time.new - startTime} - #{timeToRun - (Time.new - startTime)} TO GO / #{stopIndex - startIndex} ROWS TO GO") t = Time.new end district_name = row['Q111'].strip if district_name.blank? || district_name == "NA" puts "DISTRICT NOT FOUND: #{district_name}" next end # district_name = row['To begin, please select your district.'] if district_name.nil? district = District.find_or_create_by(name: district_name, state_id: 1) school_name = row["SchoolName"].strip if school_name.blank? || school_name == "NA" puts "BLANK SCHOOL NAME: #{district.name} - #{index}" next end school = district.schools.find_or_create_by(name: school_name) if school.nil? next if unknown_schools[school_name] puts "DATAERROR: Unable to find school: #{school_name} - #{index}" unknown_schools[school_name] = true next end respondent_id = "#{recipients}-#{index}-#{row["ResponseId"].strip}" recipient_id = respondent_map["#{school.id}-#{@year}-#{respondent_id}"] if recipient_id.present? recipient = school.recipients.where(id: recipient_id).first end if recipient.nil? begin recipient = school.recipients.create( name: "Survey Respondent Id: #{respondent_id}" ) rescue puts "DATAERROR: INDEX: #{index} ERROR AT #{index} - #{district.name} - #{school_name} #{school}: #{respondent_id}" end respondent_map["#{school.id}-#{respondent_id}"] = recipient.id end recipient_list = school.recipient_lists.find_by_name("#{recipients.titleize} List") if recipient_list.nil? recipient_list = school.recipient_lists.create(name: "#{recipients.titleize} List") end recipient_list.recipient_id_array << recipient.id recipient_list.save! row.each do |key, value| t1 = Time.new next if value.nil? or key.nil? or value.to_s == "-99" key = key.gsub(/[[:space:]]/, ' ').gsub(/\./, '-').strip.gsub(/\s+/, ' ') key = key.gsub(/-4-5/, '').gsub(/-6-12/, '') value = value.gsub(/[[:space:]]/, ' ').strip.downcase begin question = Question.created_in(@year).find_by_external_id(key) rescue Exception => e puts "DATAERROR: INDEX: #{index} Failed finding question: #{key} -> #{e}" end if question.nil? next if missing_questions[key] puts "DATAERROR: Unable to find question: #{key}" missing_questions[key] = true next else question.update_attributes(target_group: target_group) if question.unknown? end if (value.to_i.blank?) answer_index = question.option_index(value) answer_dictionary.each do |k, v| break if answer_index.present? answer_index = question.option_index(value.gsub(k.to_s, v.to_s)) answer_index = question.option_index(value.gsub(v.to_s, k.to_s)) if answer_index.nil? end if answer_index.nil? next if bad_answers[key] puts "DATAERROR: Unable to find answer: #{key} = #{value.downcase.strip} - #{question.options.inspect}" bad_answers[key] = true next end else answer_index = value.to_i end next if answer_index == 0 # answer_index = 6 - answer_index if question.reverse? responded_at = Date.strptime(row['recordedDate'], '%Y-%m-%d %H:%M:%S') rescue Date.today begin recipient.attempts.create(question: question, answer_index: answer_index, responded_at: responded_at) rescue Exception => e puts "DATAERROR: INDEX: #{index} Attempt failed for #{recipient.inspect} -> QUESTION: #{question.inspect}, ANSWER_INDEX: #{answer_index}, RESPONDED_AT: #{responded_at}, ERROR: #{e}" next end end end end ENV.delete('BULK_PROCESS') sync_school_category_aggregates # Recipient.created_in(@year).each { |r| r.update_counts } end desc 'Load in nonlikert values for each school' task load_nonlikert_values: :environment do csv_string = File.read(File.expand_path("../data/MCIEA_17-18AdminData.csv", __FILE__)) # csv_string = File.read(File.expand_path("../../../data/MCIEA_16-17_SGP.csv", __FILE__)) csv = CSV.parse(csv_string, :headers => true) puts("LOADING NONLIKERT CSV: #{csv.length} ROWS") errors = [] csv.each_with_index do |row, index| next if row["Likert_Value"].blank? base = Category category_ids = row["Category"].split("-") category_ids.each do |category_id| category_id = category_id.downcase if category_id.downcase =~ /i/ base = base.find_by_external_id(category_id) if base.nil? row["reason"] = "Unable to find category_id #{category_id} for category #{row["Category"]}" errors << row next end base = base.child_categories end nonlikert_category = base.where(name: row["NonLikert Title"]).first if nonlikert_category.nil? row["reason"] = "Unable to find nonlikert category: #{row["NonLikert Title"]} in #{}" errors << row next else if (benchmark = row["Benchmark"]).present? nonlikert_category.update(benchmark: benchmark) end end district = District.where(name: row["District"], state_id: 1).first if district.blank? row["reason"] = "DISTRICT NOT FOUND: #{row["District"]}" errors << row next end school = district.schools.where(name: row["School"]).first if school.blank? row["reason"] = "SCHOOL NOT FOUND: #{row["School"]}" errors << row next end school_category = school.school_categories.find_or_create_by(category: nonlikert_category, year: "#{@year}") if school_category.blank? row["reason"] = "SCHOOL CATEGORY NOT FOUND: #{school.name} #{nonlikert_category.name}" errors << row next end zscore = (([-2,[row["Likert_Value"].to_f-3,2].min].max * 10).to_i).to_f / 10.0 school_category.update( nonlikert: row["NL_Value"], zscore: zscore.to_f, year: "#{@year}", valid_child_count: 1 ) school_category.reload.sync_aggregated_responses end errors.each do |error| puts "#{error["reason"]}: #{error["NonLikert Title"]} -> #{error["Likert_Value"]}" end puts "COUNT: #{SchoolCategory.where(attempt_count: 0, answer_index_total: 0).where("nonlikert is not null and zscore is null").count}" end desc 'Load in custom zones for each category' task load_custom_zones: :environment do ENV['BULK_PROCESS'] = 'true' csv_string = File.read(File.expand_path("../../../data/Benchmarks2016-2017.csv", __FILE__)) csv = CSV.parse(csv_string, :headers => true) csv.each_with_index do |row, index| next if row["Warning High"].blank? category = Category.find_by_name(row["Subcategory"]) if category.nil? puts "Unable to find category #{row["Subcategory"]}" next end custom_zones = [ row["Warning High"], row["Watch High"], row["Growth High"], row["Approval High"], 5 ] puts "#{category.name} -> #{custom_zones.join(",")}" category.update(zones: custom_zones.join(",")) end ENV.delete('BULK_PROCESS') Category.all.each { |category| category.sync_child_zones } end desc 'Move all likert survey results to a new submeasure of current measure' task move_likert_to_submeasures: :environment do Question.all.each do |q| category = q.category next unless category.name.index("Scale").nil? new_category_name = "#{category.name} Scale" new_category = category.child_categories.where(name: new_category_name).first if new_category.nil? new_category = category.child_categories.create( name: new_category_name, blurb: "This measure contains all survey responses for #{category.name}.", description: "The following survey questions concern perceptions of #{category.name}.", zones: category.zones ) end q.update(category: new_category) end # sync_school_category_aggregates end desc 'Sync all school category aggregates' task sync: :environment do sync_school_category_aggregates # Recipient.created_in(@year).each { |r| r.update_counts } end desc 'Create School Questions' task create_school_questions: :environment do Category.joins(:questions).uniq.all.each do |category| category.school_categories.joins(school: :district).where("districts.name = 'Boston'").find_in_batches(batch_size: 100) do |group| group.each do |school_category| school_questions = [] new_school_questions = [] category.questions.created_in(school_category.year).each do |question| school = school_category.school next if school.district.name != "Boston" school_question = school_category.school_questions.for(school, question).first if school_question.present? school_questions << school_question else attempt_data = Attempt. joins(:question). created_in(school_category.year). for_question(question). for_school(school). select('count(attempts.answer_index) as response_count'). select('sum(case when questions.reverse then 6 - attempts.answer_index else attempts.answer_index end) as answer_index_total')[0] available_responders = school.available_responders_for(question) school_question = school_category.school_questions.new( school: school, question: question, school_category: school_category, year: school_category.year, attempt_count: available_responders, response_count: attempt_data.response_count, response_rate: attempt_data.response_count.to_f / available_responders.to_f, response_total: attempt_data.answer_index_total ) new_school_questions << school_question school_questions << school_question end end SchoolQuestion.import new_school_questions end end end end def sync_school_category_aggregates School.all.each do |school| Category.all.each do |category| school_category = SchoolCategory.for(school, category).in(@year).first if school_category.nil? school_category = school.school_categories.create(category: category, year: @year) end school_category.sync_aggregated_responses end end end end # # require 'csv' # student_counts_string = File.read(File.expand_path("data/bps_student_counts.csv")) # student_counts = CSV.parse(student_counts_string, :headers => true) # missing_schools = [] # student_counts.each_with_index do |count, index| # school = School.find_by_name(count["SCHOOL NAME"]) # # if school.nil? # puts("Unable to find school: #{count["SCHOOL NAME"]}") # missing_schools << count["SCHOOL NAME"] # next # end # # school.update(student_count: count["Student Enrollment (Grades 4-11)"]) # end # puts "" # puts "MISSING SCHOOLS: #{missing_schools.length}" # missing_schools.each { |s| puts(s) } # # # require 'csv' # teacher_counts_string = File.read(File.expand_path("data/bps_teacher_counts.csv")) # teacher_counts = CSV.parse(teacher_counts_string, :headers => true) # missing_schools = [] # teacher_counts.each_with_index do |count, index| # school = School.find_by_name(count["SCHOOL NAME"]) # # if school.nil? # puts("Unable to find school: #{count["SCHOOL NAME"]}") # missing_schools << count["SCHOOL NAME"] # next # end # # school.update(teacher_count: count["Teacher Count"]) # end # puts "" # puts "MISSING SCHOOLS: #{missing_schools.length}" # missing_schools.each { |s| puts(s) } # # # min_response_rate = 0.3 # level = 1 # categories = Category.joins(:questions).uniq.all # # categories = [Category.find_by_slug("student-emotional-safety-scale")] # categories.each do |category| # category.school_categories.joins(school: :district).where("districts.name = 'Boston'").each do |school_category| # # category.school_categories.joins(school: :district).where("districts.name = 'Boston' and schools.slug = 'boston-community-leadership-academy'").each do |school_category| # school_question_data = school_category. # school_questions. # where("response_rate > #{min_response_rate}"). # select('count(response_count) as valid_child_count'). # select('sum(response_count) as response_count'). # select('sum(response_total) as response_total')[0] # # valid_child_count = school_question_data.valid_child_count # school_questions = school_category.school_questions.joins(:question) # student_questions = school_questions.merge(Question.for_students) # teacher_questions = school_questions.merge(Question.for_teachers) # if (student_questions.count > 0 && teacher_questions.count > 0) # NEED TO CHECK IF STUDENT OR TEACHER QUESTIONS EXIST # if (student_questions.where("response_rate > #{min_response_rate}").count == 0 || # teacher_questions.where("response_rate > #{min_response_rate}").count == 0) # valid_child_count = 0 # end # end # # puts "VALID CHILD COUNT: #{valid_child_count}" # school_category.update( # valid_child_count: valid_child_count, # response_count: school_question_data.response_count, # answer_index_total: school_question_data.response_total, # zscore: (school_question_data.response_total.to_f/school_question_data.response_count.to_f) - 3.to_f # ) # end # end # # categories = Category.where("slug like '%-scale'") # # loop do # parent_categories = [] # categories.each_with_index do |category, i| # parent_category = category.parent_category # next if parent_category.nil? || parent_categories.include?(parent_category) # parent_categories << parent_category # # school_categories = parent_category.school_categories.joins(school: :district).where("districts.name = 'Boston'") # school_categories.each_with_index do |school_category, index| # school = school_category.school # # children = SchoolCategory.for_parent_category(school, parent_category).in(school_category.year) # valid_children = children.where("valid_child_count > 0") # school_category.update( # valid_child_count: valid_children.count, # response_count: valid_children.sum(&:response_count), # answer_index_total: valid_children.sum(&:answer_index_total), # zscore: (valid_children.sum(&:answer_index_total).to_f / valid_children.sum(&:response_count).to_f) - 3.to_f # ) # end # end # # puts "" # puts "" # puts "PARENT CATEGORIES: #{parent_categories.uniq.length}" # puts "" # puts "" # # # level += 1 # categories = parent_categories.uniq # break if categories.blank? # end # total = 0 # District.find_by_name('Somerville').schools.each do |school| # base_categories = Category.joins(:questions).map(&:parent_category).to_a.flatten.uniq # base_categories.each do |category| # SchoolCategory.for(school, category).in("2018").valid.each do |school_category| # dup_school_categories = SchoolCategory.for(school, category).in("2018") # if dup_school_categories.count > 1 # dup_school_categories.each { |dsc| dsc.destroy unless dsc.id == school_category.id } # school_category.sync_aggregated_responses # parent = category.parent_category # while parent != nil # SchoolCategory.for(school, parent).in("2018").valid.each do |parent_school_category| # parent_dup_school_categories = SchoolCategory.for(school, parent).in("2018") # if parent_dup_school_categories.count > 1 # parent_dup_school_categories.each { |pdsc| pdsc.destroy unless pdsc.id == parent_school_category.id } # parent_school_category.sync_aggregated_responses # end # end # parent = parent.parent_category # end # # total += 1 # end # end # end # end # # puts "TOTAL: #{total}" # [ # "baldwin-early-learning-pilot-academy" # ].each do |slug| # school = School.find_by_slug(slug) # base_categories = Category.joins(:questions).to_a.flatten.uniq # base_categories.each do |category| # SchoolCategory.for(school, category).in("2018").valid.each do |school_category| # dup_school_categories = SchoolCategory.for(school, category).in("2018") # if dup_school_categories.count > 1 # dup_school_categories.each { |dsc| dsc.destroy unless dsc.id == school_category.id } # school_category.sync_aggregated_responses # parent = category.parent_category # while parent != nil # SchoolCategory.for(school, parent).in("2018").valid.each do |parent_school_category| # parent_dup_school_categories = SchoolCategory.for(school, parent).in("2018") # if parent_dup_school_categories.count > 1 # parent_dup_school_categories.each { |pdsc| pdsc.destroy unless pdsc.id == parent_school_category.id } # parent_school_category.sync_aggregated_responses # end # end # parent = parent.parent_category # end # end # end # end # end # # category = Category.find_by_slug("professional-qualifications-scale") # category.school_categories.joins(:school).each do |school_category| # school_question_data = school_category. # school_questions. # where("response_rate > #{min_response_rate}"). # select('count(response_count) as valid_child_count'). # select('sum(response_count) as response_count'). # select('sum(response_total) as response_total')[0] # # valid_child_count = school_question_data.valid_child_count # school_questions = school_category.school_questions.joins(:question) # student_questions = school_questions.merge(Question.for_students) # teacher_questions = school_questions.merge(Question.for_teachers) # if (student_questions.count > 0 && teacher_questions.count > 0) # if (student_questions.where("response_rate > #{min_response_rate}").count == 0 || # teacher_questions.where("response_rate > #{min_response_rate}").count == 0) # valid_child_count = 0 # end # end # # puts "VALID CHILD COUNT: #{valid_child_count}" # school_category.update( # valid_child_count: valid_child_count, # response_count: school_question_data.response_count, # answer_index_total: school_question_data.response_total, # zscore: (school_question_data.response_total.to_f/school_question_data.response_count.to_f) - 3.to_f # ) # end # # # category = Category.find_by_slug("professional-qualifications-scale") # school = School.find_by_slug("young-achievers-science-and-math-k-8-school") # school_category = SchoolCategory.for(school,category).in("2018").first # school_category.school_questions.each do |school_question| # attempt_data = Attempt. # joins(:question). # created_in(school_category.year). # for_question(school_question.question). # for_school(school). # select('count(attempts.answer_index) as response_count'). # select('sum(case when questions.reverse then 6 - attempts.answer_index else attempts.answer_index end) as answer_index_total')[0] # # available_responders = school.available_responders_for(school_question.question) # # school_question.update( # attempt_count: available_responders, # response_count: attempt_data.response_count, # response_rate: attempt_data.response_count.to_f / available_responders.to_f, # response_total: attempt_data.answer_index_total # ) # end # # min_response_rate = 0.3 # school_question_data = school_category. # school_questions. # where("response_rate > #{min_response_rate}"). # select('count(response_count) as valid_child_count'). # select('sum(response_count) as response_count'). # select('sum(response_total) as response_total')[0] # # valid_child_count = school_question_data.valid_child_count # school_questions = school_category.school_questions.joins(:question) # student_questions = school_questions.merge(Question.for_students) # teacher_questions = school_questions.merge(Question.for_teachers) # if (student_questions.count > 0 && teacher_questions.count > 0) # if (student_questions.where("response_rate > #{min_response_rate}").count == 0 || # teacher_questions.where("response_rate > #{min_response_rate}").count == 0) # valid_child_count = 0 # end # end # # puts "VALID CHILD COUNT: #{valid_child_count}" # school_category.update( # valid_child_count: valid_child_count, # response_count: school_question_data.response_count, # answer_index_total: school_question_data.response_total, # zscore: (school_question_data.response_total.to_f/school_question_data.response_count.to_f) - 3.to_f # ) # # parent = category.parent_category # while parent != nil # SchoolCategory.for(school, parent).in("2018").valid.each do |parent_school_category| # parent_school_category.sync_aggregated_responses # end # parent = parent.parent_category # end # # # category = Category.find_by_slug("health") # District.find_by_name("Boston").schools.each do |school| # school_category = SchoolCategory.for(school, category).in("2018") # valid_children = SchoolCategory.for_parent_category(school, category).in("2018").where("valid_child_count > 0") # if valid_children.count == 1 && valid_children.first.category.slug == "physical-health" # school_category.update(valid_child_count: 0) # parent = category.parent_category # while parent != nil do # parent_school_category = SchoolCategory.for(school, parent).in("2018") # valid_children = SchoolCategory.for_parent_category(school, parent).in("2018").where("valid_child_count > 0") # parent_school_category.update(valid_child_count: valid_children.count) # parent = parent.parent_category # end # end # end # Question.created_in(2019).each do |question| # previous_year_question = Question.created_in(2018).find_by_external_id(question.external_id) # if previous_year_question.nil? # puts("No previous year question: #{question.external_id}") # previous_year_question = Question.created_in(2018).find_by_external_id("s-peff-q6") # end # question.update(category: previous_year_question.category) # end # categories = [] # Question.created_in(2019).where(reverse: true).each do |question| # question.attempts.each do |attempt| # attempt.update(answer_index: 6 - attempt.answer_index) # end # categories << question.category # end # categories.uniq.each do |category| # category.school_categories.in(2019).each do |school_category| # school_category.sync_aggregated_responses # end # end