edcommonwealth
/
sqm-dashboards
mirror of https://github.com/edcommonwealth/sqm-dashboards.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

finishing touches

Browse Source
pull/1/head
Jared Cosulich 8 years ago
parent c38c2a55d3
commit 4cee53ac03
1 changed files with 147 additions and 139 deletions
Show Stats Download Patch File Download Diff File

286
lib/tasks/data.rake
Unescape View File

@ -123,148 +123,156 @@ namespace :data do
desc 'Load in student and teacher responses' desc 'Load in student and teacher responses'
task load_responses: :environment do task load_responses: :environment do
ENV['BULK_PROCESS'] = 'true' # ENV['BULK_PROCESS'] = 'true'
answer_dictionary = { # answer_dictionary = {
'Slightly': 'Somewhat', # 'Slightly': 'Somewhat',
'an incredible': 'a tremendous', # 'an incredible': 'a tremendous',
'a little': 'a little bit', # 'a little': 'a little bit',
'slightly': 'somewhat', # 'slightly': 'somewhat',
'a little well': 'slightly well', # 'a little well': 'slightly well',
'quite': 'very', # 'quite': 'very',
'a tremendous': 'a very great', # 'a tremendous': 'a very great',
'somewhat clearly': 'somewhat', # 'somewhat clearly': 'somewhat',
'almost never': 'once in a while', # 'almost never': 'once in a while',
'always': 'all the time', # 'always': 'all the time',
'not at all strong': 'not strong at all', # 'not at all strong': 'not strong at all',
'each': 'every' # 'each': 'every'
} # }
respondent_map = {} # respondent_map = {}
#
unknown_schools = {} # unknown_schools = {}
missing_questions = {} # missing_questions = {}
bad_answers = {} # bad_answers = {}
year = '2017' # year = '2017'
#
timeToRun = 100 * 60 * 60 # timeToRun = 100 * 60 * 60
startIndex = 0 # startIndex = 0
stopIndex = 100000 # stopIndex = 100000
startTime = Time.new # startTime = Time.new
#
['student_responses', 'teacher_responses'].each do |file| # ['student_responses', 'teacher_responses'].each do |file|
recipients = file.split('_')[0] # recipients = file.split('_')[0]
target_group = Question.target_groups["for_#{recipients}s"] # target_group = Question.target_groups["for_#{recipients}s"]
csv_string = File.read(File.expand_path("../../../data/#{file}_#{year}.csv", __FILE__)) # csv_string = File.read(File.expand_path("../../../data/#{file}_#{year}.csv", __FILE__))
csv = CSV.parse(csv_string, :headers => true) # csv = CSV.parse(csv_string, :headers => true)
puts("LOADING CSV: #{csv.length} ROWS") # puts("LOADING CSV: #{csv.length} ROWS")
#
t = Time.new # t = Time.new
csv.each_with_index do |row, index| # csv.each_with_index do |row, index|
next if index < startIndex # next if index < startIndex
#
if Time.new - startTime >= timeToRun || index > stopIndex # if Time.new - startTime >= timeToRun || index > stopIndex
puts("ENDING #{timeToRun} SECONDS: #{Time.new - startTime} = #{startIndex} -> #{index} = #{index - startIndex} or #{(Time.new - t) / (index - startIndex)} per second") # puts("ENDING #{timeToRun} SECONDS: #{Time.new - startTime} = #{startIndex} -> #{index} = #{index - startIndex} or #{(Time.new - t) / (index - startIndex)} per second")
break # break
end # end
#
if index % 10 == 0 # if index % 10 == 0
puts("DATAMSG: PROCESSING ROW: #{index} OUT OF #{csv.length} ROWS: #{Time.new - t} - Total: #{Time.new - startTime} - #{timeToRun - (Time.new - startTime)} TO GO / #{stopIndex - startIndex} ROWS TO GO") # puts("DATAMSG: PROCESSING ROW: #{index} OUT OF #{csv.length} ROWS: #{Time.new - t} - Total: #{Time.new - startTime} - #{timeToRun - (Time.new - startTime)} TO GO / #{stopIndex - startIndex} ROWS TO GO")
t = Time.new # t = Time.new
end # end
#
district_name = row['What district is your school in?'] # district_name = row['What district is your school in?']
district_name = row['To begin, please select your district.'] if district_name.nil? # district_name = row['To begin, please select your district.'] if district_name.nil?
district = District.find_or_create_by(name: district_name, state_id: 1) # district = District.find_or_create_by(name: district_name, state_id: 1)
#
school_name = row["Please select your school in #{district_name}."] # school_name = row["Please select your school in #{district_name}."]
#
if school_name.blank? # if school_name.blank?
# puts "BLANK SCHOOL NAME: #{district.name} - #{index}" # # puts "BLANK SCHOOL NAME: #{district.name} - #{index}"
next # next
end # end
#
school = district.schools.find_or_create_by(name: school_name) # school = district.schools.find_or_create_by(name: school_name)
#
if school.nil? # if school.nil?
next if unknown_schools[school_name] # next if unknown_schools[school_name]
puts "DATAERROR: Unable to find school: #{school_name} - #{index}" # puts "DATAERROR: Unable to find school: #{school_name} - #{index}"
unknown_schools[school_name] = true # unknown_schools[school_name] = true
next # next
end # end
#
respondent_id = row['Response ID'] # respondent_id = row['Response ID']
recipient_id = respondent_map[respondent_id] # recipient_id = respondent_map[respondent_id]
if recipient_id.present? # if recipient_id.present?
recipient = school.recipients.where(id: recipient_id).first # recipient = school.recipients.where(id: recipient_id).first
else # else
begin # begin
recipient = school.recipients.create( # recipient = school.recipients.create(
name: "Survey Respondent Id: #{respondent_id}" # name: "Survey Respondent Id: #{respondent_id}"
) # )
rescue # rescue
puts "DATAERROR: INDEX: #{index} ERROR AT #{index} - #{district.name} - #{school_name} #{school}: #{respondent_id}" # puts "DATAERROR: INDEX: #{index} ERROR AT #{index} - #{district.name} - #{school_name} #{school}: #{respondent_id}"
end # end
respondent_map[respondent_id] = recipient.id # respondent_map[respondent_id] = recipient.id
end # end
#
recipient_list = school.recipient_lists.find_by_name("#{recipients.titleize} List") # recipient_list = school.recipient_lists.find_by_name("#{recipients.titleize} List")
if recipient_list.nil? # if recipient_list.nil?
recipient_list = school.recipient_lists.create(name: "#{recipients.titleize} List") # recipient_list = school.recipient_lists.create(name: "#{recipients.titleize} List")
end # end
recipient_list.recipient_id_array << recipient.id # recipient_list.recipient_id_array << recipient.id
recipient_list.save! # recipient_list.save!
#
row.each do |key, value| # row.each do |key, value|
t1 = Time.new # t1 = Time.new
next if value.nil? or key.nil? or value.to_s == "-99" # next if value.nil? or key.nil? or value.to_s == "-99"
key = key.gsub(/[[:space:]]/, ' ').strip.gsub(/\s+/, ' ') # key = key.gsub(/[[:space:]]/, ' ').strip.gsub(/\s+/, ' ')
value = value.gsub(/[[:space:]]/, ' ').strip.downcase # value = value.gsub(/[[:space:]]/, ' ').strip.downcase
#
begin # begin
question = Question.find_by_text(key) # question = Question.find_by_text(key)
rescue Exception => e # rescue Exception => e
puts "DATAERROR: INDEX: #{index} Failed finding question: #{key} -> #{e}" # puts "DATAERROR: INDEX: #{index} Failed finding question: #{key} -> #{e}"
end # end
#
if question.nil? # if question.nil?
next if missing_questions[key] # next if missing_questions[key]
puts "DATAERROR: Unable to find question: #{key}" # puts "DATAERROR: Unable to find question: #{key}"
missing_questions[key] = true # missing_questions[key] = true
next # next
else # else
question.update_attributes(target_group: target_group) if question.unknown? # question.update_attributes(target_group: target_group) if question.unknown?
end # end
#
if (value.to_i.blank?) # if (value.to_i.blank?)
answer_index = question.option_index(value) # answer_index = question.option_index(value)
answer_dictionary.each do |k, v| # answer_dictionary.each do |k, v|
break if answer_index.present? # break if answer_index.present?
answer_index = question.option_index(value.gsub(k.to_s, v.to_s)) # answer_index = question.option_index(value.gsub(k.to_s, v.to_s))
answer_index = question.option_index(value.gsub(v.to_s, k.to_s)) if answer_index.nil? # answer_index = question.option_index(value.gsub(v.to_s, k.to_s)) if answer_index.nil?
end # end
#
if answer_index.nil? # if answer_index.nil?
next if bad_answers[key] # next if bad_answers[key]
puts "DATAERROR: Unable to find answer: #{key} = #{value.downcase.strip} - #{question.options.inspect}" # puts "DATAERROR: Unable to find answer: #{key} = #{value.downcase.strip} - #{question.options.inspect}"
bad_answers[key] = true # bad_answers[key] = true
next # next
end # end
else # else
answer_index = value.to_i # answer_index = value.to_i
end # end
#
responded_at = Date.strptime(row['End Date'], '%m/%d/%Y %H:%M') # responded_at = Date.strptime(row['End Date'], '%m/%d/%Y %H:%M')
begin # begin
recipient.attempts.create(question: question, answer_index: answer_index, responded_at: responded_at) # recipient.attempts.create(question: question, answer_index: answer_index, responded_at: responded_at)
rescue Exception => e # rescue Exception => e
puts "DATAERROR: INDEX: #{index} Attempt failed for #{recipient.inspect} -> QUESTION: #{question.inspect}, ANSWER_INDEX: #{answer_index}, RESPONDED_AT: #{responded_at}, ERROR: #{e}" # puts "DATAERROR: INDEX: #{index} Attempt failed for #{recipient.inspect} -> QUESTION: #{question.inspect}, ANSWER_INDEX: #{answer_index}, RESPONDED_AT: #{responded_at}, ERROR: #{e}"
next # next
end # end
# end
# end
# end
# ENV.delete('BULK_PROCESS')
School.all.each do |school|
Category.all.each do |category|
school_category = SchoolCategory.for(school, category).first
if school_category.nil?
school_category = SchoolCategory.create(school: school, category: category)
end end
school_category.sync_aggregated_responses
end end
end end
ENV.delete('BULK_PROCESS')
SchoolCategory.all.each { |sc| sc.sync_aggregated_responses } # Recipient.all.each { |r| r.update_counts }
Recipient.all.each { |r| r.update_counts }
end end
end end

Write Preview
Loading…
Cancel
Save