updated code by adding a task to find most skipped questions on regular survey questions

lib/tasks/one_off.rake

@@ -201,7 +201,7 @@ namespace :one_off do
   end
   desc "Generate CSV report of teacher survey item responses"
   task teacher_survey_questions_csv: :environment do
-    headers = ['School ID', 'Academic Year', 'Survey Item', 'Count','Percentage_diff']
+    headers = ['School ID', 'Academic Year', 'Survey Item','Prmpt', 'Count','Percentage_diff']
     output_rows = []
     counts={}
     avg=0
@@ -212,7 +212,7 @@ namespace :one_off do
         threshold = Respondent.where(school: sc, academic_year: ay).pluck(:total_teachers)
         SurveyItem.teacher_survey_items.all.each do |si|
           c = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).group(:school_id).count
-          if c.any? && (c.values.first >= 10 || c.values.first > threshold/4)
+           if c.any?
             counts[[sc.id, ay.id, si.id]] = c
             sum+=c.values.first
             nof_si+=1
@@ -245,20 +245,87 @@ namespace :one_off do
     puts "CSV report of teacher survey item responses with removed stray responses.csv"
   end
 
+  desc "Generate CSV report of regular survey item responses"
+  task regular_survey_questions_csv: :environment do
+    headers = ['School ID', 'Academic Year', 'Survey Item','Prompt', 'Count','Percentage_diff']
+    output_rows = []
+    counts={}
+    avg=0
+    temp=0
+    count_response_grades_not_in_grades=0
+    School.all.each do |sc|
+      AcademicYear.all.each do |ay|
+        if Respondent.where(school: sc,academic_year:ay).any?
+      grades_count=Respondent.where(school:sc,academic_year:ay).first.counts_by_grade
+      grades= grades_count.keys
+      else 
+       grades=[]
+      end
+        sum=0
+        nof_si=0
+        threshold=0
+        threshold = Respondent.where(school: sc, academic_year: ay).pluck(:total_students)
+        SurveyItem.where.not(on_short_form:true).where.not("survey_items.survey_item_id LIKE '%-%-es%'").where.not("survey_items.survey_item_id LIKE 't-%'").all.each do |si|
+          c = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).group(:school_id).count
+          response_grades = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).pluck(:grade)
+          t = response_grades.count { |grade| !grades.include?(grade) }
+          count_response_grades_not_in_grades+=t
+          response_grades=[]
+           if c.any?
+            c[c.keys.first] = c.values.first - t
+          end
+
+           if threshold.any?
+          if c.any? && (c.values.first >= 10 || c.values.first > threshold.first/4)
+            counts[[sc.id, ay.id, si.id]] = c
+            sum+=c.values.first
+            nof_si+=1
+          end
+        end 
+        end
+
+          
+          avg = sum.to_f/ nof_si
+        counts.each do |key, value|
+          if key[0] == sc.id && key[1] == ay.id
+            count = value.values.first
+            percentage_diff = ((count-avg) / avg) * 100
+            counts[key] = { count: count, percentage_diff: percentage_diff } 
+        end
+      end
+    end  end 
+    
+    counts.each do |key, value|
+      output_rows << [School.where(id:key[0]).pluck(:name) , AcademicYear.where(id:key[1]).pluck(:range) , SurveyItem.where(id:key[2]).pluck(:survey_item_id), SurveyItem.where(id:key[2]).pluck(:prompt), value[:count], value[:percentage_diff]]
+    end
+    
+    file = File.new('regular_survey_questions.csv', 'w')
+    CSV.open(file, 'w', write_headers: true, headers: headers) do |csv|
+      output_rows.each do |row|
+        csv << row
+      end
+    end
+    
+    file.close
+    puts "CSV report of regular survey item responses with removed stray responses.csv"
+  end
 
   desc "Generate CSV report of short survey item responses"
   task short_survey_questions_csv: :environment do
-    headers = ['School ID', 'Academic Year', 'Survey Item', 'Count','Percentage_diff']
+    headers = ['School ID', 'Academic Year', 'Survey Item','Prompt', 'Count','Percentage_diff']
     output_rows = []
     counts={}
     avg=0
     temp=0
     count_response_grades_not_in_grades=0
     School.all.each do |sc|
-      if Respondent.where(school: sc).any?
-      grades_count=Respondent.where(school:sc).first.counts_by_grade
-      grades= grades_count.keys
       AcademicYear.all.each do |ay|
+        if Respondent.where(school: sc,academic_year:ay).any?
+      grades_count=Respondent.where(school:sc,academic_year:ay).first.counts_by_grade
+      grades= grades_count.keys
+      else 
+       grades=[]
+      end
         sum=0
         nof_si=0
         threshold=0
@@ -266,9 +333,11 @@ namespace :one_off do
         SurveyItem.short_form_items.all.each do |si|
           c = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).group(:school_id).count
           response_grades = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).pluck(:grade)
-          count_response_grades_not_in_grades = response_grades.count { |grade| !grades.include?(grade) }
+          t = response_grades.count { |grade| !grades.include?(grade) }
-           if !count_response_grades_not_in_grades.nil? && c.any?
+          count_response_grades_not_in_grades+=t
-            c[c.keys.first] = c.values.first - count_response_grades_not_in_grades
+          response_grades=[]
+           if c.any?
+            c[c.keys.first] = c.values.first - t
           end
 
            if threshold.any?
@@ -289,7 +358,7 @@ namespace :one_off do
             counts[key] = { count: count, percentage_diff: percentage_diff } 
         end
       end
-    end  end end
+    end  end 
     
     counts.each do |key, value|
       output_rows << [School.where(id:key[0]).pluck(:name) , AcademicYear.where(id:key[1]).pluck(:range) , SurveyItem.where(id:key[2]).pluck(:survey_item_id), SurveyItem.where(id:key[2]).pluck(:prompt), value[:count], value[:percentage_diff]]
@@ -309,17 +378,20 @@ namespace :one_off do
 
     desc "Generate CSV report of early_education_surveysitem responses"
   task early_education_survey_questions_csv: :environment do
-    headers = ['School ID', 'Academic Year', 'Survey Item', 'Count','Percentage_diff']
+    headers = ['School ID', 'Academic Year', 'Survey Item','Prompt', 'Count','Percentage_diff']
     output_rows = []
     counts={}
     avg=0
     temp=0
     count_response_grades_not_in_grades=0
     School.all.each do |sc|
-      if Respondent.where(school: sc).any?
-      grades_count=Respondent.where(school:sc).first.counts_by_grade
-      grades= grades_count.keys
       AcademicYear.all.each do |ay|
+        if Respondent.where(school: sc,academic_year:ay).any?
+      grades_count=Respondent.where(school:sc,academic_year:ay).first.counts_by_grade
+      grades= grades_count.keys
+      else 
+       grades=[]
+      end
         sum=0
         nof_si=0
         threshold=0
@@ -327,9 +399,11 @@ namespace :one_off do
         SurveyItem.early_education_surveys.all.each do |si|
           c = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).group(:school_id).count
           response_grades = SurveyItemResponse.where(school: sc, academic_year: ay, survey_item: si).pluck(:grade)
-          count_response_grades_not_in_grades = response_grades.count { |grade| !grades.include?(grade) }
+          t = response_grades.count { |grade| !grades.include?(grade) }
-           if !count_response_grades_not_in_grades.nil? && c.any?
+          count_response_grades_not_in_grades+=t
-            c[c.keys.first] = c.values.first - count_response_grades_not_in_grades
+          response_grades=[]
+           if c.any?
+            c[c.keys.first] = c.values.first - t
           end
 
            if threshold.any?
@@ -350,7 +424,7 @@ namespace :one_off do
             counts[key] = { count: count, percentage_diff: percentage_diff } 
         end
       end
-    end  end end
+     end end
     
     counts.each do |key, value|
       output_rows << [School.where(id:key[0]).pluck(:name) , AcademicYear.where(id:key[1]).pluck(:range) , SurveyItem.where(id:key[2]).pluck(:survey_item_id), SurveyItem.where(id:key[2]).pluck(:prompt), value[:count], value[:percentage_diff]]