sqm-dashboards/app/services/survey_responses_data_loader.rb

# frozen_string_literal: true

class SurveyResponsesDataLoader
  def load_data(filepath:)
    File.open(filepath) do |file|
      headers = file.first
      headers_array = CSV.parse(headers).first
      all_survey_items = survey_items(headers:)

      file.lazy.each_slice(500) do |lines|
        survey_item_responses = CSV.parse(lines.join, headers:).map do |row|
          process_row(row: SurveyItemValues.new(row:, headers: headers_array, survey_items: all_survey_items, schools:,
                                                academic_years:))
        end

        SurveyItemResponse.import(
          survey_item_responses.compact.flatten,
          batch_size: 500,
          on_duplicate_key_update: :all
        )
      end
    end
    SocioEconomicCalculator.update_socio_economic_scores
  end

  def from_file(file:)
    headers = file.gets
    headers_array = CSV.parse(headers).first
    all_survey_items = survey_items(headers:)

    batch_size = 1000

    lines = []
    until file.eof?
      line = file.gets
      next unless line.present?

      lines << line
    end
    slices = []

    lines.each_slice(batch_size) do |slice|
      slices << slice
    end

    pool_size = 4
    jobs = Queue.new
    slices.each { |slice| jobs << slice }

    workers = pool_size.times.map do
      Thread.new do
        while (slice = jobs.pop(true))

          slice.each do |line|
            survey_item_responses = []
            CSV.parse(line, headers:).map do |row|
              row = process_row(row: SurveyItemValues.new(row:, headers: headers_array,
                                                          survey_items: all_survey_items, schools:, academic_years:))

              survey_item_responses.concat(row) unless row.nil?
            end

            survey_item_responses = survey_item_responses.compact.flatten

            SurveyItemResponse.import(survey_item_responses, batch_size:, on_duplicate_key_update: :all)

            survey_item_responses = []
          end
        end
      rescue ThreadError
      end
    end

    workers.each(&:join)

    GC.start
  end

  private

  def schools
    @schools ||= School.by_dese_id
  end

  def genders
    @genders ||= Gender.by_qualtrics_code
  end

  def races
    @races ||= Race.by_qualtrics_code
  end

  def incomes
    @incomes ||= Income.by_slug
  end

  def ells
    @ells ||= Ell.by_designation
  end

  def speds
    @speds ||= Sped.by_designation
  end

  def employments
    @employments ||= Employment.by_designation
  end

  def educations
    @educations ||= Education.by_designation
  end

  def benefits
    @benefits ||= Benefit.by_designation
  end

  def academic_years
    @academic_years ||= AcademicYear.all
  end

  def process_row(row:)
    return unless row.dese_id?
    return unless row.school.present?

    process_survey_items(row:)
  end

  def languages
    @languages ||= Language.by_designation
  end

  def housings
    @housings ||= Housing.by_designation
  end

  def process_survey_items(row:)
    student = nil
    parent = nil
    if row.respondent_type == :student
      student = Student.find_or_create_by(response_id: row.response_id, lasid: row.lasid)
      student.races.delete_all
      tmp_races = row.races.map { |race| races[race] }.reject(&:nil?)
      student.races += tmp_races
    end

    if row.respondent_type == :parent
      parent = Parent.find_or_create_by(response_id: row.response_id)
      parent.number_of_children = row.number_of_children
      parent.education = educations[row.education] if row.education.present?
      parent.benefit = benefits[row.benefits] if row.benefits.present?
      tmp_languages = row.languages.map { |language| languages[language] }.reject(&:nil?)
      parent.languages.delete_all
      parent.languages.concat(tmp_languages)

      parent.races.delete_all
      tmp_races = row.races_of_children.map { |race| races[race] }.reject(&:nil?)
      parent.races.concat(tmp_races)

      parent.genders.delete_all
      tmp_genders = row.genders_of_children.map { |race| genders[race] }.reject(&:nil?)
      parent.genders.concat(tmp_genders)

      parent.employments.delete_all
      tmp_employments = row.employments.map { |employment| employments[employment] }.reject(&:nil?)
      parent.employments.concat(tmp_employments)

      parent.housing = housings[row.housing] if row.housing.present?

      parent.save
    end

    row
      .survey_items
      .map do |survey_item|
        likert_score = row.likert_score(survey_item_id: survey_item.survey_item_id) || next

        unless likert_score.valid_likert_score?
          # puts("Response ID: #{row.response_id}, Likert score: #{likert_score} rejected") unless likert_score == "NA"
          next
        end

        response = row.survey_item_response(survey_item:)
        create_or_update_response(survey_item_response: response, likert_score:, row:, survey_item:, student:, parent:)
      end
      .compact
  end

  def create_or_update_response(survey_item_response:, likert_score:, row:, survey_item:, student:, parent:)
    gender = genders[row.gender]
    grade = row.grade
    income = incomes[row.income.parameterize]
    ell = ells[row.ell]
    sped = speds[row.sped]

    if survey_item_response.present?
      survey_item_response.likert_score = likert_score
      survey_item_response.grade = grade
      survey_item_response.gender = gender
      survey_item_response.recorded_date = row.recorded_date
      survey_item_response.income = income
      survey_item_response.ell = ell
      survey_item_response.sped = sped
      survey_item_response.student = student
      survey_item_response.parent = parent
      survey_item_response.save
      []
    else
      SurveyItemResponse.new(
        response_id: row.response_id,
        academic_year: row.academic_year,
        school: row.school,
        survey_item:,
        likert_score:,
        grade:,
        gender:,
        recorded_date: row.recorded_date,
        income:,
        ell:,
        sped:,
        student:,
        parent:
      )
    end
  end

  def survey_items(headers:)
    SurveyItem.where(survey_item_id: get_survey_item_ids_from_headers(headers:))
  end

  def get_survey_item_ids_from_headers(headers:)
    CSV
      .parse(headers)
      .first
      .filter(&:present?)
      .map(&:downcase)
      .filter { |header| header.start_with?("t-", "s-", "p-") }
  end
end

module StringMonkeyPatches
  def valid_likert_score?
    to_i.between?(1, 5)
  end
end

String.include(StringMonkeyPatches)