PR #4 for silent failure work - Add lighthouse files and changes for uploading a document

.github/CODEOWNERS

@@ -663,6 +663,7 @@ app/sidekiq/kms_key_rotation @department-of-veterans-affairs/va-api-engineers @d
 app/sidekiq/lighthouse @department-of-veterans-affairs/backend-review-group
 app/sidekiq/lighthouse/submit_career_counseling_job.rb @department-of-veterans-affairs/my-education-benefits @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/lighthouse/create_intent_to_file_job.rb @department-of-veterans-affairs/pension-and-burials @department-of-veterans-affairs/backend-review-group
+app/sidekiq/lighthouse/evidence_submissions/document_upload.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
 app/sidekiq/lighthouse/income_and_assets_intake_job.rb @department-of-veterans-affairs/pension-and-burials @department-of-veterans-affairs/backend-review-group
 app/sidekiq/lighthouse/failure_notification.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/benefits-management-tools-be
 app/sidekiq/load_average_days_for_claim_completion_job.rb @department-of-veterans-affairs/benefits-microservices @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -950,6 +951,9 @@ lib/lighthouse @department-of-veterans-affairs/backend-review-group
 lib/lighthouse/benefits_claims @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/lighthouse/benefits_claims/monitor.rb @department-of-veterans-affairs/pension-and-burials @department-of-veterans-affairs/backend-review-group
 lib/lighthouse/benefits_documents/constants.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+lib/lighthouse/benefits_documents/service.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+lib/lighthouse/benefits_documents/update_documents_status_service.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
+lib/lighthouse/benefits_documents/upload_status_updater.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
 lib/lighthouse/benefits_documents/utilities/helpers.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/lighthouse/benefits_intake @department-of-veterans-affairs/pension-and-burials @department-of-veterans-affairs/backend-review-group
 lib/lighthouse/letters_generator @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -1057,6 +1061,7 @@ modules/ivc_champva @department-of-veterans-affairs/champva-engineering @departm
 modules/meb_api @department-of-veterans-affairs/my-education-benefits
 modules/mobile @department-of-veterans-affairs/mobile-api-team
 modules/mobile/app/assets/translations @department-of-veterans-affairs/flagship-mobile-content @department-of-veterans-affairs/backend-review-group
+modules/mobile/spec/requests/mobile/v0/claim/documents_spec.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 modules/mocked_authentication @department-of-veterans-affairs/octo-identity
 modules/my_health @department-of-veterans-affairs/vfs-mhv-secure-messaging @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 modules/my_health/app/controllers/my_health/v1/prescriptions_controller.rb @department-of-veterans-affairs/vfs-mhv-medications @department-of-veterans-affairs/backend-review-group
@@ -1402,6 +1407,7 @@ spec/sidekiq/income_limits @department-of-veterans-affairs/vfs-public-websites-f
 spec/sidekiq/in_progress_form_cleaner_spec.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/kms_key_rotation @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/lighthouse @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+spec/sidekiq/lighthouse/evidence_submissions/document_upload_spec.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/lighthouse/failure_notification_spec.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/benefits-management-tools-be
 spec/sidekiq/load_average_days_for_claim_completion_job_spec.rb @department-of-veterans-affairs/benefits-microservices @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/mhv @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -1499,7 +1505,9 @@ spec/lib/lighthouse @department-of-veterans-affairs/backend-review-group
 spec/lib/lighthouse/auth @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/lighthouse/benefits_claims @department-of-veterans-affairs/disability-experience @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/lighthouse/benefits_documents @department-of-veterans-affairs/backend-review-group
-spec/lib/lighthouse/benefits_documents/service_spec.rb @department-of-veterans-affairs/backend-review-group
+spec/lib/lighthouse/benefits_documents/service_spec.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+spec/lib/lighthouse/benefits_documents/update_documents_status_service_spec.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
+spec/lib/lighthouse/benefits_documents/upload_status_updater_spec.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
 spec/lib/lighthouse/benefits_intake @department-of-veterans-affairs/pension-and-burials @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/lighthouse/direct_deposit @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/dbex-trex @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/lighthouse/direct_deposit/payment_account_spec.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group

app/sidekiq/lighthouse/evidence_submissions/document_upload.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require 'datadog'
+require 'timeout'
+require 'lighthouse/benefits_documents/worker_service'
+require 'lighthouse/benefits_documents/constants'
+require 'lighthouse/benefits_documents/utilities/helpers'
+
+module Lighthouse
+  module EvidenceSubmissions
+    class DocumentUpload
+      include Sidekiq::Job
+      attr_accessor :user_icn, :document_hash
+
+      # retry for  2d 1h 47m 12s
+      # https://github.com/sidekiq/sidekiq/wiki/Error-Handling
+      sidekiq_options retry: 16, queue: 'low'
+      # Set minimum retry time to ~1 hour
+      sidekiq_retry_in do |count, _exception|
+        rand(3600..3660) if count < 9
+      end
+
+      sidekiq_retries_exhausted do |msg, _ex|
+        verify_msg(msg)
+
+        if Flipper.enabled?(:cst_send_evidence_submission_failure_emails)
+          update_evidence_submission_for_failure(msg)
+        else
+          call_failure_notification(msg)
+        end
+      end
+
+      def perform(user_icn, document_hash)
+        @user_icn = user_icn
+        @document_hash = document_hash
+
+        initialize_upload_document
+        perform_document_upload_to_lighthouse
+        clean_up!
+      end
+
+      def self.verify_msg(msg)
+        raise StandardError, "Missing fields in #{name}" if invalid_msg_fields?(msg) || invalid_msg_args?(msg['args'])
+      end
+
+      def self.invalid_msg_fields?(msg)
+        !(%w[jid args created_at failed_at] - msg.keys).empty?
+      end
+
+      def self.invalid_msg_args?(args)
+        return true unless args[1].is_a?(Hash)
+
+        !(%w[first_name claim_id document_type file_name tracked_item_id] - args[1].keys).empty?
+      end
+
+      def self.update_evidence_submission_for_failure(msg)
+        evidence_submission = EvidenceSubmission.find_by(job_id: msg['jid'])
+        current_personalisation = JSON.parse(evidence_submission.template_metadata)['personalisation']
+        evidence_submission.update(
+          upload_status: BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED],
+          template_metadata: {
+            personalisation: update_personalisation(current_personalisation, msg['failed_at'])
+          }.to_json
+        )
+        message = "#{name} EvidenceSubmission updated"
+        ::Rails.logger.info(message)
+        StatsD.increment('silent_failure_avoided_no_confirmation',
+                         tags: ['service:claim-status', "function: #{message}"])
+      rescue => e
+        error_message = "#{name} failed to update EvidenceSubmission"
+        ::Rails.logger.info(error_message, { messsage: e.message })
+        StatsD.increment('silent_failure', tags: ['service:claim-status', "function: #{error_message}"])
+      end
+
+      def self.call_failure_notification(msg)
+        return unless Flipper.enabled?(:cst_send_evidence_failure_emails)
+
+        icn = msg['args'].first
+
+        Lighthouse::FailureNotification.perform_async(icn, create_personalisation(msg))
+
+        ::Rails.logger.info("#{name} exhaustion handler email queued")
+        StatsD.increment('silent_failure_avoided_no_confirmation',
+                         tags: ['service:claim-status', 'function: evidence upload to Lighthouse'])
+      rescue => e
+        ::Rails.logger.error("#{name} exhaustion handler email error",
+                             { message: e.message })
+        StatsD.increment('silent_failure', tags: ['service:claim-status', 'function: evidence upload to Lighthouse'])
+        log_exception_to_sentry(e)
+      end
+
+      # Update personalisation here since an evidence submission record was previously created
+      def self.update_personalisation(current_personalisation, failed_at)
+        personalisation = current_personalisation.clone
+        personalisation['date_failed'] = BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(failed_at)
+        personalisation
+      end
+
+      # This will be used by Lighthouse::FailureNotification
+      def self.create_personalisation(msg)
+        first_name = msg['args'][1]['first_name'].titleize unless msg['args'][1]['first_name'].nil?
+        document_type = msg['args'][1]['document_type']
+        # Obscure the file name here since this will be used to generate a failed email
+        # NOTE: the template that we use for va_notify.send_email uses `filename` but we can also pass in `file_name`
+        filename = BenefitsDocuments::Utilities::Helpers.generate_obscured_file_name(msg['args'][1]['file_name'])
+        date_submitted = BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(msg['created_at'])
+        date_failed = BenefitsDocuments::Utilities::Helpers.format_date_for_mailers(msg['failed_at'])
+
+        { first_name:, document_type:, filename:, date_submitted:, date_failed: }
+      end
+
+      private
+
+      def initialize_upload_document
+        Datadog::Tracing.trace('Config/Initialize Upload Document') do
+          Sentry.set_tags(source: 'documents-upload')
+          validate_document!
+          uploader.retrieve_from_store!(document.file_name)
+        end
+      end
+
+      def validate_document!
+        raise Common::Exceptions::ValidationErrors, document unless document.valid?
+      end
+
+      def perform_document_upload_to_lighthouse
+        Datadog::Tracing.trace('Sidekiq Upload Document') do |span|
+          span.set_tag('Document File Size', file_body.size)
+          response = client.upload_document(file_body, document) # returns upload response which includes requestId
+          if Flipper.enabled?(:cst_send_evidence_submission_failure_emails)
+            update_evidence_submission_for_success(jid, response)
+          end
+        end
+      end
+
+      def clean_up!
+        Datadog::Tracing.trace('Remove Upload Document') do
+          uploader.remove!
+        end
+      end
+
+      def client
+        @client ||= BenefitsDocuments::WorkerService.new
+      end
+
+      def document
+        @document ||= LighthouseDocument.new(document_hash)
+      end
+
+      def uploader
+        @uploader ||= LighthouseDocumentUploader.new(user_icn, document.uploader_ids)
+      end
+
+      def perform_initial_file_read
+        Datadog::Tracing.trace('Sidekiq read_for_upload') do
+          uploader.read_for_upload
+        end
+      end
+
+      def file_body
+        @file_body ||= perform_initial_file_read
+      end
+
+      def update_evidence_submission_for_success(job_id, response)
+        evidence_submission = EvidenceSubmission.find_by(job_id:)
+        request_successful = response.body.dig('data', 'success')
+        if request_successful
+          request_id = response.body.dig('data', 'requestId')
+          evidence_submission.update(
+            upload_status: BenefitsDocuments::Constants::UPLOAD_STATUS[:SUCCESS],
+            request_id:
+          )
+        else
+          raise StandardError
+        end
+      end
+    end
+  end
+end

app/sidekiq/lighthouse/failure_notification.rb

@@ -22,11 +22,12 @@ def notify_client
     VaNotify::Service.new(NOTIFY_SETTINGS.api_key)
   end
 
-  def perform(icn, first_name, filename, date_submitted, date_failed)
+  def perform(icn, personalisation)
+    # NOTE: The filename in the personalisation that is passed in is obscured
     notify_client.send_email(
       recipient_identifier: { id_value: icn, id_type: 'ICN' },
       template_id: MAILER_TEMPLATE_ID,
-      personalisation: { first_name:, filename:, date_submitted:, date_failed: }
+      personalisation:
     )
 
     ::Rails.logger.info('Lighthouse::FailureNotification email sent')