Fix org update job (#20317)

* make sure active record throws an error if org can't be found

* update some keys

* wip

* clean up

* [WIP] Fixing tests

* Finish fixing tests

* [WIP] Add slack notifications

* [WIP] Working through slack testing impact

* Tests passing

* Log progress every run

* Cleanup and disable slack

* [WIP] updates

* [WIP] Clairfying api responses

* Finish fixing and versioning api responses

* Remove commented out binding.pry calls

* Restore Slack notify using existing webhook_url

---------

Co-authored-by: Josh Fike <[email protected]>

modules/veteran/app/sidekiq/organizations/queue_updates.rb

@@ -51,7 +51,7 @@ def queue_address_updates(data)
 
     def rows_to_process(rows)
       rows.map do |row|
-        org = Veteran::Service::Organization.find_by(poa: row[:id])
+        org = Veteran::Service::Organization.find_by!(poa: row[:id])
         diff = org.diff(row)
         row.merge(diff.merge({ address_exists: org.location.present? })) if diff.values.any?
       rescue ActiveRecord::RecordNotFound => e

modules/veteran/app/sidekiq/organizations/update.rb

@@ -15,14 +15,24 @@ class Update
     include Sidekiq::Job
     include SentryLogging
 
+    attr_accessor :slack_messages, :orgs_data
+
+    def initialize
+      @slack_messages = []
+    end
+
     # Processes each organization's data provided in JSON format.
     # This method parses the JSON, validates each organization's address, and updates the database records.
     # @param orgs_json [String] JSON string containing an array of organization data.
     def perform(orgs_json)
-      orgs_data = JSON.parse(orgs_json)
-      orgs_data.each { |org_data| process_org_data(org_data) }
+      @orgs_data = JSON.parse(orgs_json)
+      @orgs_data.each { |org_data| process_org_data(org_data) }
     rescue => e
       log_error("Error processing job: #{e.message}")
+    ensure
+      @slack_messages.unshift("Orgs processed: #{@orgs_data.size}") if @orgs_data&.any?
+      @slack_messages.unshift('Organizations::Update')
+      log_to_slack(@slack_messages.join("\n")) unless @slack_messages.empty?
     end
 
     private
@@ -31,20 +41,15 @@ def perform(orgs_json)
     # If the address validation fails or an error occurs during the update, the error is logged and the process
     # is halted for the current organization.
     # @param org_data [Hash] The organization data including id and address.
-    def process_org_data(org_data) # rubocop:disable Metrics/MethodLength
+    def process_org_data(org_data)
       return unless record_can_be_updated?(org_data)
 
       address_validation_api_response = nil
 
       if org_data['address_changed']
+        api_response = get_best_address_candidate(org_data['address'])
 
-        api_response = if Flipper.enabled?(:va_v3_contact_information_service)
-                         get_best_address_candidate(org_data)
-                       else
-                         get_best_address_candidate(org_data['address'])
-                       end
-
-        # don't update the record if there is not a valid address with non-zero lat and long at this point
+        # Don't update the record if there is not a valid address with non-zero lat and long at this point
         if api_response.nil?
           return
         else
@@ -71,23 +76,19 @@ def record_can_be_updated?(org_data)
     # @param address [Hash] A hash containing the details of the organization's address.
     # @return [VAProfile::Models::ValidationAddress] A validation address object ready for address validation service.
     def build_validation_address(address)
-      if Flipper.enabled?(:va_v3_contact_information_service)
-        validation_model = VAProfile::Models::V3::ValidationAddress
-        state_code = address['state']['state_code']
-        city = address['city_name']
-      else
-        validation_model = VAProfile::Models::ValidationAddress
-        state_code = address['state_province']['code']
-        city = address['city']
-      end
+      validation_model = if Flipper.enabled?(:va_v3_contact_information_service)
+                           VAProfile::Models::V3::ValidationAddress
+                         else
+                           VAProfile::Models::ValidationAddress
+                         end
 
       validation_model.new(
         address_pou: address['address_pou'],
         address_line1: address['address_line1'],
         address_line2: address['address_line2'],
         address_line3: address['address_line3'],
-        city: city,
-        state_code: state_code,
+        city: address['city'],
+        state_code: address['state']['state_code'],
         zip_code: address['zip_code5'],
         zip_code_suffix: address['zip_code4'],
         country_code_iso3: address['country_code_iso3']
@@ -138,6 +139,7 @@ def build_address_attributes(org_data, api_response)
         address = api_response['candidate_addresses'].first['address']
         geocode = api_response['candidate_addresses'].first['geocode']
         meta = api_response['candidate_addresses'].first['address_meta_data']
+
         build_address(address, geocode, meta).merge({ raw_address: org_data['address'].to_json })
       end
     end
@@ -155,7 +157,7 @@ def build_address(address, geocode, meta)
         address_line3: address['address_line3'],
         city: address['city'],
         province: address['state_province']['name'],
-        state_code: address['state_province']['code'],
+        state_code: address['state_province']['state_code'],
         zip_code: address['zip_code5'],
         zip_suffix: address['zip_code4'],
         country_code_iso3: address['country']['iso3_code'],
@@ -179,7 +181,7 @@ def build_v3_address(address)
         state_code: address['state']['state_code'],
         zip_code: address['zip_code5'],
         zip_suffix: address['zip_code4'],
-        country_code_iso3: address['country']['iso3_code'],
+        country_code_iso3: address['country']['country_code_iso3'],
         country_name: address['country']['country_name'],
         county_name: address.dig('county', 'county_name'),
         county_code: address.dig('county', 'county_code'),
@@ -192,7 +194,9 @@ def build_v3_address(address)
     # Logs an error to Sentry.
     # @param error [Exception] The error string to be logged.
     def log_error(error)
-      log_message_to_sentry("Organizations::Update: #{error}", :error)
+      message = "Organizations::Update: #{error}"
+      log_message_to_sentry(message, :error)
+      @slack_messages << message
     end
 
     # Checks if the latitude and longitude of an address are both set to zero, which are the default values
@@ -261,6 +265,11 @@ def retry_validation(org_address)
     # @param org_address [Hash] the address provided by OGC
     # @return [Hash, Nil] the response from the address validation service
     def get_best_address_candidate(org_address)
+      if org_address.nil?
+        log_error('In #get_best_address_candidate, org_address is nil')
+        return nil
+      end
+
       candidate_address = build_validation_address(org_address)
       original_response = validate_address(candidate_address)
       return nil unless address_valid?(original_response)
@@ -278,6 +287,15 @@ def get_best_address_candidate(org_address)
       else
         original_response
       end
+    rescue => e
+      log_error("In #get_best_address_candidate, address: #{org_address}, error message: #{e.message}")
+    end
+
+    def log_to_slack(message)
+      client = SlackNotify::Client.new(webhook_url: Settings.edu.slack.webhook_url,
+                                       channel: '#benefits-representation-management-notifications',
+                                       username: 'Organizations::Update Bot')
+      client.notify(message)
     end
   end
 end

modules/veteran/app/sidekiq/organizations/xlsx_file_processor.rb

@@ -138,7 +138,7 @@ def process_row(row, sheet_name, column_map)
           address_line2: get_value(row, column_map, 'OrganizationAddressLine2'),
           address_line3: get_value(row, column_map, 'OrganizationAddressLine3'),
           city: get_value(row, column_map, 'OrganizationCity'),
-          state_province: { code: get_value(row, column_map, 'OrganizationState') },
+          state: { state_code: get_value(row, column_map, 'OrganizationState') },
           zip_code5:,
           zip_code4:,
           country_code_iso3: 'US'