From 2e1e836573c6d29b2da63449544a3ce4352a3260 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 09:56:48 -0700 Subject: [PATCH 1/7] --- (#2832) updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- auth-api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth-api/requirements.txt b/auth-api/requirements.txt index 8a79a0d5b0..d699de029a 100644 --- a/auth-api/requirements.txt +++ b/auth-api/requirements.txt @@ -73,7 +73,7 @@ python-jose==3.3.0 python-memcached==1.62 pytz==2024.1 redis==5.0.4 -requests==2.31.0 +requests==2.32.0 rsa==4.9 semver==3.0.2 sentry-sdk==2.0.1 From 543c4f52785a123b6d441877719b393ec86baf0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 10:02:04 -0700 Subject: [PATCH 2/7] --- (#2833) updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- queue_services/auth-queue/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queue_services/auth-queue/requirements.txt b/queue_services/auth-queue/requirements.txt index cb3c89c358..5e91c09030 100644 --- a/queue_services/auth-queue/requirements.txt +++ b/queue_services/auth-queue/requirements.txt @@ -36,7 +36,7 @@ pyrsistent==0.20.0 python-dateutil==2.9.0.post0 python-dotenv==1.0.1 pytz==2024.1 -requests==2.31.0 +requests==2.32.0 rsa==4.9 semver==3.0.2 sentry-sdk==2.0.1 From 7f88114907847ade2a719b7f4d1bdb8f850ea810 Mon Sep 17 00:00:00 2001 From: Travis Semple Date: Tue, 21 May 2024 10:26:05 -0700 Subject: [PATCH 3/7] 20732 - Queue fixes (#2828) * Debugging print * more logging to figure out an issue * more debug * more debug * Disable update env * remove self._publisher * Fix up the queue * Change logging a bit, fix make files so it's rapid fire. * Move logging down a tad * Move logging.conf * Add in error for unknown message type, add in enums for missing sections * Put in detection against duplicate queue messages for account-mailer. * Point at new auth-api * lint fix * lint fix * lint fixes * Add in unit test for duplicate message ids * rename accident * Fix enums in account-mailer so they match sbc-common * use correct enum * Update URL, other one causes an error * Fix indent * Add in duplicate message handling for auth-queue. * fix up requirements * Fix linting issues * See if unit test passes again * restore update-env for makefiles * lint + test fix --- ...5_b3a741249edc_duplicate_queue_messages.py | 33 +++++++++++++++++++ auth-api/src/auth_api/models/__init__.py | 1 + auth-api/src/auth_api/models/membership.py | 2 +- .../models/pubsub_message_processing.py | 32 ++++++++++++++++++ .../src/auth_api/services/authorization.py | 1 + .../auth_api/services/gcp_queue/gcp_queue.py | 2 +- auth-api/src/auth_api/services/membership.py | 6 ++-- auth-api/src/auth_api/services/org.py | 4 +-- .../services/validators/payment_type.py | 1 + auth-api/src/auth_api/utils/custom_query.py | 2 +- .../tests/unit/services/test_authorization.py | 6 ++-- .../account-mailer/requirements.txt | 2 +- .../src/account_mailer/__init__.py | 4 +++ .../email_processors/pad_confirmation.py | 2 ++ .../email_templates/reset_passcode.html | 2 +- .../src/account_mailer/enums.py | 14 ++++---- .../{ => src/account_mailer}/logging.conf | 0 .../src/account_mailer/resources/worker.py | 23 +++++++++++-- .../tests/unit/test_worker_queue.py | 26 +++++++++++++++ .../account-mailer/tests/unit/utils.py | 8 ++--- queue_services/auth-queue/requirements.txt | 2 +- .../auth-queue/src/auth_queue/__init__.py | 4 +++ .../{ => src/auth_queue}/logging.conf | 0 .../src/auth_queue/resources/worker.py | 21 ++++++++++-- 24 files changed, 169 insertions(+), 29 deletions(-) create mode 100644 auth-api/migrations/versions/2024_05_15_b3a741249edc_duplicate_queue_messages.py create mode 100644 auth-api/src/auth_api/models/pubsub_message_processing.py rename queue_services/account-mailer/{ => src/account_mailer}/logging.conf (100%) rename queue_services/auth-queue/{ => src/auth_queue}/logging.conf (100%) diff --git a/auth-api/migrations/versions/2024_05_15_b3a741249edc_duplicate_queue_messages.py b/auth-api/migrations/versions/2024_05_15_b3a741249edc_duplicate_queue_messages.py new file mode 100644 index 0000000000..2cb3a8a858 --- /dev/null +++ b/auth-api/migrations/versions/2024_05_15_b3a741249edc_duplicate_queue_messages.py @@ -0,0 +1,33 @@ +"""Add in new table for account mailer for pubsub message processing. + +Revision ID: b3a741249edc +Revises: e2d1d6417607 +Create Date: 2024-05-15 14:52:45.780399 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b3a741249edc' +down_revision = 'e2d1d6417607' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('pubsub_message_processing', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('cloud_event_id', sa.String(length=250), nullable=False), + sa.Column('created', sa.DateTime(), nullable=True), + sa.Column('message_type', sa.String(length=250), nullable=False), + sa.Column('processed', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_pubsub_message_processing_id'), 'pubsub_message_processing', ['id'], unique=False) + + +def downgrade(): + op.drop_index(op.f('ix_pubsub_message_processing_id'), table_name='pubsub_message_processing') + op.drop_table('pubsub_message_processing') diff --git a/auth-api/src/auth_api/models/__init__.py b/auth-api/src/auth_api/models/__init__.py index fdb30738e2..7710f712dc 100644 --- a/auth-api/src/auth_api/models/__init__.py +++ b/auth-api/src/auth_api/models/__init__.py @@ -50,6 +50,7 @@ from .product_subscription import ProductSubscription from .product_subscriptions_status import ProductSubscriptionsStatus from .product_type_code import ProductTypeCode +from .pubsub_message_processing import PubSubMessageProcessing from .suspension_reason_code import SuspensionReasonCode from .task import Task from .user import User diff --git a/auth-api/src/auth_api/models/membership.py b/auth-api/src/auth_api/models/membership.py index a4bacd7406..ce049b0aaa 100644 --- a/auth-api/src/auth_api/models/membership.py +++ b/auth-api/src/auth_api/models/membership.py @@ -51,7 +51,7 @@ class Membership(VersionedModel): # pylint: disable=too-few-public-methods # Te user = relationship('User', foreign_keys=[user_id], lazy='select') org = relationship('Org', foreign_keys=[org_id], lazy='select') - def __init__(self, **kwargs): + def __init__(self, **kwargs): # pylint: disable=super-init-not-called """Initialize a new membership.""" self.org_id = kwargs.get('org_id') self.user_id = kwargs.get('user_id') diff --git a/auth-api/src/auth_api/models/pubsub_message_processing.py b/auth-api/src/auth_api/models/pubsub_message_processing.py new file mode 100644 index 0000000000..2e3dc2234f --- /dev/null +++ b/auth-api/src/auth_api/models/pubsub_message_processing.py @@ -0,0 +1,32 @@ +"""This model manages pubsub message processing. + +NOTE: Only use this when it's not possible to use other indicators to track message processing. + Currently used by the account-mailer / auth-queue. This prevents duplicates. +""" +import datetime as dt +import pytz + +from sqlalchemy import Column, DateTime, Integer, String +from .db import db + + +class PubSubMessageProcessing(db.Model): + """PubSub Message Processing for cloud event messages.""" + + __tablename__ = 'pubsub_message_processing' + + id = Column(Integer, index=True, primary_key=True) + cloud_event_id = Column(String(250), nullable=False) + created = Column(DateTime, default=dt.datetime.now(pytz.utc)) + message_type = Column(String(250), nullable=False) + processed = Column(DateTime, nullable=True) + + @classmethod + def find_by_id(cls, identifier): + """Find a pubsub message processing by id.""" + return cls.query.filter_by(id=identifier).one_or_none() + + @classmethod + def find_by_cloud_event_id_and_type(cls, cloud_event_id, message_type): + """Find a pubsub message processing for cloud event id and type.""" + return cls.query.filter_by(cloud_event_id=cloud_event_id, message_type=message_type).one_or_none() diff --git a/auth-api/src/auth_api/services/authorization.py b/auth-api/src/auth_api/services/authorization.py index 991a25adb5..e7f39e51ea 100644 --- a/auth-api/src/auth_api/services/authorization.py +++ b/auth-api/src/auth_api/services/authorization.py @@ -226,6 +226,7 @@ def check_auth(**kwargs): if product_code_in_jwt == 'ALL': # Product code for super admin service account (sbc-auth-admin) return + auth = None if business_identifier: auth = Authorization.get_user_authorizations_for_entity(business_identifier) elif org_identifier: diff --git a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py index df51d20c7d..7d27cf7c15 100644 --- a/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py +++ b/auth-api/src/auth_api/services/gcp_queue/gcp_queue.py @@ -89,7 +89,7 @@ def publisher(self): """Returns the publisher.""" if not self._publisher and self.credentials_pub: self._publisher = pubsub_v1.PublisherClient(credentials=self.credentials_pub) - else: + if not self._publisher: self._publisher = pubsub_v1.PublisherClient() return self._publisher diff --git a/auth-api/src/auth_api/services/membership.py b/auth-api/src/auth_api/services/membership.py index bd76951b6e..7df0b1b737 100644 --- a/auth-api/src/auth_api/services/membership.py +++ b/auth-api/src/auth_api/services/membership.py @@ -168,7 +168,7 @@ def send_notification_to_member(self, origin_url, notification_type): notification_type_for_mailer = '' data = {} if notification_type == NotificationType.ROLE_CHANGED.value: - notification_type_for_mailer = 'roleChangedNotification' + notification_type_for_mailer = QueueMessageTypes.ROLE_CHANGED_NOTIFICATION.value data = { 'accountId': org_id, 'emailAddresses': recipient, @@ -181,9 +181,9 @@ def send_notification_to_member(self, origin_url, notification_type): # TODO how to check properly if user is bceid user is_bceid_user = self._model.user.username.find('@bceid') > 0 if is_bceid_user: - notification_type_for_mailer = 'membershipApprovedNotificationForBceid' + notification_type_for_mailer = QueueMessageTypes.MEMBERSHIP_APPROVED_NOTIFICATION_FOR_BCEID.value else: - notification_type_for_mailer = 'membershipApprovedNotification' + notification_type_for_mailer = QueueMessageTypes.MEMBERSHIP_APPROVED_NOTIFICATION.value data = { 'accountId': org_id, diff --git a/auth-api/src/auth_api/services/org.py b/auth-api/src/auth_api/services/org.py index f92b499a75..5d83fa20ad 100644 --- a/auth-api/src/auth_api/services/org.py +++ b/auth-api/src/auth_api/services/org.py @@ -878,9 +878,9 @@ def send_approved_rejected_notification(receipt_admin_emails, org_name, org_id, current_app.logger.debug(' ValidatorResponse: OrgType.SBC_STAFF: non_ejv_payment_methods, OrgType.STAFF: non_ejv_payment_methods, } + payment_type = None if access_type == AccessType.GOVM.value: payment_type = PaymentMethod.EJV.value elif selected_payment_method: diff --git a/auth-api/src/auth_api/utils/custom_query.py b/auth-api/src/auth_api/utils/custom_query.py index fc6ed4f938..e4356e2b71 100644 --- a/auth-api/src/auth_api/utils/custom_query.py +++ b/auth-api/src/auth_api/utils/custom_query.py @@ -18,7 +18,7 @@ from sqlalchemy import String, func -class CustomQuery(BaseQuery): +class CustomQuery(BaseQuery): # pylint: disable=too-few-public-methods """Custom Query class to extend the base query class for helper functionality.""" def filter_conditionally(self, search_criteria, model_attribute, is_like: bool = False): diff --git a/auth-api/tests/unit/services/test_authorization.py b/auth-api/tests/unit/services/test_authorization.py index 4952359fc2..5d1e6f57fe 100644 --- a/auth-api/tests/unit/services/test_authorization.py +++ b/auth-api/tests/unit/services/test_authorization.py @@ -284,8 +284,8 @@ def test_check_auth_staff_path(session, monkeypatch, test_desc, test_expect, add 'test_desc,test_expect,additional_kwargs,is_org_member,is_entity_affiliated,product_code_in_jwt', [ ( - 'Test UnboundLocalError when no role checks provided in kwargs, and no org_id or business_identifier.', - pytest.raises(UnboundLocalError), {}, False, False, ProductCode.BUSINESS.value + 'Test 403 when no role checks provided in kwargs, and no org_id or business_identifier.', + pytest.raises(Forbidden), {}, False, False, ProductCode.BUSINESS.value ), ( 'Test OK when no role checks provided in kwargs, but has ALL product in jwt. (bypass all checks).', @@ -359,7 +359,7 @@ def test_check_auth_system_path(session, monkeypatch, test_desc, test_expect, ad 'test_desc,test_expect,additional_kwargs,is_org_member,is_entity_affiliated', [ ( - 'Test UnboundLocalError when no role checks provided in kwargs.', + 'Test UnboundLocalError (403) when no role checks provided in kwargs.', pytest.raises(UnboundLocalError), {}, False, False ), ( diff --git a/queue_services/account-mailer/requirements.txt b/queue_services/account-mailer/requirements.txt index 7f295d4a80..72554213e7 100644 --- a/queue_services/account-mailer/requirements.txt +++ b/queue_services/account-mailer/requirements.txt @@ -34,5 +34,5 @@ tornado==6.4 urllib3==1.26.18 zipp==3.18.1 -e git+https://github.com/bcgov/sbc-common-components.git#egg=sbc-common-components&subdirectory=python --e git+https://github.com/seeker25/sbc-auth.git@queue_upgrades#egg=auth-api&subdirectory=auth-api +-e git+https://github.com/bcgov/sbc-auth.git#egg=auth-api&subdirectory=auth-api git+https://github.com/daxiom/simple-cloudevent.py.git diff --git a/queue_services/account-mailer/src/account_mailer/__init__.py b/queue_services/account-mailer/src/account_mailer/__init__.py index b8b2acf7be..9b5d148e24 100644 --- a/queue_services/account-mailer/src/account_mailer/__init__.py +++ b/queue_services/account-mailer/src/account_mailer/__init__.py @@ -25,6 +25,7 @@ from auth_api.services.flags import flags from auth_api.services.gcp_queue import queue from auth_api.utils.cache import cache +from auth_api.utils.util_logging import setup_logging from flask import Flask from sentry_sdk.integrations.flask import FlaskIntegration @@ -32,6 +33,9 @@ from account_mailer.resources.worker import bp as worker_endpoint +setup_logging(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'logging.conf')) # important to do this first + + def register_endpoints(app: Flask): """Register endpoints with the flask application.""" # Allow base route to match with, and without a trailing slash diff --git a/queue_services/account-mailer/src/account_mailer/email_processors/pad_confirmation.py b/queue_services/account-mailer/src/account_mailer/email_processors/pad_confirmation.py index 3379a41ef2..f1b5234b33 100644 --- a/queue_services/account-mailer/src/account_mailer/email_processors/pad_confirmation.py +++ b/queue_services/account-mailer/src/account_mailer/email_processors/pad_confirmation.py @@ -69,6 +69,8 @@ def _get_admin_emails(username): admin_emails = admin_user.contacts[0].contact.email else: admin_emails = admin_user.email + else: + raise ValueError('Admin user not found, cannot determine email address.') return admin_emails, admin_name diff --git a/queue_services/account-mailer/src/account_mailer/email_templates/reset_passcode.html b/queue_services/account-mailer/src/account_mailer/email_templates/reset_passcode.html index 57b6bb7991..6b720a6fa2 100644 --- a/queue_services/account-mailer/src/account_mailer/email_templates/reset_passcode.html +++ b/queue_services/account-mailer/src/account_mailer/email_templates/reset_passcode.html @@ -6,7 +6,7 @@ ### To access the new application and get started on your online filing: -* Go to: [https://www.bcregistry.ca/business>](https://www.bcregistry.ca/business) +* Go to: [https://www.account.bcregistry.gov.bc.ca/](https://www.account.bcregistry.gov.bc.ca/) * If you have not yet accessed this website, and need to create a new account, select the "Create a BC Registries Account" button to start. Otherwise, please select the "Login" drop down in the upper right corner of the screen to login to your existing account. * Once logged into your new account, you can add your business on the manage businesses dashboard. Please select the "+ Add an Existing..." button on the right, and select "Business" from the dropdown selection. * Use your business incorporation number: {{ businessIdentifier }} and passcode, {{ passCode }} to link your business to your account. diff --git a/queue_services/account-mailer/src/account_mailer/enums.py b/queue_services/account-mailer/src/account_mailer/enums.py index cb71f8e1fc..5de5443768 100644 --- a/queue_services/account-mailer/src/account_mailer/enums.py +++ b/queue_services/account-mailer/src/account_mailer/enums.py @@ -43,9 +43,9 @@ class SubjectType(Enum): MEMBERSHIP_APPROVED_NOTIFICATION = '[BC Registries and Online Services] Welcome to the account {account_name}' MEMBERSHIP_APPROVED_NOTIFICATION_FOR_BCEID = '[BC Registries and Online Services] Welcome to the account ' \ '{account_name}' - NONBCSC_ORG_APPROVED_NOTIFICATION = '[BC Registries and Online Services] APPROVED Business Registry Account' - NONBCSC_ORG_REJECTED_NOTIFICATION = '[BC Registries and Online Services] YOUR ACTION REQUIRED: ' \ - 'Business Registry Account cannot be approved' + NON_BCSC_ORG_APPROVED_NOTIFICATION = '[BC Registries and Online Services] APPROVED Business Registry Account' + NON_BCSC_ORG_REJECTED_NOTIFICATION = '[BC Registries and Online Services] YOUR ACTION REQUIRED: ' \ + 'Business Registry Account cannot be approved' OTP_AUTHENTICATOR_RESET_NOTIFICATION = '[BC Registries and Online Services] Authenticator Has Been Reset' ROLE_CHANGED_NOTIFICATION = '[BC Registries and Online Services] Your Role Has Been Changed' STAFF_REVIEW_ACCOUNT = '[BC Registries and Online Services] An out of province account needs to be approved.' @@ -86,8 +86,8 @@ class TitleType(Enum): GOVM_MEMBER_INVITATION = 'Invitation to Join an Account at Business Registry' MEMBERSHIP_APPROVED_NOTIFICATION = 'Your Membership Has Been Approved' MEMBERSHIP_APPROVED_NOTIFICATION_FOR_BCEID = 'Your Membership Has Been Approved' - NONBCSC_ORG_APPROVED_NOTIFICATION = 'Your Membership Has Been Approved' - NONBCSC_ORG_REJECTED_NOTIFICATION = 'Your Membership Has Been Rejected' + NON_BCSC_ORG_APPROVED_NOTIFICATION = 'Your Membership Has Been Approved' + NON_BCSC_ORG_REJECTED_NOTIFICATION = 'Your Membership Has Been Rejected' OTP_AUTHENTICATOR_RESET_NOTIFICATION = 'Your Authenticator Has Been Reset' ROLE_CHANGED_NOTIFICATION = 'Your Role Has Been Changed' STAFF_REVIEW_ACCOUNT = 'Notification from Business Registry' @@ -130,8 +130,8 @@ class TemplateType(Enum): GOVM_MEMBER_INVITATION_TEMPLATE_NAME = 'govm_member_invitation_email' MEMBERSHIP_APPROVED_NOTIFICATION_TEMPLATE_NAME = 'membership_approved_notification_email' MEMBERSHIP_APPROVED_NOTIFICATION_FOR_BCEID_TEMPLATE_NAME = 'membership_approved_notification_email_for_bceid' - NONBCSC_ORG_APPROVED_NOTIFICATION_TEMPLATE_NAME = 'nonbcsc_org_approved_notification_email' - NONBCSC_ORG_REJECTED_NOTIFICATION_TEMPLATE_NAME = 'nonbcsc_org_rejected_notification_email' + NON_BCSC_ORG_APPROVED_NOTIFICATION_TEMPLATE_NAME = 'nonbcsc_org_approved_notification_email' + NON_BCSC_ORG_REJECTED_NOTIFICATION_TEMPLATE_NAME = 'nonbcsc_org_rejected_notification_email' OTP_AUTHENTICATOR_RESET_NOTIFICATION_TEMPLATE_NAME = 'otp_authenticator_reset_notification_email' ROLE_CHANGED_NOTIFICATION_TEMPLATE_NAME = 'role_changed_notification_email' STAFF_REVIEW_ACCOUNT_TEMPLATE_NAME = 'staff_review_account_email' diff --git a/queue_services/account-mailer/logging.conf b/queue_services/account-mailer/src/account_mailer/logging.conf similarity index 100% rename from queue_services/account-mailer/logging.conf rename to queue_services/account-mailer/src/account_mailer/logging.conf diff --git a/queue_services/account-mailer/src/account_mailer/resources/worker.py b/queue_services/account-mailer/src/account_mailer/resources/worker.py index 0c9ef47399..545b178fca 100644 --- a/queue_services/account-mailer/src/account_mailer/resources/worker.py +++ b/queue_services/account-mailer/src/account_mailer/resources/worker.py @@ -14,9 +14,11 @@ """The unique worker functionality for this service is contained here.""" import dataclasses import json -from datetime import datetime +from datetime import datetime, timezone from http import HTTPStatus +from auth_api.models import db +from auth_api.models.pubsub_message_processing import PubSubMessageProcessing from auth_api.services.gcp_queue import queue from auth_api.services.gcp_queue.gcp_auth import ensure_authorized_queue_user from auth_api.services.rest_service import RestService @@ -44,7 +46,10 @@ def worker(): return {}, HTTPStatus.OK try: - current_app.logger.info('Event Message Received: %s', json.dumps(dataclasses.asdict(event_message))) + current_app.logger.info('Event message received: %s', json.dumps(dataclasses.asdict(event_message))) + if is_message_processed(event_message): + current_app.logger.info('Event message already processed, skipping.') + return {}, HTTPStatus.OK message_type, email_msg = event_message.type, event_message.data email_msg['logo_url'] = minio_service.MinioService.get_minio_public_url('bc_logo_for_email.png') @@ -72,6 +77,19 @@ def worker(): return {}, HTTPStatus.OK +def is_message_processed(event_message): + """Check if the queue message is processed.""" + if PubSubMessageProcessing.find_by_cloud_event_id_and_type(event_message.id, event_message.type): + return True + pubsub_message_processing = PubSubMessageProcessing() + pubsub_message_processing.cloud_event_id = event_message.id + pubsub_message_processing.message_type = event_message.type + pubsub_message_processing.processed = datetime.now(timezone.utc) + db.session.add(pubsub_message_processing) + db.session.commit() + return False + + def handle_drawdown_request(message_type, email_msg): """Handle the drawdown request message.""" if message_type != QueueMessageTypes.REFUND_DRAWDOWN_REQUEST.value: @@ -443,6 +461,7 @@ def handle_other_messages(message_type, email_msg): ) template_name = TemplateType[f'{QueueMessageTypes(message_type).name}_TEMPLATE_NAME'].value else: + current_app.logger.error('Unknown message type: %s', message_type) return kwargs = { diff --git a/queue_services/account-mailer/tests/unit/test_worker_queue.py b/queue_services/account-mailer/tests/unit/test_worker_queue.py index 5673837af3..a61859e394 100644 --- a/queue_services/account-mailer/tests/unit/test_worker_queue.py +++ b/queue_services/account-mailer/tests/unit/test_worker_queue.py @@ -45,6 +45,32 @@ def test_refund_request(app, session, client): mock_send.assert_called +def test_duplicate_messages(app, session, client): + """Assert that duplicate messages are handled by the queue..""" + user = factory_user_model_with_contact() + org = factory_org_model() + factory_membership_model(user.id, org.id) + id = org.id + mail_details = { + 'accountId': id, + 'accountName': org.name + } + + with patch.object(notification_service, 'send_email', return_value=None) as mock_send: + helper_add_event_to_queue(client, message_type=QueueMessageTypes.NSF_LOCK_ACCOUNT.value, + mail_details=mail_details, message_id='f76e5ca9-93f3-44ee-a0f8-f47ee83b1971') + mock_send.assert_called + assert mock_send.call_args.args[0].get('recipients') == 'foo@bar.com' + assert mock_send.call_args.args[0].get('content').get('subject') == SubjectType.NSF_LOCK_ACCOUNT_SUBJECT.value + assert mock_send.call_args.args[0].get('attachments') is None + assert True + + with patch.object(notification_service, 'send_email', return_value=None) as mock_send: + helper_add_event_to_queue(client, message_type=QueueMessageTypes.NSF_LOCK_ACCOUNT.value, + mail_details=mail_details, message_id='f76e5ca9-93f3-44ee-a0f8-f47ee83b1971') + mock_send.assert_not_called + + def test_lock_account_mailer_queue(app, session, client): """Assert that events can be retrieved and decoded from the Queue.""" user = factory_user_model_with_contact() diff --git a/queue_services/account-mailer/tests/unit/utils.py b/queue_services/account-mailer/tests/unit/utils.py index fc555fbcaa..e470ef24d4 100644 --- a/queue_services/account-mailer/tests/unit/utils.py +++ b/queue_services/account-mailer/tests/unit/utils.py @@ -20,10 +20,10 @@ from simple_cloudevent import SimpleCloudEvent, to_queue_message -def build_request_for_queue_push(message_type, payload): +def build_request_for_queue_push(message_type, payload, message_id=None): """Build request for queue message.""" queue_message_bytes = to_queue_message(SimpleCloudEvent( - id=str(uuid.uuid4()), + id=str(message_id if message_id else uuid.uuid4()), source='account-mailer', subject=None, time=datetime.now(tz=timezone.utc).isoformat(), @@ -46,10 +46,10 @@ def post_to_queue(client, request_payload): assert response.status_code == 200 -def helper_add_event_to_queue(client, message_type: str, mail_details: dict): +def helper_add_event_to_queue(client, message_type: str, mail_details: dict, message_id=None): """Add event to the Queue.""" if not mail_details: mail_details = { } - payload = build_request_for_queue_push(message_type, mail_details) + payload = build_request_for_queue_push(message_type, mail_details, message_id) post_to_queue(client, payload) diff --git a/queue_services/auth-queue/requirements.txt b/queue_services/auth-queue/requirements.txt index 5e91c09030..ef6134f0a3 100644 --- a/queue_services/auth-queue/requirements.txt +++ b/queue_services/auth-queue/requirements.txt @@ -47,5 +47,5 @@ tornado==6.4 urllib3==1.26.18 zipp==3.18.1 -e git+https://github.com/bcgov/sbc-common-components.git#egg=sbc-common-components&subdirectory=python --e git+https://github.com/seeker25/sbc-auth.git@queue_upgrades#egg=auth-api&subdirectory=auth-api +-e git+https://github.com/bcgov/sbc-auth.git#egg=auth-api&subdirectory=auth-api git+https://github.com/daxiom/simple-cloudevent.py.git diff --git a/queue_services/auth-queue/src/auth_queue/__init__.py b/queue_services/auth-queue/src/auth_queue/__init__.py index 9668577ea8..597218c2ea 100644 --- a/queue_services/auth-queue/src/auth_queue/__init__.py +++ b/queue_services/auth-queue/src/auth_queue/__init__.py @@ -20,6 +20,7 @@ from auth_api.services.flags import flags from auth_api.services.gcp_queue import queue from auth_api.utils.cache import cache +from auth_api.utils.util_logging import setup_logging from flask import Flask from sentry_sdk.integrations.flask import FlaskIntegration @@ -27,6 +28,9 @@ from auth_queue.resources.worker import bp as worker_endpoint +setup_logging(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'logging.conf')) # important to do this first + + def register_endpoints(app: Flask): """Register endpoints with the flask application.""" # Allow base route to match with, and without a trailing slash diff --git a/queue_services/auth-queue/logging.conf b/queue_services/auth-queue/src/auth_queue/logging.conf similarity index 100% rename from queue_services/auth-queue/logging.conf rename to queue_services/auth-queue/src/auth_queue/logging.conf diff --git a/queue_services/auth-queue/src/auth_queue/resources/worker.py b/queue_services/auth-queue/src/auth_queue/resources/worker.py index ac3fd351fa..053bd6e5d4 100644 --- a/queue_services/auth-queue/src/auth_queue/resources/worker.py +++ b/queue_services/auth-queue/src/auth_queue/resources/worker.py @@ -14,7 +14,7 @@ """The unique worker functionality for this service is contained here.""" import dataclasses import json -from datetime import datetime +from datetime import datetime, timezone from http import HTTPStatus from auth_api.models import ActivityLog as ActivityLogModel @@ -22,6 +22,7 @@ from auth_api.models import Entity as EntityModel from auth_api.models import Org as OrgModel from auth_api.models import db +from auth_api.models.pubsub_message_processing import PubSubMessageProcessing from auth_api.services.gcp_queue import queue from auth_api.services.gcp_queue.gcp_auth import ensure_authorized_queue_user from auth_api.services.rest_service import RestService @@ -45,7 +46,10 @@ def worker(): return {}, HTTPStatus.OK try: - current_app.logger.info('Event Message Received: %s', json.dumps(dataclasses.asdict(event_message))) + current_app.logger.info('Event message received: %s', json.dumps(dataclasses.asdict(event_message))) + if is_message_processed(event_message): + current_app.logger.info('Event message already processed, skipping.') + return {}, HTTPStatus.OK if event_message.type == QueueMessageTypes.NAMES_EVENT.value: process_name_events(event_message) elif event_message.type == QueueMessageTypes.ACTIVITY_LOG.value: @@ -59,6 +63,19 @@ def worker(): return {}, HTTPStatus.OK +def is_message_processed(event_message): + """Check if the queue message is processed.""" + if PubSubMessageProcessing.find_by_cloud_event_id_and_type(event_message.id, event_message.type): + return True + pubsub_message_processing = PubSubMessageProcessing() + pubsub_message_processing.cloud_event_id = event_message.id + pubsub_message_processing.message_type = event_message.type + pubsub_message_processing.processed = datetime.now(timezone.utc) + db.session.add(pubsub_message_processing) + db.session.commit() + return False + + def process_activity_log(data): """Process activity log events.""" current_app.logger.debug('>>>>>>>process_activity_log>>>>>') From 06c8641def09bc076ba4313ca7a6649f6a9e3554 Mon Sep 17 00:00:00 2001 From: Karim El Jazzar <122301442+JazzarKarim@users.noreply.github.com> Date: Wed, 22 May 2024 16:32:31 -0700 Subject: [PATCH 4/7] 19026 - Implemented the number of businesses ready for D1 dissolution (#2831) * 19026 - Implemented the number of businesses ready for D1 dissolution * Updated the endpoint url * added fallback * removed fallback * Added spinner while waiting to grab the number of businesses ready for D1 Dissolution * removed unnecessary line * Updated as per my conversation with Travis * Changed in response to conversation with Travis part 2 --- auth-web/package-lock.json | 4 +- auth-web/package.json | 2 +- .../auth/staff/DissolutionSchedule.vue | 6 +-- auth-web/src/models/Staff.ts | 4 ++ auth-web/src/services/staff.services.ts | 6 ++- auth-web/src/stores/staff.ts | 18 ++++++--- auth-web/src/util/constants.ts | 7 +++- .../auth/staff/InvoluntaryDissolution.vue | 38 +++++++++++++++---- 8 files changed, 62 insertions(+), 23 deletions(-) diff --git a/auth-web/package-lock.json b/auth-web/package-lock.json index 8bb4f638a5..a6fb1dc8f3 100644 --- a/auth-web/package-lock.json +++ b/auth-web/package-lock.json @@ -1,12 +1,12 @@ { "name": "auth-web", - "version": "2.6.12", + "version": "2.6.13", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "auth-web", - "version": "2.6.12", + "version": "2.6.13", "dependencies": { "@bcrs-shared-components/base-address": "2.0.3", "@bcrs-shared-components/bread-crumb": "1.0.8", diff --git a/auth-web/package.json b/auth-web/package.json index 572a4c1620..723d4d3544 100644 --- a/auth-web/package.json +++ b/auth-web/package.json @@ -1,6 +1,6 @@ { "name": "auth-web", - "version": "2.6.12", + "version": "2.6.13", "appName": "Auth Web", "sbcName": "SBC Common Components", "private": true, diff --git a/auth-web/src/components/auth/staff/DissolutionSchedule.vue b/auth-web/src/components/auth/staff/DissolutionSchedule.vue index 8fb5823cd6..38a7a0f949 100644 --- a/auth-web/src/components/auth/staff/DissolutionSchedule.vue +++ b/auth-web/src/components/auth/staff/DissolutionSchedule.vue @@ -121,8 +121,8 @@ export default defineComponent({ setup () { const state = reactive({ menu: false, - numberOfBusinessesEdit: 0, - numberOfBusinessesNonEdit: 0, + numberOfBusinessesEdit: -1, + numberOfBusinessesNonEdit: -1, numberOfBusinessesRef: null, isEdit: false, isSaving: false @@ -135,7 +135,7 @@ export default defineComponent({ await staffStore.getDissolutionConfigurations() // Get the batch size current value (number of businesses to be dissolved per job run) - const numDissolutions = staffStore.involuntaryDissolutionConfigurations.configurations.find( + const numDissolutions = staffStore.involuntaryDissolutionConfigurations?.configurations?.find( config => config.name === 'NUM_DISSOLUTIONS_ALLOWED').value state.numberOfBusinessesNonEdit = parseInt(numDissolutions) }) diff --git a/auth-web/src/models/Staff.ts b/auth-web/src/models/Staff.ts index 4404c7818e..140703106b 100644 --- a/auth-web/src/models/Staff.ts +++ b/auth-web/src/models/Staff.ts @@ -71,3 +71,7 @@ export interface InvoluntaryDissolutionConfigurationIF { export interface Configurations { configurations: InvoluntaryDissolutionConfigurationIF[] } + +export interface DissolutionStatistics { + data: { eligibleCount: number } +} diff --git a/auth-web/src/services/staff.services.ts b/auth-web/src/services/staff.services.ts index e1fbe421c8..46f2e99c1e 100644 --- a/auth-web/src/services/staff.services.ts +++ b/auth-web/src/services/staff.services.ts @@ -1,4 +1,4 @@ -import { AccountType, Configurations, ProductCode, Products, ProductsRequestBody } from '@/models/Staff' +import { AccountType, Configurations, DissolutionStatistics, ProductCode, Products, ProductsRequestBody } from '@/models/Staff' import { OrgFilterParams, OrgList, Organizations } from '@/models/Organization' import { AxiosResponse } from 'axios' import ConfigHelper from '@/util/config-helper' @@ -53,6 +53,10 @@ export default class StaffService { return axios.get(`${ConfigHelper.getLegalAPIV2Url()}/admin/configurations`) } + static async getDissolutionStatistics (): Promise> { + return axios.get(`${ConfigHelper.getLegalAPIV2Url()}/admin/dissolutions/statistics`) + } + static async updateInvoluntaryDissolutionConfigurations (configurations: Configurations): Promise> { return axios.put(`${ConfigHelper.getLegalAPIV2Url()}/admin/configurations`, configurations) } diff --git a/auth-web/src/stores/staff.ts b/auth-web/src/stores/staff.ts index d7666dd8a9..94e1d38c63 100644 --- a/auth-web/src/stores/staff.ts +++ b/auth-web/src/stores/staff.ts @@ -1,5 +1,5 @@ import { AccountStatus, AffidavitStatus, TaskAction, TaskRelationshipStatus, TaskRelationshipType, TaskType } from '@/util/constants' -import { AccountType, Configurations, GLCode, ProductCode } from '@/models/Staff' +import { AccountType, Configurations, DissolutionStatistics, GLCode, ProductCode } from '@/models/Staff' import { MembershipType, OrgFilterParams, Organization } from '@/models/Organization' import { SyncAccountPayload, Task } from '@/models/Task' import { computed, reactive, toRefs } from '@vue/composition-api' @@ -25,6 +25,7 @@ export const useStaffStore = defineStore('staff', () => { accountUnderReviewAdminContact: {} as Contact, accountUnderReviewAffidavitInfo: {} as AffidavitInformation, activeStaffOrgs: [] as Organization[], + dissolutionStatistics: {} as DissolutionStatistics, involuntaryDissolutionConfigurations: {} as Configurations, pendingInvitationOrgs: [] as Organization[], pendingStaffOrgs: [] as Organization[], @@ -42,6 +43,7 @@ export const useStaffStore = defineStore('staff', () => { state.accountUnderReviewAdminContact = {} as Contact state.accountUnderReviewAffidavitInfo = {} as AffidavitInformation state.activeStaffOrgs = [] as Organization[] + state.dissolutionStatistics = {} as DissolutionStatistics state.involuntaryDissolutionConfigurations = {} as Configurations state.pendingInvitationOrgs = [] as Organization[] state.pendingStaffOrgs = [] as Organization[] @@ -81,12 +83,15 @@ export const useStaffStore = defineStore('staff', () => { } /** Get the involuntary dissolution configurations array from Legal API. */ - async function getDissolutionConfigurations (): Promise { + async function getDissolutionConfigurations (): Promise { const response = await StaffService.getInvoluntaryDissolutionConfigurations() - if (response?.data && response.status === 200) { - state.involuntaryDissolutionConfigurations = response.data - return response.data - } + if (response?.data && response.status === 200) state.involuntaryDissolutionConfigurations = response.data + } + + /** Get the businesses dissolution statistics (businesses ready for D1 dissolution). */ + async function getDissolutionStatistics (): Promise { + const response = await StaffService.getDissolutionStatistics() + if (response?.data && response.status === 200) state.dissolutionStatistics = response.data } async function getAccountTypes (): Promise { @@ -307,6 +312,7 @@ export const useStaffStore = defineStore('staff', () => { deleteOrg, getAccountTypes, getDissolutionConfigurations, + getDissolutionStatistics, getGLCode, getGLCodeList, getGLCodeFiling, diff --git a/auth-web/src/util/constants.ts b/auth-web/src/util/constants.ts index e315f1858f..bbfd5d5485 100644 --- a/auth-web/src/util/constants.ts +++ b/auth-web/src/util/constants.ts @@ -660,7 +660,10 @@ export enum CfsAccountStatus { export enum InvoluntaryDissolutionConfigNames { DISSOLUTIONS_ON_HOLD = 'DISSOLUTIONS_ON_HOLD', + DISSOLUTIONS_STAGE_1_SCHEDULE='DISSOLUTIONS_STAGE_1_SCHEDULE', + DISSOLUTIONS_STAGE_2_SCHEDULE='DISSOLUTIONS_STAGE_2_SCHEDULE', + DISSOLUTIONS_STAGE_3_SCHEDULE='DISSOLUTIONS_STAGE_3_SCHEDULE', + DISSOLUTIONS_SUMMARY_EMAIL = 'DISSOLUTIONS_SUMMARY_EMAIL', MAX_DISSOLUTIONS_ALLOWED = 'MAX_DISSOLUTIONS_ALLOWED', - NUM_DISSOLUTIONS_ALLOWED = 'NUM_DISSOLUTIONS_ALLOWED', - NEW_DISSOLUTIONS_SCHEDULE = 'NEW_DISSOLUTIONS_SCHEDULE' + NUM_DISSOLUTIONS_ALLOWED = 'NUM_DISSOLUTIONS_ALLOWED' } diff --git a/auth-web/src/views/auth/staff/InvoluntaryDissolution.vue b/auth-web/src/views/auth/staff/InvoluntaryDissolution.vue index e9ee7f59ae..d13c0d0cfb 100644 --- a/auth-web/src/views/auth/staff/InvoluntaryDissolution.vue +++ b/auth-web/src/views/auth/staff/InvoluntaryDissolution.vue @@ -3,6 +3,20 @@ id="involuntary-dissolution" class="view-container" > + + +
+ +
+

Staff Involuntary Dissolution Batch @@ -55,9 +69,10 @@