From fdfb0fe6c5783f9f6e49f88caeeb9acec5b5de02 Mon Sep 17 00:00:00 2001 From: Doug Lovett Date: Tue, 9 Jul 2024 08:44:40 -0700 Subject: [PATCH] Add job id to PPR batch discharges job API endpoint, update SE amendment report. (#1971) Signed-off-by: Doug Lovett --- ppr-api/src/ppr_api/models/mail_report.py | 50 ++++++++++++++++++- .../src/ppr_api/reports/v2/report_utils.py | 30 ++++++++++- ppr-api/src/ppr_api/resources/v1/callbacks.py | 4 +- ppr-api/src/ppr_api/version.py | 2 +- ppr-api/tests/unit/api/test_callback.py | 5 ++ ppr-api/tests/unit/models/test_mail_report.py | 45 +++++++++++++---- 6 files changed, 120 insertions(+), 16 deletions(-) diff --git a/ppr-api/src/ppr_api/models/mail_report.py b/ppr-api/src/ppr_api/models/mail_report.py index 1f27cafba..26455aa1c 100644 --- a/ppr-api/src/ppr_api/models/mail_report.py +++ b/ppr-api/src/ppr_api/models/mail_report.py @@ -34,6 +34,16 @@ WHERE create_ts BETWEEN :query_start AND :query_end AND doc_storage_url IS NOT NULL """ +UPDATE_BATCH_JOB_DATE_START = """ +UPDATE mail_reports SET batch_job_id = :job_id + WHERE create_ts > :query_start + AND doc_storage_url IS NOT NULL +""" +UPDATE_BATCH_JOB_DATE_RANGE = """ +UPDATE mail_reports SET batch_job_id = :job_id + WHERE create_ts BETWEEN :query_start AND :query_end + AND doc_storage_url IS NOT NULL +""" class MailReport(db.Model): @@ -48,6 +58,7 @@ class MailReport(db.Model): retry_count = db.mapped_column('retry_count', db.Integer, nullable=True) status = db.mapped_column('status', db.Integer, nullable=True) message = db.mapped_column('message', db.String(2000), nullable=True) + batch_job_id = db.mapped_column('batch_job_id', db.Integer, nullable=True) # parent keys registration_id = db.mapped_column('registration_id', db.Integer, db.ForeignKey('registrations.id'), @@ -69,7 +80,8 @@ def json(self) -> dict: 'documentStorageURL': self.doc_storage_url if self.doc_storage_url else '', 'retryCount': self.retry_count if self.retry_count else 0, 'status': self.status if self.status else 0, - 'message': self.message if self.message else '' + 'message': self.message if self.message else '', + 'jobId': self.batch_job_id if self.batch_job_id else 0 } return result @@ -122,8 +134,16 @@ def find_by_registration_party_id(cls, registration_id: int, party_id: int): return mail_report @classmethod - def find_list_by_timestamp(cls, start: datetime, end: datetime): + def find_list_by_timestamp(cls, start: datetime, end: datetime, job_id=None): """Return a list of mail reports that matches the start and optional end timestamps.""" + batch_job_id: int = 0 + if job_id and isinstance(job_id, str): + try: + batch_job_id = int(job_id) + except Exception: # noqa: B902; ignore if invalid. + current_app.logger.error(f'DB find_list_by_timestamp invalid job_id={job_id}: ignoring.') + elif job_id: + batch_job_id = job_id try: results = [] if not start: @@ -143,8 +163,34 @@ def find_list_by_timestamp(cls, start: datetime, end: datetime): 'dateTime': model_utils.format_ts(row[1]), 'docStorageRef': str(row[2]) } + if batch_job_id > 0: + result['jobId'] = batch_job_id results.append(result) + if results and batch_job_id > 0: # Capture job id + cls.save_job_id(start, end, batch_job_id) return results except Exception as db_exception: # noqa: B902; return nicer error current_app.logger.error('DB find_list_by_timestamp exception: ' + str(db_exception)) raise DatabaseException(db_exception) + + @classmethod + def save_job_id(cls, start: datetime, end: datetime, batch_job_id: int): + """If a job id was submitted then save it for the records included in the previous results timestamp range.""" + if not batch_job_id or batch_job_id < 1: + current_app.logger.info('No batch_job_id: skipping update.') + return + if not start: + current_app.logger.info('No start timestamp: skipping update.') + return + + try: + if not end: + update_statement = text(UPDATE_BATCH_JOB_DATE_START) + db.session.execute(update_statement, {'job_id': batch_job_id, 'query_start': start}) + else: + update_statement = text(UPDATE_BATCH_JOB_DATE_RANGE) + db.session.execute(update_statement, {'job_id': batch_job_id, 'query_start': start, 'query_end': end}) + db.session.commit() + except Exception as db_exception: # noqa: B902; return nicer error + current_app.logger.error('DB save_job_id exception: ' + str(db_exception)) + raise DatabaseException(db_exception) diff --git a/ppr-api/src/ppr_api/reports/v2/report_utils.py b/ppr-api/src/ppr_api/reports/v2/report_utils.py index f39d28e4d..dc5139a9b 100755 --- a/ppr-api/src/ppr_api/reports/v2/report_utils.py +++ b/ppr-api/src/ppr_api/reports/v2/report_utils.py @@ -684,7 +684,27 @@ def is_order_unchanged(order1: dict, order2: dict) -> bool: (order1.get('effectOfOrder') == order2.get('effectOfOrder')) -def set_modified_order(add_notice: dict, del_notice: dict): +def order_exists(order: dict, del_notice: dict) -> bool: + """Determine if order in added Securities Act Notice is unchanged.""" + if not order or not del_notice: + return False + for del_order in del_notice.get('securitiesActOrders'): + if is_order_unchanged(order, del_order): + return True + return False + + +def order_deleted(del_order: dict, add_notice: dict) -> bool: + """Determine if a order in a deleted Securities Act Notice is deleted and not amended/edited.""" + for add_order in add_notice.get('securitiesActOrders'): + if add_order.get('amendOrderId') and add_order.get('amendOrderId') == del_order.get('orderId'): + return False + if is_order_unchanged(del_order, add_order): + return False + return True + + +def set_modified_order(add_notice: dict, del_notice: dict): # pylint: disable=too-many-branches; 1 more """Conditionally set amended notice order unchanged flag.""" for add_order in add_notice.get('securitiesActOrders'): if add_order.get('amendOrderId'): @@ -693,8 +713,16 @@ def set_modified_order(add_notice: dict, del_notice: dict): is_order_unchanged(add_order, del_order): # If identical mark as unchanged add_order['unchanged'] = True + # Check if an identical order exists that is not linked by the UI + elif order_exists(add_order, del_notice): + add_order['unchanged'] = True # Amended notice deleted orders added to new orders but marked as deleted. merged_orders = [] + for del_order in del_notice.get('securitiesActOrders'): + if order_deleted(del_order, add_notice): + order = copy.deepcopy(del_order) + order['amendDeleted'] = True + merged_orders.append(order) for add_order in add_notice.get('securitiesActOrders'): if add_order.get('amendOrderId'): for del_order in del_notice.get('securitiesActOrders'): diff --git a/ppr-api/src/ppr_api/resources/v1/callbacks.py b/ppr-api/src/ppr_api/resources/v1/callbacks.py index ff5417dd7..d20e8d029 100644 --- a/ppr-api/src/ppr_api/resources/v1/callbacks.py +++ b/ppr-api/src/ppr_api/resources/v1/callbacks.py @@ -31,6 +31,7 @@ url_prefix='/api/v1/callbacks') START_TS_PARAM = 'startDateTime' END_TS_PARAM = 'endDateTime' +JOB_ID_PARAM = 'jobId' @bp.route('/mail-report', methods=['POST', 'OPTIONS']) @@ -82,6 +83,7 @@ def get_mail_list(): """Fetch recent event storage names by request parameter startDateTime and optional endDateTime.""" start_ts = request.args.get(START_TS_PARAM, None) end_ts = request.args.get(END_TS_PARAM, None) + job_id = request.args.get(JOB_ID_PARAM, None) try: # Authenticate with request api key if not resource_utils.valid_api_key(request): @@ -108,7 +110,7 @@ def get_mail_list(): if message: return resource_utils.error_response(status, message) - results = MailReport.find_list_by_timestamp(start, end) + results = MailReport.find_list_by_timestamp(start, end, job_id) return jsonify(results), HTTPStatus.OK except DatabaseException as db_exception: return resource_utils.db_exception_response(db_exception, diff --git a/ppr-api/src/ppr_api/version.py b/ppr-api/src/ppr_api/version.py index c91026d94..1e27e0aa0 100644 --- a/ppr-api/src/ppr_api/version.py +++ b/ppr-api/src/ppr_api/version.py @@ -22,4 +22,4 @@ Development release segment: .devN """ -__version__ = '1.2.5' # pylint: disable=invalid-name +__version__ = '1.2.6' # pylint: disable=invalid-name diff --git a/ppr-api/tests/unit/api/test_callback.py b/ppr-api/tests/unit/api/test_callback.py index 519a39838..31c0a6480 100644 --- a/ppr-api/tests/unit/api/test_callback.py +++ b/ppr-api/tests/unit/api/test_callback.py @@ -48,6 +48,7 @@ ('Invalid range', HTTPStatus.BAD_REQUEST, '2023-01-31T00:00:01-08:00', '2023-01-30T00:00:01-08:00'), ('Invalid end ts', HTTPStatus.BAD_REQUEST, None, '2023-01-31TXX:00:01-08:00'), ('Valid start', HTTPStatus.OK, '2023-01-31T00:00:01-08:00', None), + ('Valid start with job id', HTTPStatus.OK, '2023-01-31T00:00:01-08:00', None), ('Unauthorized', HTTPStatus.UNAUTHORIZED, None, None) ] @@ -92,6 +93,8 @@ def test_list_mail_report(session, client, jwt, desc, status, start_ts, end_ts): params += f'&endDateTime={end_ts}' elif end_ts: params += f'?endDateTime={end_ts}' + if desc == 'Valid start with job id': + params += '&jobId=1234' headers = None if status != HTTPStatus.UNAUTHORIZED: @@ -112,3 +115,5 @@ def test_list_mail_report(session, client, jwt, desc, status, start_ts, end_ts): assert result.get('id') assert result.get('dateTime') assert result.get('docStorageRef') + if desc == 'Valid start with job id': + assert result.get('jobId') diff --git a/ppr-api/tests/unit/models/test_mail_report.py b/ppr-api/tests/unit/models/test_mail_report.py index 20e8345d0..7ad900905 100644 --- a/ppr-api/tests/unit/models/test_mail_report.py +++ b/ppr-api/tests/unit/models/test_mail_report.py @@ -33,11 +33,17 @@ (True, 200000000, 200000004, 200000013, True, False), (True, 200000002, 200000008, 200000023, False, True) ] -# testdata pattern is ({desc}, {has_data}, {start_ts}, {end_ts}) +# testdata pattern is ({desc}, {has_data}, {start_ts}, {end_ts}, job_id) TEST_MAIL_LIST_DATA = [ - ('Valid start', True, '2023-02-08T00:00:01-08:00', None), - ('Valid range', True, '2023-02-08T00:00:01-08:00', None), - ('Valid range no data', False, '2023-02-01T00:00:01-08:00', '2023-02-03T00:00:01-08:00') + ('Valid start', True, '2023-02-08T00:00:01-08:00', None, None), + ('Valid range', True, '2023-02-08T00:00:01-08:00', None, None), + ('Valid range with job id', True, '2023-02-08T00:00:01-08:00', None, 1234), + ('Valid range no data', False, '2023-02-01T00:00:01-08:00', '2023-02-03T00:00:01-08:00', None) +] +# testdata pattern is ({desc}, {start_ts}, job_id) +TEST_MAIL_LIST_JOB_DATA = [ + ('Valid start', '2023-02-08T00:00:01-08:00', 1234), + ('Valid range', '2023-02-08T00:00:01-08:00', 1234) ] @@ -143,7 +149,8 @@ def test_mail_report_json(session): party_id=3000, report_data=json.dumps(TEST_REPORT_DATA), doc_storage_url='http%3A%2F%2Fmocktarget.apigee.net', - status=200 + status=200, + batch_job_id=1234 ) report_json = { 'id': mail_report.id, @@ -153,28 +160,44 @@ def test_mail_report_json(session): 'reportData': mail_report.report_data, 'documentStorageURL': mail_report.doc_storage_url, 'retryCount': 0, - 'status': 200, - 'message': '' + 'status': mail_report.status, + 'message': '', + 'jobId': mail_report.batch_job_id } assert mail_report.json == report_json -@pytest.mark.parametrize('desc,has_data,start_ts,end_ts', TEST_MAIL_LIST_DATA) -def test_find_list_by_timestamp(session, desc, has_data, start_ts, end_ts): +@pytest.mark.parametrize('desc,has_data,start_ts,end_ts,job_id', TEST_MAIL_LIST_DATA) +def test_find_list_by_timestamp(session, desc, has_data, start_ts, end_ts, job_id): start = None end = None if start_ts: start = ts_from_iso_format(start_ts) - if desc == 'Valid range': + if desc in ('Valid range', 'Valid range with job id'): end = now_ts() elif end_ts: end = ts_from_iso_format(end_ts) - list_json = MailReport.find_list_by_timestamp(start, end) + list_json = MailReport.find_list_by_timestamp(start, end, job_id) if has_data: assert list_json for result in list_json: assert result.get('id') assert result.get('dateTime') assert result.get('docStorageRef') + if job_id and job_id > 0: + assert result.get('jobId') == job_id + else: + assert 'jobId' not in result else: assert not list_json + + +@pytest.mark.parametrize('desc,start_ts,job_id', TEST_MAIL_LIST_JOB_DATA) +def test_list_job_update(session, desc, start_ts, job_id): + start = None + end = None + if start_ts: + start = ts_from_iso_format(start_ts) + if desc == 'Valid range': + end = now_ts() + MailReport.save_job_id(start, end, job_id)