Add job id to PPR batch discharges job API endpoint, update SE amendm…

…ent report. (#1971)

Signed-off-by: Doug Lovett <[email protected]>

ppr-api/src/ppr_api/models/mail_report.py

@@ -34,6 +34,16 @@
  WHERE create_ts BETWEEN :query_start AND :query_end
    AND doc_storage_url IS NOT NULL
 """
+UPDATE_BATCH_JOB_DATE_START = """
+UPDATE mail_reports SET batch_job_id = :job_id
+ WHERE create_ts > :query_start
+   AND doc_storage_url IS NOT NULL
+"""
+UPDATE_BATCH_JOB_DATE_RANGE = """
+UPDATE mail_reports SET batch_job_id = :job_id
+ WHERE create_ts BETWEEN :query_start AND :query_end
+   AND doc_storage_url IS NOT NULL
+"""
 
 
 class MailReport(db.Model):
@@ -48,6 +58,7 @@ class MailReport(db.Model):
     retry_count = db.mapped_column('retry_count', db.Integer, nullable=True)
     status = db.mapped_column('status', db.Integer, nullable=True)
     message = db.mapped_column('message', db.String(2000), nullable=True)
+    batch_job_id = db.mapped_column('batch_job_id', db.Integer, nullable=True)
 
     # parent keys
     registration_id = db.mapped_column('registration_id', db.Integer, db.ForeignKey('registrations.id'),
@@ -69,7 +80,8 @@ def json(self) -> dict:
             'documentStorageURL': self.doc_storage_url if self.doc_storage_url else '',
             'retryCount': self.retry_count if self.retry_count else 0,
             'status': self.status if self.status else 0,
-            'message': self.message if self.message else ''
+            'message': self.message if self.message else '',
+            'jobId': self.batch_job_id if self.batch_job_id else 0
         }
         return result
 
@@ -122,8 +134,16 @@ def find_by_registration_party_id(cls, registration_id: int, party_id: int):
         return mail_report
 
     @classmethod
-    def find_list_by_timestamp(cls, start: datetime, end: datetime):
+    def find_list_by_timestamp(cls, start: datetime, end: datetime, job_id=None):
         """Return a list of mail reports that matches the start and optional end timestamps."""
+        batch_job_id: int = 0
+        if job_id and isinstance(job_id, str):
+            try:
+                batch_job_id = int(job_id)
+            except Exception:   # noqa: B902; ignore if invalid.
+                current_app.logger.error(f'DB find_list_by_timestamp invalid job_id={job_id}: ignoring.')
+        elif job_id:
+            batch_job_id = job_id
         try:
             results = []
             if not start:
@@ -143,8 +163,34 @@ def find_list_by_timestamp(cls, start: datetime, end: datetime):
                         'dateTime': model_utils.format_ts(row[1]),
                         'docStorageRef': str(row[2])
                     }
+                    if batch_job_id > 0:
+                        result['jobId'] = batch_job_id
                     results.append(result)
+            if results and batch_job_id > 0:  # Capture job id
+                cls.save_job_id(start, end, batch_job_id)
             return results
         except Exception as db_exception:   # noqa: B902; return nicer error
             current_app.logger.error('DB find_list_by_timestamp exception: ' + str(db_exception))
             raise DatabaseException(db_exception)
+
+    @classmethod
+    def save_job_id(cls, start: datetime, end: datetime, batch_job_id: int):
+        """If a job id was submitted then save it for the records included in the previous results timestamp range."""
+        if not batch_job_id or batch_job_id < 1:
+            current_app.logger.info('No batch_job_id: skipping update.')
+            return
+        if not start:
+            current_app.logger.info('No start timestamp: skipping update.')
+            return
+
+        try:
+            if not end:
+                update_statement = text(UPDATE_BATCH_JOB_DATE_START)
+                db.session.execute(update_statement, {'job_id': batch_job_id, 'query_start': start})
+            else:
+                update_statement = text(UPDATE_BATCH_JOB_DATE_RANGE)
+                db.session.execute(update_statement, {'job_id': batch_job_id, 'query_start': start, 'query_end': end})
+            db.session.commit()
+        except Exception as db_exception:   # noqa: B902; return nicer error
+            current_app.logger.error('DB save_job_id exception: ' + str(db_exception))
+            raise DatabaseException(db_exception)

ppr-api/src/ppr_api/reports/v2/report_utils.py

@@ -684,7 +684,27 @@ def is_order_unchanged(order1: dict, order2: dict) -> bool:
         (order1.get('effectOfOrder') == order2.get('effectOfOrder'))
 
 
-def set_modified_order(add_notice: dict, del_notice: dict):
+def order_exists(order: dict, del_notice: dict) -> bool:
+    """Determine if order in added Securities Act Notice is unchanged."""
+    if not order or not del_notice:
+        return False
+    for del_order in del_notice.get('securitiesActOrders'):
+        if is_order_unchanged(order, del_order):
+            return True
+    return False
+
+
+def order_deleted(del_order: dict, add_notice: dict) -> bool:
+    """Determine if a order in a deleted Securities Act Notice is deleted and not amended/edited."""
+    for add_order in add_notice.get('securitiesActOrders'):
+        if add_order.get('amendOrderId') and add_order.get('amendOrderId') == del_order.get('orderId'):
+            return False
+        if is_order_unchanged(del_order, add_order):
+            return False
+    return True
+
+
+def set_modified_order(add_notice: dict, del_notice: dict):  # pylint: disable=too-many-branches; 1 more
     """Conditionally set amended notice order unchanged flag."""
     for add_order in add_notice.get('securitiesActOrders'):
         if add_order.get('amendOrderId'):
@@ -693,8 +713,16 @@ def set_modified_order(add_notice: dict, del_notice: dict):
                         is_order_unchanged(add_order, del_order):
                     # If identical mark as unchanged
                     add_order['unchanged'] = True
+        # Check if an identical order exists that is not linked by the UI
+        elif order_exists(add_order, del_notice):
+            add_order['unchanged'] = True
     # Amended notice deleted orders added to new orders but marked as deleted.
     merged_orders = []
+    for del_order in del_notice.get('securitiesActOrders'):
+        if order_deleted(del_order, add_notice):
+            order = copy.deepcopy(del_order)
+            order['amendDeleted'] = True
+            merged_orders.append(order)
     for add_order in add_notice.get('securitiesActOrders'):
         if add_order.get('amendOrderId'):
             for del_order in del_notice.get('securitiesActOrders'):

ppr-api/src/ppr_api/resources/v1/callbacks.py

@@ -31,6 +31,7 @@
                url_prefix='/api/v1/callbacks')
 START_TS_PARAM = 'startDateTime'
 END_TS_PARAM = 'endDateTime'
+JOB_ID_PARAM = 'jobId'
 
 
 @bp.route('/mail-report', methods=['POST', 'OPTIONS'])
@@ -82,6 +83,7 @@ def get_mail_list():
     """Fetch recent event storage names by request parameter startDateTime and optional endDateTime."""
     start_ts = request.args.get(START_TS_PARAM, None)
     end_ts = request.args.get(END_TS_PARAM, None)
+    job_id = request.args.get(JOB_ID_PARAM, None)
     try:
         # Authenticate with request api key
         if not resource_utils.valid_api_key(request):
@@ -108,7 +110,7 @@ def get_mail_list():
 
         if message:
             return resource_utils.error_response(status, message)
-        results = MailReport.find_list_by_timestamp(start, end)
+        results = MailReport.find_list_by_timestamp(start, end, job_id)
         return jsonify(results), HTTPStatus.OK
     except DatabaseException as db_exception:
         return resource_utils.db_exception_response(db_exception,

ppr-api/src/ppr_api/version.py

@@ -22,4 +22,4 @@
 Development release segment: .devN
 """
 
-__version__ = '1.2.5'  # pylint: disable=invalid-name
+__version__ = '1.2.6'  # pylint: disable=invalid-name

ppr-api/tests/unit/api/test_callback.py

@@ -48,6 +48,7 @@
     ('Invalid range', HTTPStatus.BAD_REQUEST, '2023-01-31T00:00:01-08:00', '2023-01-30T00:00:01-08:00'),
     ('Invalid end ts', HTTPStatus.BAD_REQUEST, None, '2023-01-31TXX:00:01-08:00'),
     ('Valid start', HTTPStatus.OK, '2023-01-31T00:00:01-08:00', None),
+    ('Valid start with job id', HTTPStatus.OK, '2023-01-31T00:00:01-08:00', None),
     ('Unauthorized', HTTPStatus.UNAUTHORIZED, None, None)
 ]
 
@@ -92,6 +93,8 @@ def test_list_mail_report(session, client, jwt, desc, status, start_ts, end_ts):
             params += f'&endDateTime={end_ts}'
     elif end_ts:
         params += f'?endDateTime={end_ts}'
+    if desc == 'Valid start with job id':
+        params += '&jobId=1234'
 
     headers = None
     if status != HTTPStatus.UNAUTHORIZED:
@@ -112,3 +115,5 @@ def test_list_mail_report(session, client, jwt, desc, status, start_ts, end_ts):
             assert result.get('id')
             assert result.get('dateTime')
             assert result.get('docStorageRef')
+            if desc == 'Valid start with job id':
+                assert result.get('jobId')

ppr-api/tests/unit/models/test_mail_report.py

@@ -33,11 +33,17 @@
     (True, 200000000, 200000004, 200000013, True, False),
     (True, 200000002, 200000008, 200000023, False, True)
 ]
-# testdata pattern is ({desc}, {has_data}, {start_ts}, {end_ts})
+# testdata pattern is ({desc}, {has_data}, {start_ts}, {end_ts}, job_id)
 TEST_MAIL_LIST_DATA = [
-    ('Valid start', True, '2023-02-08T00:00:01-08:00', None),
-    ('Valid range', True, '2023-02-08T00:00:01-08:00', None),
-    ('Valid range no data', False, '2023-02-01T00:00:01-08:00', '2023-02-03T00:00:01-08:00')
+    ('Valid start', True, '2023-02-08T00:00:01-08:00', None, None),
+    ('Valid range', True, '2023-02-08T00:00:01-08:00', None, None),
+    ('Valid range with job id', True, '2023-02-08T00:00:01-08:00', None, 1234),
+    ('Valid range no data', False, '2023-02-01T00:00:01-08:00', '2023-02-03T00:00:01-08:00', None)
+]
+# testdata pattern is ({desc}, {start_ts}, job_id)
+TEST_MAIL_LIST_JOB_DATA = [
+    ('Valid start', '2023-02-08T00:00:01-08:00', 1234),
+    ('Valid range', '2023-02-08T00:00:01-08:00', 1234)
 ]
 
 
@@ -143,7 +149,8 @@ def test_mail_report_json(session):
         party_id=3000,
         report_data=json.dumps(TEST_REPORT_DATA),
         doc_storage_url='http%3A%2F%2Fmocktarget.apigee.net',
-        status=200
+        status=200,
+        batch_job_id=1234
     )
     report_json = {
         'id': mail_report.id,
@@ -153,28 +160,44 @@ def test_mail_report_json(session):
         'reportData': mail_report.report_data,
         'documentStorageURL': mail_report.doc_storage_url,
         'retryCount': 0,
-        'status': 200,
-        'message': ''
+        'status': mail_report.status,
+        'message': '',
+        'jobId': mail_report.batch_job_id
     }
     assert mail_report.json == report_json
 
 
-@pytest.mark.parametrize('desc,has_data,start_ts,end_ts', TEST_MAIL_LIST_DATA)
-def test_find_list_by_timestamp(session, desc, has_data, start_ts, end_ts):
+@pytest.mark.parametrize('desc,has_data,start_ts,end_ts,job_id', TEST_MAIL_LIST_DATA)
+def test_find_list_by_timestamp(session, desc, has_data, start_ts, end_ts, job_id):
     start = None
     end = None
     if start_ts:
         start = ts_from_iso_format(start_ts)
-    if desc == 'Valid range':
+    if desc in ('Valid range', 'Valid range with job id'):
         end = now_ts()
     elif end_ts:
         end = ts_from_iso_format(end_ts)
-    list_json = MailReport.find_list_by_timestamp(start, end)
+    list_json = MailReport.find_list_by_timestamp(start, end, job_id)
     if has_data:
         assert list_json
         for result in list_json:
             assert result.get('id')
             assert result.get('dateTime')
             assert result.get('docStorageRef')
+            if job_id and job_id > 0:
+                assert result.get('jobId') == job_id
+            else:
+                assert 'jobId' not in result
     else:
         assert not list_json
+
+
+@pytest.mark.parametrize('desc,start_ts,job_id', TEST_MAIL_LIST_JOB_DATA)
+def test_list_job_update(session, desc, start_ts, job_id):
+    start = None
+    end = None
+    if start_ts:
+        start = ts_from_iso_format(start_ts)
+    if desc == 'Valid range':
+        end = now_ts()
+    MailReport.save_job_id(start, end, job_id)