From ad82576ebdfe5bda073d58410aae121139d34cea Mon Sep 17 00:00:00 2001 From: Ioannis Foukarakis Date: Wed, 4 Sep 2024 14:13:12 +0300 Subject: [PATCH 1/5] Chore: improve hightouch sync status readability --- airflow/dags/mattermost_dags/general/_helpers.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/airflow/dags/mattermost_dags/general/_helpers.py b/airflow/dags/mattermost_dags/general/_helpers.py index bf583b5ea..471eb9396 100644 --- a/airflow/dags/mattermost_dags/general/_helpers.py +++ b/airflow/dags/mattermost_dags/general/_helpers.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta from mattermost.operators.mattermost_operator import MattermostOperator +from tabulate import tabulate task_logger = logging.getLogger('airflow.task') @@ -99,13 +100,13 @@ def hightouch_check_syncs(response): status = fail or warning, fail """ syncs = json.loads(''.join(response)) - failed_syncs = {} + failed_syncs = [] try: if ('data' not in syncs) or len(syncs) == 0: raise HightouchApiException('Invalid response from syncs api') - failed_syncs = { - sync['slug']: sync['status'] for sync in syncs.get('data') if sync['status'] not in ('success', 'disabled') - } + failed_syncs = [ + sync for sync in syncs.get('data') if sync['status'] not in ('success', 'disabled') + ] except KeyError as e: task_logger.error('Error in check syncs ...', exc_info=True) raise e @@ -129,7 +130,9 @@ def resolve_hightouch(ti=None, **kwargs): task_logger.info('There are no failed syncs') else: status = ':red_circle:' - message = f"**HIGHTOUCH**: {status}\nFailed syncs: {failed_syncs}" + msg = tabulate([(sync['slug'], sync['status']) for sync in failed_syncs)], headers=['Sync', 'Status'], tablefmt='github') + + message = f"**HIGHTOUCH**: {status}\n{msg}" MattermostOperator(mattermost_conn_id='mattermost', text=message, task_id='resolve_hightouch_message').execute( None ) From caf43b17a3c929a22966dd97c890a95149d0e0d7 Mon Sep 17 00:00:00 2001 From: Ioannis Foukarakis Date: Wed, 4 Sep 2024 14:17:45 +0300 Subject: [PATCH 2/5] Convert to link --- airflow/dags/mattermost_dags/general/_helpers.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/airflow/dags/mattermost_dags/general/_helpers.py b/airflow/dags/mattermost_dags/general/_helpers.py index 471eb9396..de14de1f4 100644 --- a/airflow/dags/mattermost_dags/general/_helpers.py +++ b/airflow/dags/mattermost_dags/general/_helpers.py @@ -104,9 +104,7 @@ def hightouch_check_syncs(response): try: if ('data' not in syncs) or len(syncs) == 0: raise HightouchApiException('Invalid response from syncs api') - failed_syncs = [ - sync for sync in syncs.get('data') if sync['status'] not in ('success', 'disabled') - ] + failed_syncs = [sync for sync in syncs.get('data') if sync['status'] not in ('success', 'disabled')] except KeyError as e: task_logger.error('Error in check syncs ...', exc_info=True) raise e @@ -130,7 +128,14 @@ def resolve_hightouch(ti=None, **kwargs): task_logger.info('There are no failed syncs') else: status = ':red_circle:' - msg = tabulate([(sync['slug'], sync['status']) for sync in failed_syncs)], headers=['Sync', 'Status'], tablefmt='github') + msg = tabulate( + [ + (f"[{sync['slug']}](https://app.hightouch.com/mattermost-com/syncs/{sync['id']})", sync['status']) + for sync in failed_syncs + ], + headers=['Sync', 'Status'], + tablefmt='github', + ) message = f"**HIGHTOUCH**: {status}\n{msg}" MattermostOperator(mattermost_conn_id='mattermost', text=message, task_id='resolve_hightouch_message').execute( From 89a2af4180a8158cf0165eda0244cdd0d1a0c2c9 Mon Sep 17 00:00:00 2001 From: Ioannis Foukarakis Date: Wed, 4 Sep 2024 14:21:45 +0300 Subject: [PATCH 3/5] Fix test --- airflow/tests/general/test_monitoring.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/airflow/tests/general/test_monitoring.py b/airflow/tests/general/test_monitoring.py index af8f94e77..c48094719 100644 --- a/airflow/tests/general/test_monitoring.py +++ b/airflow/tests/general/test_monitoring.py @@ -62,7 +62,12 @@ def test_hightouch_check_syncs_fail(load_data): response = load_data('monitoring/syncs_fail.json') failed_syncs = hightouch_check_syncs(response) # Expected to return dict containing failed loads - assert failed_syncs == {'test_sync_1': 'warning'} + assert failed_syncs == [{ + "id": 99999, + "slug": "test_sync_1", + "status": "warning", + "updatedAt": "2022-12-07T12:59:20Z" + }] def test_hightouch_check_syncs_error(load_data): From dcbc10969350a592014be926dd4aa86af42358e1 Mon Sep 17 00:00:00 2001 From: Ioannis Foukarakis Date: Wed, 4 Sep 2024 14:27:27 +0300 Subject: [PATCH 4/5] Fix test --- airflow/tests/general/test_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/tests/general/test_monitoring.py b/airflow/tests/general/test_monitoring.py index c48094719..79392736c 100644 --- a/airflow/tests/general/test_monitoring.py +++ b/airflow/tests/general/test_monitoring.py @@ -55,7 +55,7 @@ def test_hightouch_check_syncs_pass(load_data): response = load_data('monitoring/syncs_pass.json') failed_syncs = hightouch_check_syncs(response) # Expected to return empty dict as no loads failed - assert failed_syncs == {} + assert failed_syncs == [] def test_hightouch_check_syncs_fail(load_data): From c24d772cfdf1646424c3637e79a2bb1d97f2aa84 Mon Sep 17 00:00:00 2001 From: Ioannis Foukarakis Date: Wed, 4 Sep 2024 14:28:10 +0300 Subject: [PATCH 5/5] Add extra newline --- airflow/dags/mattermost_dags/general/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/dags/mattermost_dags/general/_helpers.py b/airflow/dags/mattermost_dags/general/_helpers.py index de14de1f4..699242f62 100644 --- a/airflow/dags/mattermost_dags/general/_helpers.py +++ b/airflow/dags/mattermost_dags/general/_helpers.py @@ -137,7 +137,7 @@ def resolve_hightouch(ti=None, **kwargs): tablefmt='github', ) - message = f"**HIGHTOUCH**: {status}\n{msg}" + message = f"**HIGHTOUCH**: {status}\n\n{msg}" MattermostOperator(mattermost_conn_id='mattermost', text=message, task_id='resolve_hightouch_message').execute( None )