From 2346f24fcc3181e8346eb16d40f8fb91b36d78c6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 19:43:04 +0300 Subject: [PATCH 01/10] Update auth service staging image tag to stage-6631dc02-1729615305 --- k8s/auth-service/values-stage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/auth-service/values-stage.yaml b/k8s/auth-service/values-stage.yaml index 1aec9ce57c..1a4190d1b4 100644 --- a/k8s/auth-service/values-stage.yaml +++ b/k8s/auth-service/values-stage.yaml @@ -6,7 +6,7 @@ app: replicaCount: 2 image: repository: eu.gcr.io/airqo-250220/airqo-stage-auth-api - tag: stage-b55d4116-1729602971 + tag: stage-6631dc02-1729615305 nameOverride: '' fullnameOverride: '' podAnnotations: {} From cc9a5ba9470ca4f60157b608e0868933d765473a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 19:43:44 +0300 Subject: [PATCH 02/10] Update AirQo exceedance production image tag to prod-65aa1824-1729615373 --- k8s/exceedance/values-prod-airqo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/exceedance/values-prod-airqo.yaml b/k8s/exceedance/values-prod-airqo.yaml index d571cf40c4..b7a11b30eb 100644 --- a/k8s/exceedance/values-prod-airqo.yaml +++ b/k8s/exceedance/values-prod-airqo.yaml @@ -4,6 +4,6 @@ app: configmap: env-exceedance-production image: repository: eu.gcr.io/airqo-250220/airqo-exceedance-job - tag: prod-d58d02be-1729614498 + tag: prod-65aa1824-1729615373 nameOverride: '' fullnameOverride: '' From 8e99b4399a7fcb5c8efd7c02583efefeb718d510 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 19:43:51 +0300 Subject: [PATCH 03/10] Update KCCA exceedance production image tag to prod-65aa1824-1729615373 --- k8s/exceedance/values-prod-kcca.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/exceedance/values-prod-kcca.yaml b/k8s/exceedance/values-prod-kcca.yaml index 23b95b6ef2..90e974b3e0 100644 --- a/k8s/exceedance/values-prod-kcca.yaml +++ b/k8s/exceedance/values-prod-kcca.yaml @@ -4,6 +4,6 @@ app: configmap: env-exceedance-production image: repository: eu.gcr.io/airqo-250220/kcca-exceedance-job - tag: prod-d58d02be-1729614498 + tag: prod-65aa1824-1729615373 nameOverride: '' fullnameOverride: '' From 867df4ee1f7b21ca8ba1e83dd3adf19da3e17808 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 19:44:32 +0300 Subject: [PATCH 04/10] Update device registry production image tag to prod-65aa1824-1729615373 --- k8s/device-registry/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/device-registry/values-prod.yaml b/k8s/device-registry/values-prod.yaml index 848c8c0153..fe94a71bc5 100644 --- a/k8s/device-registry/values-prod.yaml +++ b/k8s/device-registry/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-device-registry-api - tag: prod-d58d02be-1729614498 + tag: prod-65aa1824-1729615373 nameOverride: '' fullnameOverride: '' podAnnotations: {} From c14fff61d9397ca602034000c7c3c9454f83626c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 19:44:40 +0300 Subject: [PATCH 05/10] Update auth service production image tag to prod-65aa1824-1729615373 --- k8s/auth-service/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/auth-service/values-prod.yaml b/k8s/auth-service/values-prod.yaml index d924926178..d782478c8f 100644 --- a/k8s/auth-service/values-prod.yaml +++ b/k8s/auth-service/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-auth-api - tag: prod-d58d02be-1729614498 + tag: prod-65aa1824-1729615373 nameOverride: '' fullnameOverride: '' podAnnotations: {} From 67e09e5b019064e8625af8c6fa4c081c2c1fad69 Mon Sep 17 00:00:00 2001 From: NicholasTurner23 Date: Tue, 22 Oct 2024 19:45:21 +0300 Subject: [PATCH 06/10] Clean up --- src/workflows/airqo_etl_utils/message_broker_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/workflows/airqo_etl_utils/message_broker_utils.py b/src/workflows/airqo_etl_utils/message_broker_utils.py index 2a505ca624..1fa573e0d8 100644 --- a/src/workflows/airqo_etl_utils/message_broker_utils.py +++ b/src/workflows/airqo_etl_utils/message_broker_utils.py @@ -336,9 +336,9 @@ def publish_to_topic( logger.info(f"Preparing to publish data to topic: {topic}") data.replace(np.nan, None, inplace=True) dataframe_list = data.to_dict("records") + message_counts = 0 if column_key: logger.info(f"Using '{column_key}' as the key for messages") - message_counts = 0 for row in dataframe_list: key = row.pop(column_key, None) if key is None: @@ -351,14 +351,15 @@ def publish_to_topic( selected_partition = ( None if auto_partition else self.__get_least_loaded_partition() ) - message_counts += 1 self._send_message(producer, topic, key, message, selected_partition) if not auto_partition: self.partition_loads[selected_partition] += 1 + message_counts += 1 else: logger.info("No key provided, splitting data into chunks and publishing") for chunk_data in self._generate_chunks(dataframe_list): + message_counts += len(chunk_data) message = json.dumps({"data": chunk_data}).encode("utf-8") selected_partition = ( From 6757a1dd9c89f95af175fe7dcacdedb7e66cd89f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 19:45:33 +0300 Subject: [PATCH 07/10] Update analytics production image tag to prod-65aa1824-1729615373 --- k8s/analytics/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/analytics/values-prod.yaml b/k8s/analytics/values-prod.yaml index a2983a6786..f2fab2ee23 100644 --- a/k8s/analytics/values-prod.yaml +++ b/k8s/analytics/values-prod.yaml @@ -8,7 +8,7 @@ images: celeryWorker: eu.gcr.io/airqo-250220/airqo-analytics-celery-worker reportJob: eu.gcr.io/airqo-250220/airqo-analytics-report-job devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-analytics-devices-summary-job - tag: prod-d58d02be-1729614498 + tag: prod-65aa1824-1729615373 api: name: airqo-analytics-api label: analytics-api From e33d351344c0aea65b2c701f080480ff01a4cc16 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 19:45:40 +0300 Subject: [PATCH 08/10] Update workflows prod image tag to prod-65aa1824-1729615373 --- k8s/workflows/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/workflows/values-prod.yaml b/k8s/workflows/values-prod.yaml index e98cc097ce..56d67c12c8 100644 --- a/k8s/workflows/values-prod.yaml +++ b/k8s/workflows/values-prod.yaml @@ -10,7 +10,7 @@ images: initContainer: eu.gcr.io/airqo-250220/airqo-workflows-xcom redisContainer: eu.gcr.io/airqo-250220/airqo-redis containers: eu.gcr.io/airqo-250220/airqo-workflows - tag: prod-d58d02be-1729614498 + tag: prod-65aa1824-1729615373 nameOverride: '' fullnameOverride: '' podAnnotations: {} From 8595bb8ae6171502ac2b827e74d5c64325d7ee48 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 19:45:54 +0300 Subject: [PATCH 09/10] Update spatial production image tag to prod-65aa1824-1729615373 --- k8s/spatial/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/spatial/values-prod.yaml b/k8s/spatial/values-prod.yaml index f2ad886421..0f116a5c75 100644 --- a/k8s/spatial/values-prod.yaml +++ b/k8s/spatial/values-prod.yaml @@ -6,7 +6,7 @@ app: replicaCount: 3 image: repository: eu.gcr.io/airqo-250220/airqo-spatial-api - tag: prod-d58d02be-1729614498 + tag: prod-65aa1824-1729615373 nameOverride: '' fullnameOverride: '' podAnnotations: {} From 69fbbccdcd2da058d487b82a38038cd413c1ad49 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 19:46:18 +0300 Subject: [PATCH 10/10] Update predict production image tag to prod-65aa1824-1729615373 --- k8s/predict/values-prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/predict/values-prod.yaml b/k8s/predict/values-prod.yaml index fe566878d7..d5f989352d 100644 --- a/k8s/predict/values-prod.yaml +++ b/k8s/predict/values-prod.yaml @@ -7,7 +7,7 @@ images: predictJob: eu.gcr.io/airqo-250220/airqo-predict-job trainJob: eu.gcr.io/airqo-250220/airqo-train-job predictPlaces: eu.gcr.io/airqo-250220/airqo-predict-places-air-quality - tag: prod-d58d02be-1729614498 + tag: prod-65aa1824-1729615373 api: name: airqo-prediction-api label: prediction-api