From 93e1f1721b321ac4f7166a6a8f2e9571d86c9066 Mon Sep 17 00:00:00 2001 From: mhmdksh Date: Tue, 5 Nov 2024 16:44:58 +0400 Subject: [PATCH] Updating the Zero-Deployment Steps in Pipelines --- .github/workflows/main-deploy-only.yml | 86 +++++++++++++++-------- .github/workflows/main-pipeline.yml | 86 +++++++++++++++-------- .github/workflows/main-publish-deploy.yml | 86 +++++++++++++++-------- .github/workflows/staging-deploy-only.yml | 86 +++++++++++++++-------- .github/workflows/staging-pipeline.yml | 86 +++++++++++++++-------- 5 files changed, 275 insertions(+), 155 deletions(-) diff --git a/.github/workflows/main-deploy-only.yml b/.github/workflows/main-deploy-only.yml index abee80812..25bf043b4 100644 --- a/.github/workflows/main-deploy-only.yml +++ b/.github/workflows/main-deploy-only.yml @@ -24,49 +24,73 @@ jobs: ## Update each backend service one by one ## First Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=0 qacc-be-graph-ql2=1 qacc-be-graph-ql3=1 - docker compose up -d --no-deps --scale qacc-be-job1=0 qacc-be-job2=1 qacc-be-job3=1 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=0 --scale qacc-be-graph-ql2=1 --scale qacc-be-graph-ql3=1 + docker compose up -d --no-deps --scale qacc-be-job1=0 --scale qacc-be-job2=1 --scale qacc-be-job3=1 docker compose up -d - # Check the health of qacc-be-graph-ql1 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql1)" != "\"running\"" ]; then - echo "qacc-be-graph-ql1 is not running, stopping deployment" - exit 1 + + # Wait for qacc-be-graph-ql1 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql1 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql1)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql1 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql1 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql1 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql1)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql1 is not healthy, stopping deployment" + exit 1 fi - # Check the health of qacc-be-job1 + # Check if qacc-be-job1 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job1)" != "\"running\"" ]; then - echo "qacc-be-job1 is not running, stopping deployment" - exit 1 + echo "qacc-be-job1 is not running, stopping deployment" + exit 1 fi + echo "First deployment phase completed successfully" ## Second Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 qacc-be-graph-ql2=0 qacc-be-graph-ql3=1 - docker compose up -d --no-deps --scale qacc-be-job1=1 qacc-be-job2=0 qacc-be-job3=1 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 --scale qacc-be-graph-ql2=0 --scale qacc-be-graph-ql3=1 + docker compose up -d --no-deps --scale qacc-be-job1=1 --scale qacc-be-job2=0 --scale qacc-be-job3=1 docker compose up -d - # Check the health of qacc-be-graph-ql2 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql2)" != "\"running\"" ]; then - echo "qacc-be-graph-ql2 is not running, stopping deployment" - exit 1 - fi - # Check the health of qacc-be-job2 + # Wait for qacc-be-graph-ql2 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql2 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql2)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql2 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql2 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql2 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql2)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql2 is not healthy, stopping deployment" + exit 1 + fi + # Check if qacc-be-job2 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job2)" != "\"running\"" ]; then - echo "qacc-be-job2 is not running, stopping deployment" - exit 1 + echo "qacc-be-job2 is not running, stopping deployment" + exit 1 fi - + echo "Second deployment phase completed successfully" ## Third Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 qacc-be-graph-ql2=1 qacc-be-graph-ql3=0 - docker compose up -d --no-deps --scale qacc-be-job1=1 qacc-be-job2=1 qacc-be-job3=0 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 --scale qacc-be-graph-ql2=1 --scale qacc-be-graph-ql3=0 + docker compose up -d --no-deps --scale qacc-be-job1=1 --scale qacc-be-job2=1 --scale qacc-be-job3=0 docker compose up -d - # Check the health of qacc-be-graph-ql3 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql3)" != "\"running\"" ]; then - echo "qacc-be-graph-ql3 is not running, stopping deployment" - exit 1 - fi - # Check the health of qacc-be-job3 + # Wait for qacc-be-graph-ql3 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql3 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql3)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql3 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql3 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql3 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql3)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql3 is not healthy, stopping deployment" + exit 1 + fi + # Check if qacc-be-job3 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job3)" != "\"running\"" ]; then - echo "qacc-be-job3 is not running, stopping deployment" - exit 1 - fi \ No newline at end of file + echo "qacc-be-job3 is not running, stopping deployment" + exit 1 + fi + echo "First deployment phase completed successfully" \ No newline at end of file diff --git a/.github/workflows/main-pipeline.yml b/.github/workflows/main-pipeline.yml index c7670e062..3c0b2a77e 100644 --- a/.github/workflows/main-pipeline.yml +++ b/.github/workflows/main-pipeline.yml @@ -166,49 +166,73 @@ jobs: ## Update each backend service one by one ## First Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=0 qacc-be-graph-ql2=1 qacc-be-graph-ql3=1 - docker compose up -d --no-deps --scale qacc-be-job1=0 qacc-be-job2=1 qacc-be-job3=1 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=0 --scale qacc-be-graph-ql2=1 --scale qacc-be-graph-ql3=1 + docker compose up -d --no-deps --scale qacc-be-job1=0 --scale qacc-be-job2=1 --scale qacc-be-job3=1 docker compose up -d - # Check the health of qacc-be-graph-ql1 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql1)" != "\"running\"" ]; then - echo "qacc-be-graph-ql1 is not running, stopping deployment" - exit 1 + + # Wait for qacc-be-graph-ql1 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql1 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql1)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql1 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql1 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql1 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql1)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql1 is not healthy, stopping deployment" + exit 1 fi - # Check the health of qacc-be-job1 + # Check if qacc-be-job1 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job1)" != "\"running\"" ]; then - echo "qacc-be-job1 is not running, stopping deployment" - exit 1 + echo "qacc-be-job1 is not running, stopping deployment" + exit 1 fi + echo "First deployment phase completed successfully" ## Second Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 qacc-be-graph-ql2=0 qacc-be-graph-ql3=1 - docker compose up -d --no-deps --scale qacc-be-job1=1 qacc-be-job2=0 qacc-be-job3=1 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 --scale qacc-be-graph-ql2=0 --scale qacc-be-graph-ql3=1 + docker compose up -d --no-deps --scale qacc-be-job1=1 --scale qacc-be-job2=0 --scale qacc-be-job3=1 docker compose up -d - # Check the health of qacc-be-graph-ql2 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql2)" != "\"running\"" ]; then - echo "qacc-be-graph-ql2 is not running, stopping deployment" - exit 1 - fi - # Check the health of qacc-be-job2 + # Wait for qacc-be-graph-ql2 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql2 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql2)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql2 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql2 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql2 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql2)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql2 is not healthy, stopping deployment" + exit 1 + fi + # Check if qacc-be-job2 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job2)" != "\"running\"" ]; then - echo "qacc-be-job2 is not running, stopping deployment" - exit 1 + echo "qacc-be-job2 is not running, stopping deployment" + exit 1 fi - + echo "Second deployment phase completed successfully" ## Third Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 qacc-be-graph-ql2=1 qacc-be-graph-ql3=0 - docker compose up -d --no-deps --scale qacc-be-job1=1 qacc-be-job2=1 qacc-be-job3=0 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 --scale qacc-be-graph-ql2=1 --scale qacc-be-graph-ql3=0 + docker compose up -d --no-deps --scale qacc-be-job1=1 --scale qacc-be-job2=1 --scale qacc-be-job3=0 docker compose up -d - # Check the health of qacc-be-graph-ql3 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql3)" != "\"running\"" ]; then - echo "qacc-be-graph-ql3 is not running, stopping deployment" - exit 1 - fi - # Check the health of qacc-be-job3 + # Wait for qacc-be-graph-ql3 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql3 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql3)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql3 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql3 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql3 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql3)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql3 is not healthy, stopping deployment" + exit 1 + fi + # Check if qacc-be-job3 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job3)" != "\"running\"" ]; then - echo "qacc-be-job3 is not running, stopping deployment" - exit 1 - fi \ No newline at end of file + echo "qacc-be-job3 is not running, stopping deployment" + exit 1 + fi + echo "First deployment phase completed successfully" \ No newline at end of file diff --git a/.github/workflows/main-publish-deploy.yml b/.github/workflows/main-publish-deploy.yml index db1e94b7f..dff8bcafe 100644 --- a/.github/workflows/main-publish-deploy.yml +++ b/.github/workflows/main-publish-deploy.yml @@ -71,49 +71,73 @@ jobs: ## Update each backend service one by one ## First Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=0 qacc-be-graph-ql2=1 qacc-be-graph-ql3=1 - docker compose up -d --no-deps --scale qacc-be-job1=0 qacc-be-job2=1 qacc-be-job3=1 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=0 --scale qacc-be-graph-ql2=1 --scale qacc-be-graph-ql3=1 + docker compose up -d --no-deps --scale qacc-be-job1=0 --scale qacc-be-job2=1 --scale qacc-be-job3=1 docker compose up -d - # Check the health of qacc-be-graph-ql1 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql1)" != "\"running\"" ]; then - echo "qacc-be-graph-ql1 is not running, stopping deployment" - exit 1 + + # Wait for qacc-be-graph-ql1 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql1 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql1)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql1 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql1 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql1 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql1)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql1 is not healthy, stopping deployment" + exit 1 fi - # Check the health of qacc-be-job1 + # Check if qacc-be-job1 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job1)" != "\"running\"" ]; then - echo "qacc-be-job1 is not running, stopping deployment" - exit 1 + echo "qacc-be-job1 is not running, stopping deployment" + exit 1 fi + echo "First deployment phase completed successfully" ## Second Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 qacc-be-graph-ql2=0 qacc-be-graph-ql3=1 - docker compose up -d --no-deps --scale qacc-be-job1=1 qacc-be-job2=0 qacc-be-job3=1 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 --scale qacc-be-graph-ql2=0 --scale qacc-be-graph-ql3=1 + docker compose up -d --no-deps --scale qacc-be-job1=1 --scale qacc-be-job2=0 --scale qacc-be-job3=1 docker compose up -d - # Check the health of qacc-be-graph-ql2 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql2)" != "\"running\"" ]; then - echo "qacc-be-graph-ql2 is not running, stopping deployment" - exit 1 - fi - # Check the health of qacc-be-job2 + # Wait for qacc-be-graph-ql2 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql2 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql2)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql2 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql2 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql2 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql2)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql2 is not healthy, stopping deployment" + exit 1 + fi + # Check if qacc-be-job2 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job2)" != "\"running\"" ]; then - echo "qacc-be-job2 is not running, stopping deployment" - exit 1 + echo "qacc-be-job2 is not running, stopping deployment" + exit 1 fi - + echo "Second deployment phase completed successfully" ## Third Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 qacc-be-graph-ql2=1 qacc-be-graph-ql3=0 - docker compose up -d --no-deps --scale qacc-be-job1=1 qacc-be-job2=1 qacc-be-job3=0 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 --scale qacc-be-graph-ql2=1 --scale qacc-be-graph-ql3=0 + docker compose up -d --no-deps --scale qacc-be-job1=1 --scale qacc-be-job2=1 --scale qacc-be-job3=0 docker compose up -d - # Check the health of qacc-be-graph-ql3 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql3)" != "\"running\"" ]; then - echo "qacc-be-graph-ql3 is not running, stopping deployment" - exit 1 - fi - # Check the health of qacc-be-job3 + # Wait for qacc-be-graph-ql3 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql3 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql3)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql3 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql3 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql3 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql3)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql3 is not healthy, stopping deployment" + exit 1 + fi + # Check if qacc-be-job3 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job3)" != "\"running\"" ]; then - echo "qacc-be-job3 is not running, stopping deployment" - exit 1 - fi \ No newline at end of file + echo "qacc-be-job3 is not running, stopping deployment" + exit 1 + fi + echo "First deployment phase completed successfully" \ No newline at end of file diff --git a/.github/workflows/staging-deploy-only.yml b/.github/workflows/staging-deploy-only.yml index 86ea47f4b..95e25b33a 100644 --- a/.github/workflows/staging-deploy-only.yml +++ b/.github/workflows/staging-deploy-only.yml @@ -24,49 +24,73 @@ jobs: ## Update each backend service one by one ## First Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=0 qacc-be-graph-ql2=1 qacc-be-graph-ql3=1 - docker compose up -d --no-deps --scale qacc-be-job1=0 qacc-be-job2=1 qacc-be-job3=1 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=0 --scale qacc-be-graph-ql2=1 --scale qacc-be-graph-ql3=1 + docker compose up -d --no-deps --scale qacc-be-job1=0 --scale qacc-be-job2=1 --scale qacc-be-job3=1 docker compose up -d - # Check the health of qacc-be-graph-ql1 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql1)" != "\"running\"" ]; then - echo "qacc-be-graph-ql1 is not running, stopping deployment" - exit 1 + + # Wait for qacc-be-graph-ql1 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql1 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql1)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql1 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql1 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql1 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql1)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql1 is not healthy, stopping deployment" + exit 1 fi - # Check the health of qacc-be-job1 + # Check if qacc-be-job1 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job1)" != "\"running\"" ]; then - echo "qacc-be-job1 is not running, stopping deployment" - exit 1 + echo "qacc-be-job1 is not running, stopping deployment" + exit 1 fi + echo "First deployment phase completed successfully" ## Second Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 qacc-be-graph-ql2=0 qacc-be-graph-ql3=1 - docker compose up -d --no-deps --scale qacc-be-job1=1 qacc-be-job2=0 qacc-be-job3=1 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 --scale qacc-be-graph-ql2=0 --scale qacc-be-graph-ql3=1 + docker compose up -d --no-deps --scale qacc-be-job1=1 --scale qacc-be-job2=0 --scale qacc-be-job3=1 docker compose up -d - # Check the health of qacc-be-graph-ql2 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql2)" != "\"running\"" ]; then - echo "qacc-be-graph-ql2 is not running, stopping deployment" - exit 1 - fi - # Check the health of qacc-be-job2 + # Wait for qacc-be-graph-ql2 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql2 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql2)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql2 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql2 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql2 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql2)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql2 is not healthy, stopping deployment" + exit 1 + fi + # Check if qacc-be-job2 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job2)" != "\"running\"" ]; then - echo "qacc-be-job2 is not running, stopping deployment" - exit 1 + echo "qacc-be-job2 is not running, stopping deployment" + exit 1 fi - + echo "Second deployment phase completed successfully" ## Third Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 qacc-be-graph-ql2=1 qacc-be-graph-ql3=0 - docker compose up -d --no-deps --scale qacc-be-job1=1 qacc-be-job2=1 qacc-be-job3=0 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 --scale qacc-be-graph-ql2=1 --scale qacc-be-graph-ql3=0 + docker compose up -d --no-deps --scale qacc-be-job1=1 --scale qacc-be-job2=1 --scale qacc-be-job3=0 docker compose up -d - # Check the health of qacc-be-graph-ql3 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql3)" != "\"running\"" ]; then - echo "qacc-be-graph-ql3 is not running, stopping deployment" - exit 1 - fi - # Check the health of qacc-be-job3 + # Wait for qacc-be-graph-ql3 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql3 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql3)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql3 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql3 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql3 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql3)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql3 is not healthy, stopping deployment" + exit 1 + fi + # Check if qacc-be-job3 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job3)" != "\"running\"" ]; then - echo "qacc-be-job3 is not running, stopping deployment" - exit 1 - fi \ No newline at end of file + echo "qacc-be-job3 is not running, stopping deployment" + exit 1 + fi + echo "First deployment phase completed successfully" \ No newline at end of file diff --git a/.github/workflows/staging-pipeline.yml b/.github/workflows/staging-pipeline.yml index 85eeae2c6..4c85a159c 100644 --- a/.github/workflows/staging-pipeline.yml +++ b/.github/workflows/staging-pipeline.yml @@ -166,49 +166,73 @@ jobs: ## Update each backend service one by one ## First Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=0 qacc-be-graph-ql2=1 qacc-be-graph-ql3=1 - docker compose up -d --no-deps --scale qacc-be-job1=0 qacc-be-job2=1 qacc-be-job3=1 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=0 --scale qacc-be-graph-ql2=1 --scale qacc-be-graph-ql3=1 + docker compose up -d --no-deps --scale qacc-be-job1=0 --scale qacc-be-job2=1 --scale qacc-be-job3=1 docker compose up -d - # Check the health of qacc-be-graph-ql1 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql1)" != "\"running\"" ]; then - echo "qacc-be-graph-ql1 is not running, stopping deployment" - exit 1 + + # Wait for qacc-be-graph-ql1 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql1 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql1)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql1 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql1 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql1 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql1)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql1 is not healthy, stopping deployment" + exit 1 fi - # Check the health of qacc-be-job1 + # Check if qacc-be-job1 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job1)" != "\"running\"" ]; then - echo "qacc-be-job1 is not running, stopping deployment" - exit 1 + echo "qacc-be-job1 is not running, stopping deployment" + exit 1 fi + echo "First deployment phase completed successfully" ## Second Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 qacc-be-graph-ql2=0 qacc-be-graph-ql3=1 - docker compose up -d --no-deps --scale qacc-be-job1=1 qacc-be-job2=0 qacc-be-job3=1 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 --scale qacc-be-graph-ql2=0 --scale qacc-be-graph-ql3=1 + docker compose up -d --no-deps --scale qacc-be-job1=1 --scale qacc-be-job2=0 --scale qacc-be-job3=1 docker compose up -d - # Check the health of qacc-be-graph-ql2 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql2)" != "\"running\"" ]; then - echo "qacc-be-graph-ql2 is not running, stopping deployment" - exit 1 - fi - # Check the health of qacc-be-job2 + # Wait for qacc-be-graph-ql2 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql2 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql2)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql2 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql2 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql2 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql2)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql2 is not healthy, stopping deployment" + exit 1 + fi + # Check if qacc-be-job2 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job2)" != "\"running\"" ]; then - echo "qacc-be-job2 is not running, stopping deployment" - exit 1 + echo "qacc-be-job2 is not running, stopping deployment" + exit 1 fi - + echo "Second deployment phase completed successfully" ## Third Deployment - docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 qacc-be-graph-ql2=1 qacc-be-graph-ql3=0 - docker compose up -d --no-deps --scale qacc-be-job1=1 qacc-be-job2=1 qacc-be-job3=0 + docker compose up -d --no-deps --scale qacc-be-graph-ql1=1 --scale qacc-be-graph-ql2=1 --scale qacc-be-graph-ql3=0 + docker compose up -d --no-deps --scale qacc-be-job1=1 --scale qacc-be-job2=1 --scale qacc-be-job3=0 docker compose up -d - # Check the health of qacc-be-graph-ql3 - if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-graph-ql3)" != "\"running\"" ]; then - echo "qacc-be-graph-ql3 is not running, stopping deployment" - exit 1 - fi - # Check the health of qacc-be-job3 + # Wait for qacc-be-graph-ql3 to be healthy (timeout after 5 minutes) + echo "Waiting for qacc-be-graph-ql3 to become healthy..." + timeout 300 bash -c 'until [ "$(docker inspect --format="{{json .State.Health.Status}}" qacc-be-graph-ql3)" == "\"healthy\"" ]; do echo "Waiting for qacc-be-graph-ql3 to be healthy..."; sleep 5; done' + if [ $? -eq 124 ]; then + echo "Timeout waiting for qacc-be-graph-ql3 to become healthy" + exit 1 + fi + # Check if qacc-be-graph-ql3 is healthy + if [ "$(docker inspect --format='{{json .State.Health.Status}}' qacc-be-graph-ql3)" != "\"healthy\"" ]; then + echo "qacc-be-graph-ql3 is not healthy, stopping deployment" + exit 1 + fi + # Check if qacc-be-job3 is running if [ "$(docker inspect --format='{{json .State.Status}}' qacc-be-job3)" != "\"running\"" ]; then - echo "qacc-be-job3 is not running, stopping deployment" - exit 1 - fi \ No newline at end of file + echo "qacc-be-job3 is not running, stopping deployment" + exit 1 + fi + echo "First deployment phase completed successfully" \ No newline at end of file