From 1d3571bd99c776429b18ee47f7f425a5678bb5f8 Mon Sep 17 00:00:00 2001 From: wojciechsromek <157375010+wojciechsromek@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:13:40 +0200 Subject: [PATCH] [POP-1861] Provide scripts to reset the queues and dbs in the SMPCv2 stage (#353) * feat: Provide scripts to reset the queues and dbs in the SMPCv2 stage * add accounts checker --- scripts/README.md | 37 ++++++++++ scripts/accounts_checker.sh | 36 ++++++++++ .../db-cleaner-helper-pod.yaml | 13 +--- scripts/purge_stage_dbs.sh | 63 +++++++++++++++++ scripts/purge_stage_queues.sh | 68 +++++++++++++++++++ 5 files changed, 207 insertions(+), 10 deletions(-) create mode 100644 scripts/README.md create mode 100644 scripts/accounts_checker.sh rename {deploy => scripts}/db-cleaner-helper-pod.yaml (55%) create mode 100755 scripts/purge_stage_dbs.sh create mode 100755 scripts/purge_stage_queues.sh diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 000000000..eddb6ba08 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,37 @@ +## SMPCv2 Databases & Queues Purging + +To make sure the scripts included in this repository are working properly, you need to have the following AWS profiles defined in your environment: + +### SMPCv2 +```yaml +[profile worldcoin-iam] +sso_start_url = https://worldcoin.awsapps.com/start +sso_region = us-east-1 +sso_account_id = 033662022620 +sso_role_name = AssumeSMPCV2Role + +[profile worldcoin-smpcv2-vpc] +source_profile=worldcoin-iam +role_arn=arn:aws:iam::590183936500:role/smpcv2-cross-account-role + +[profile worldcoin-smpcv2-1] +source_profile=worldcoin-iam +role_arn=arn:aws:iam::767397983205:role/smpcv2-cross-account-role + +[profile worldcoin-smpcv2-2] +source_profile=worldcoin-iam +role_arn=arn:aws:iam::381492197851:role/smpcv2-cross-account-role + +[profile worldcoin-smpcv2-3] +source_profile=worldcoin-iam +role_arn=arn:aws:iam::590184084615:role/smpcv2-cross-account-role +``` + +### Orb +```yaml +[profile worldcoin-stage] +sso_start_url = https://worldcoin.awsapps.com/start +sso_region = us-east-1 +sso_account_id = 510867353226 +sso_role_name = PowerUserAccess +``` diff --git a/scripts/accounts_checker.sh b/scripts/accounts_checker.sh new file mode 100644 index 000000000..cb26ae45a --- /dev/null +++ b/scripts/accounts_checker.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +ORB_STAGE_ACCOUNT_ID="510867353226" +MPC_1_STAGE_ACCOUNT_ID="767397983205" +MPC_2_STAGE_ACCOUNT_ID="381492197851" +MPC_3_STAGE_ACCOUNT_ID="590184084615" + +ACTUAL_ORB_ACCOUNT_ID=$(aws sts get-caller-identity --profile worldcoin-stage --query Account --output text) +if [ "$ACTUAL_ORB_ACCOUNT_ID" != "$ORB_STAGE_ACCOUNT_ID" ]; then + echo "The actual account ID does not match the expected account ID for the 'worldcoin-stage' profile." + echo "$ACTUAL_ORB_ACCOUNT_ID != $ORB_STAGE_ACCOUNT_ID" + exit 1 +fi + +ACTUAL_MPC_1_ACCOUNT_ID=$(aws sts get-caller-identity --profile worldcoin-smpcv2-1 --query Account --output text) +if [ "$ACTUAL_MPC_1_ACCOUNT_ID" != "$MPC_1_STAGE_ACCOUNT_ID" ]; then + echo "The actual account ID does not match the expected account ID for the 'worldcoin-smpcv2-1' profile." + echo "$ACTUAL_MPC_1_ACCOUNT_ID != $MPC_1_STAGE_ACCOUNT_ID" + exit 1 +fi + +ACTUAL_MPC_2_ACCOUNT_ID=$(aws sts get-caller-identity --profile worldcoin-smpcv2-2 --query Account --output text) +if [ "$ACTUAL_MPC_2_ACCOUNT_ID" != "$MPC_2_STAGE_ACCOUNT_ID" ]; then + echo "The actual account ID does not match the expected account ID for the 'worldcoin-smpcv2-2' profile." + echo "$ACTUAL_MPC_2_ACCOUNT_ID != $MPC_2_STAGE_ACCOUNT_ID" + exit 1 +fi + +ACTUAL_MPC_3_ACCOUNT_ID=$(aws sts get-caller-identity --profile worldcoin-smpcv2-3 --query Account --output text) +if [ "$ACTUAL_MPC_3_ACCOUNT_ID" != "$MPC_3_STAGE_ACCOUNT_ID" ]; then + echo "The actual account ID does not match the expected account ID for the 'worldcoin-smpcv2-3' profile." + echo "$ACTUAL_MPC_3_ACCOUNT_ID != $MPC_3_STAGE_ACCOUNT_ID" + exit 1 +fi + +echo "Accounts check succeeded, we are running on staging accounts!" diff --git a/deploy/db-cleaner-helper-pod.yaml b/scripts/db-cleaner-helper-pod.yaml similarity index 55% rename from deploy/db-cleaner-helper-pod.yaml rename to scripts/db-cleaner-helper-pod.yaml index b834ece9d..0ac2f240c 100644 --- a/deploy/db-cleaner-helper-pod.yaml +++ b/scripts/db-cleaner-helper-pod.yaml @@ -2,22 +2,15 @@ apiVersion: v1 kind: Pod metadata: name: db-cleaner - namespace: gpu-iris-mpc + namespace: iris-mpc spec: hostNetwork: true - serviceAccountName: gpu-iris-mpc # Add this line - tolerations: - - key: "dedicated" - operator: "Equal" - value: "gpuGroup" - effect: "NoSchedule" - imagePullSecrets: - - name: github-secret + serviceAccountName: iris-mpc # Add this line securityContext: runAsUser: 0 containers: - name: db-cleaner - image: ghcr.io/worldcoin/gpu-iris-mpc-debug:34b305f6e9acafe9043636fb32fc11870615f34e + image: ubuntu imagePullPolicy: Always command: [ "/bin/bash" ] args: [ "-c", "while true; do ping localhost; sleep 60; done" ] diff --git a/scripts/purge_stage_dbs.sh b/scripts/purge_stage_dbs.sh new file mode 100755 index 000000000..5a81c184c --- /dev/null +++ b/scripts/purge_stage_dbs.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +source accounts_checker.sh + +get_aws_secret() { + local SECRET_NAME=$1 + local REGION=$2 + local SECRET_KEY=$3 + local PROFILE=$4 + + # Fetch the secret value from AWS Secrets Manager + SECRET_VALUE=$(aws secretsmanager get-secret-value --profile "$PROFILE" --secret-id "$SECRET_NAME" --region "$REGION" --query SecretString --output text) + + # Check if the secret value was retrieved successfully + if [ -z "$SECRET_VALUE" ]; then + echo "Failed to retrieve secret: $SECRET_NAME" + exit 1 + fi + + # Extract the specific key (e.g., DATABASE_URL) from the JSON structure + SECRET_KEY_VALUE=$(echo "$SECRET_VALUE" | jq -r ".${SECRET_KEY}") + + if [ -z "$SECRET_KEY_VALUE" ]; then + echo "Failed to retrieve key: $SECRET_KEY from secret: $SECRET_NAME" + exit 1 + fi + + echo "$SECRET_KEY_VALUE" +} + +SECRET_NAME="stage/iris-mpc/rds-master-password" +REGION="eu-north-1" + +MPC_1_DATABASE_URL=$(get_aws_secret "$SECRET_NAME" "$REGION" "DATABASE_URL" "worldcoin-smpcv2-1") +MPC_2_DATABASE_URL=$(get_aws_secret "$SECRET_NAME" "$REGION" "DATABASE_URL" "worldcoin-smpcv2-2") +MPC_3_DATABASE_URL=$(get_aws_secret "$SECRET_NAME" "$REGION" "DATABASE_URL" "worldcoin-smpcv2-3") + +kubectx smpcv2-stage-1 +kubens iris-mpc +kubectl apply -f db-cleaner-helper-pod.yaml +echo "Waiting 10s for db-cleaner pod to be ready..." +sleep 10 +kubectl exec -it db-cleaner -- bash -c "apt update && apt install -y postgresql-client" +kubectl exec -it db-cleaner -- bash -c "psql -H $MPC_1_DATABASE_URL -c 'SET search_path TO \"SMPC_stage_0\"; TRUNCATE irises, results, sync;'" +kubectl delete pod --force db-cleaner + +kubectx smpcv2-stage-2 +kubens iris-mpc +kubectl apply -f db-cleaner-helper-pod.yaml +echo "Waiting 10s for db-cleaner pod to be ready..." +sleep 10 +kubectl exec -it db-cleaner -- bash -c "apt update && apt install -y postgresql-client" +kubectl exec -it db-cleaner -- bash -c "psql -H $MPC_2_DATABASE_URL -c 'SET search_path TO \"SMPC_stage_1\"; TRUNCATE irises, results, sync;'" +kubectl delete pod --force db-cleaner + +kubectx smpcv2-stage-3 +kubens iris-mpc +kubectl apply -f db-cleaner-helper-pod.yaml +echo "Waiting 10s for db-cleaner pod to be ready..." +sleep 10 +kubectl exec -it db-cleaner -- bash -c "apt update && apt install -y postgresql-client" +kubectl exec -it db-cleaner -- bash -c "psql -H $MPC_3_DATABASE_URL -c 'SET search_path TO \"SMPC_stage_2\"; TRUNCATE irises, results, sync;'" +kubectl delete pod --force db-cleaner diff --git a/scripts/purge_stage_queues.sh b/scripts/purge_stage_queues.sh new file mode 100755 index 000000000..78850ed46 --- /dev/null +++ b/scripts/purge_stage_queues.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +source accounts_checker.sh + +purge_queues() { + local PROFILE_NAME=$1 + local REGION=$2 + shift + shift + local QUEUE_NAMES=("$@") + + for QUEUE_NAME in "${QUEUE_NAMES[@]}"; do + # Get the Queue URL from the queue name, using the profile if specified + if [ -n "$PROFILE_NAME" ]; then + QUEUE_URL=$(aws sqs get-queue-url --region "$REGION" --queue-name "$QUEUE_NAME" --output text --query 'QueueUrl' --profile "$PROFILE_NAME") + else + QUEUE_URL=$(aws sqs get-queue-url --region "$REGION" --queue-name "$QUEUE_NAME" --output text --query 'QueueUrl') + fi + + if [ $? -ne 0 ]; then + echo "Failed to get URL for queue: $QUEUE_NAME" + continue + fi + + # Purge the queue + echo "Purging queue: $QUEUE_NAME (URL: $QUEUE_URL)" + if [ -n "$PROFILE_NAME" ]; then + aws sqs purge-queue --region "$REGION" --queue-url "$QUEUE_URL" --profile "$PROFILE_NAME" + else + aws sqs purge-queue --region "$REGION" --queue-url "$QUEUE_URL" + fi + + if [ $? -ne 0 ]; then + echo "Failed to purge queue: $QUEUE_NAME" + else + echo "Successfully purged queue: $QUEUE_NAME" + fi + + sleep 2 + done +} + +ORB_QUEUE_NAMES=( +"iris-mpc-identity-deletion-results-dlq-eu-central-1.fifo" +"iris-mpc-identity-deletion-results-eu-central-1.fifo" +"iris-mpc-results-dlq-eu-central-1.fifo" +"iris-mpc-results-eu-central-1.fifo" +) + +MPC_1_QUEUE_NAMES=( +"mpc1-stage.fifo" +"mpc1-stage-dlq.fifo" +) + +MPC_2_QUEUE_NAMES=( +"mpc2-stage.fifo" +"mpc2-stage-dlq.fifo" +) + +MPC_3_QUEUE_NAMES=( +"mpc3-stage.fifo" +"mpc3-stage-dlq.fifo" +) + +purge_queues "worldcoin-stage" "eu-central-1" "${ORB_QUEUE_NAMES[@]}" +purge_queues "worldcoin-smpcv2-1" "eu-north-1" "${MPC_1_QUEUE_NAMES[@]}" +purge_queues "worldcoin-smpcv2-2" "eu-north-1" "${MPC_2_QUEUE_NAMES[@]}" +purge_queues "worldcoin-smpcv2-3" "eu-north-1" "${MPC_3_QUEUE_NAMES[@]}"